A straightforward method for training your LLM, from downloading data to generating text.
Improve modern GPU compatibility and document RTX 5090 validation
吴佳浩dfe8db05b1f5
diff --git a/README.md b/README.mdindex c3ccd4e..2292fd5 100644--- a/README.md+++ b/README.md@@ -96,6 +96,7 @@ You will need a GPU to train your model. Colab or Kaggle T4 will work for traini| NVIDIA RTX 4070 Ti | 12 GB | Medium | ✘ | ✔ | ~1.5B || NVIDIA RTX 4080 | 16 GB | Medium | ✘ | ✔ | ~2B || NVIDIA RTX 4090 | 24 GB | Large | ✔ | ✔ | ~4B |+| NVIDIA RTX 5090 | 32 GB* | Large | ✔* | ✔ | 13M verified; larger configs TBD* || NVIDIA RTX 4060 Ti | 8 GB | Small | ✘ | ✔ | ~1B || NVIDIA RTX 4060 | 8 GB | Small | ✘ | ✔ | ~1B || NVIDIA RTX 4050 | 6 GB | Small | ✘ | ✔ | ~0.75B |@@ -112,6 +113,21 @@ You will need a GPU to train your model. Colab or Kaggle T4 will work for trainiThe 13M LLM training is the training of a 13+ million-parameter model, and the 2B LLM training is the training of a 2+ billion-parameter model. The data size is categorized as small, medium, and large. The small data size is around 1 GB, the medium data size is around 5 GB, and the large data size is around 10 GB.+* Community validation on an NVIDIA GeForce RTX 5090 has confirmed that the official README flow works for the 13M configuration using PyTorch 2.11.0+cu128 and CUDA 12.8. In one successful run, the 13M model trained to completion, wrote a checkpoint, and the generated checkpoint was successfully loaded for text generation. Practical limits for larger configurations still depend on training dtype, batch size, context length, optimizer state, and any memory-saving techniques used.++### Community-tested modern GPU notes++A community run on an NVIDIA GeForce RTX 5090 validated the following:+- official dataset download and preprocessing flow completed successfully+- the README-recommended 13M configuration trained successfully to completion+- checkpoint saving and text generation both worked+- PyTorch version: 2.11.0+cu128+- CUDA version: 12.8+- PyTorch-reported peak VRAM during 13M training stayed around 0.60 GiB allocated / 0.67 GiB reserved+- observed training throughput during eval intervals was typically in the tens of thousands of tokens/sec on this small config++The training script now prints lightweight runtime diagnostics to make these reports easier to collect. Newer PyTorch versions may also require explicit checkpoint loading compatibility handling during generation.+## Code StructureThe codebase is organized as follows:diff --git a/config/config.py b/config/config.pyindex ae4bf50..015b731 100644--- a/config/config.py+++ b/config/config.py@@ -1,5 +1,7 @@# --- Configuration ---+import torch+# Define vocabulary size and transformer configuration (3 Billion)VOCAB_SIZE = 50304 # Number of unique tokens in the vocabularyCONTEXT_LENGTH = 512 # Maximum sequence length for the model@@ -23,7 +25,7 @@ T_LR_DECAYED = 5e-5 # Learning rate after decayT_OUT_PATH = "models/transformer_B.pt" # Path to save the trained model# Device configuration-DEVICE = 'cuda'+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'# Store all configurations in a dictionary for easy access and modificationdefault_config = {@@ -44,4 +46,4 @@ default_config = {'t_lr_decayed': T_LR_DECAYED,'t_out_path': T_OUT_PATH,'device': DEVICE,-}\ No newline at end of file+}diff --git a/scripts/generate_text.py b/scripts/generate_text.pyindex f02328e..ce7a4aa 100644--- a/scripts/generate_text.py+++ b/scripts/generate_text.py@@ -4,6 +4,13 @@ import argparsefrom config.config import default_config as configfrom src.models.transformer import Transformer # Assuming your Transformer class is in this module++def load_checkpoint(model_path: str, device: str):+ try:+ return torch.load(model_path, map_location=torch.device(device), weights_only=False)+ except TypeError:+ return torch.load(model_path, map_location=torch.device(device))+def generate_text(model_path: str, input_text: str, max_new_tokens: int = 100, device: str = 'cuda') -> str:"""Generates text using a pre-trained Transformer model.@@ -18,7 +25,7 @@ def generate_text(model_path: str, input_text: str, max_new_tokens: int = 100, dstr: The generated text."""# Load the model checkpoint- checkpoint = torch.load(model_path, map_location=torch.device(device))+ checkpoint = load_checkpoint(model_path, device)# Initialize the model using the configuration from config.pymodel = Transformer(diff --git a/scripts/train_transformer.py b/scripts/train_transformer.pyindex 8cbb14a..2e6f004 100644--- a/scripts/train_transformer.py+++ b/scripts/train_transformer.py@@ -1,6 +1,7 @@import torchimport torch.nn.functional as Fimport os+import timefrom tqdm import tqdmimport numpy as npfrom config.config import default_config as config@@ -8,8 +9,51 @@ from src.models.transformer import Transformerfrom data_loader.data_loader import get_batch_iteratorfrom typing import Dict++def bytes_to_gib(num_bytes: int) -> float:+ return num_bytes / (1024 ** 3)+++def get_device_report(device: str) -> str:+ lines = [+ f"PyTorch version: {torch.__version__}",+ f"Configured device: {device}",+ f"CUDA available: {torch.cuda.is_available()}",+ f"CUDA version: {torch.version.cuda}",+ ]++ if device.startswith('cuda') and torch.cuda.is_available():+ device_index = torch.cuda.current_device()+ props = torch.cuda.get_device_properties(device_index)+ total_vram_gib = bytes_to_gib(props.total_memory)+ lines.extend([+ f"GPU name: {torch.cuda.get_device_name(device_index)}",+ f"GPU capability: {props.major}.{props.minor}",+ f"Total VRAM: {total_vram_gib:.2f} GiB",+ ])+ else:+ lines.append("GPU name: N/A (running without CUDA)")++ return "\n".join(lines)+++def get_peak_memory_report(device: str) -> str:+ if device.startswith('cuda') and torch.cuda.is_available():+ peak_allocated = bytes_to_gib(torch.cuda.max_memory_allocated())+ peak_reserved = bytes_to_gib(torch.cuda.max_memory_reserved())+ return (+ f"Peak VRAM allocated: {peak_allocated:.2f} GiB | "+ f"Peak VRAM reserved: {peak_reserved:.2f} GiB"+ )+ return "Peak VRAM allocated: N/A | Peak VRAM reserved: N/A"++# --- Initialize the Model and Print Parameters ---+print(get_device_report(config['device']))+if config['device'].startswith('cuda') and torch.cuda.is_available():+ torch.cuda.reset_peak_memory_stats()+model = Transformer(n_head=config['n_head'],n_embed=config['n_embed'],@@ -24,61 +68,38 @@ print(f"Total number of parameters in the model: {total_params:,}")# --- Optimizer Setup and Loss Tracking ----# Set up the AdamW optimizer with the specified learning rate.optimizer = torch.optim.AdamW(model.parameters(), lr=config['t_lr'])--# List to track loss values during training.losses = []--# Define a window size for averaging recent losses in the training loop.AVG_WINDOW = 64-# Helper function to estimate the average loss for training and development data.+@torch.no_grad()def estimate_loss(steps: int) -> Dict[str, float]:- """- Evaluate the model on training and development datasets and calculate average loss.-- Args:- steps (int): Number of steps to evaluate.-- Returns:- dict: Dictionary containing average losses for 'train' and 'dev' splits.- """out = {}- model.eval() # Set the model to evaluation mode.+ model.eval()for split in ['train', 'dev']:- # Select the appropriate data path for the current split.data_path = config['train_path'] if split == 'train' else config['dev_path']-- # Create a batch iterator for evaluation.batch_iterator_eval = get_batch_iterator(data_path, config['t_batch_size'], config['t_context_length'], device=config['device'])- # Initialize a tensor to track loss values for each evaluation step.losses_eval = torch.zeros(steps)for k in range(steps):try:- # Fetch a batch and calculate the loss.xb, yb = next(batch_iterator_eval)_, loss = model(xb, yb)losses_eval[k] = loss.item()except StopIteration:- # Handle the case where the data iterator ends early.print(f"Warning: Iterator for {split} ended early.")break- # Compute the mean loss for the current split.out[split] = losses_eval[:k + 1].mean()- model.train() # Restore the model to training mode.+ model.train()return out-# --- Training Loop ----# Create a batch iterator for the training data.batch_iterator = get_batch_iterator(config['train_path'],config['t_batch_size'],@@ -86,53 +107,54 @@ batch_iterator = get_batch_iterator(device=config['device'])-# Create a progress bar to monitor training progress.+tokens_per_step = config['t_batch_size'] * config['t_context_length']+last_eval_time = time.perf_counter()+pbar = tqdm(range(config['t_train_steps']))for step in pbar:try:- # Fetch a batch of input and target data.+ step_start_time = time.perf_counter()xb, yb = next(batch_iterator)-- # Perform a forward pass and compute the loss._, loss = model(xb, yb)- # Record the loss for tracking.losses.append(loss.item())pbar.set_description(f"Train loss: {np.mean(losses[-AVG_WINDOW:]):.4f}")- # Backpropagate the loss and update the model parameters.optimizer.zero_grad(set_to_none=True)loss.backward()optimizer.step()- # Periodically evaluate the model on training and development data.+ step_time = time.perf_counter() - step_start_time+ tokens_per_second = tokens_per_step / step_time if step_time > 0 else float('inf')+if step % config['t_eval_steps'] == 0:evaluation_losses = estimate_loss(config['t_eval_iters'])train_loss = evaluation_losses['train']dev_loss = evaluation_losses['dev']- print(f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")+ now = time.perf_counter()+ elapsed_since_eval = now - last_eval_time+ last_eval_time = now+ print(+ f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}, "+ f"Step time: {step_time:.3f}s, Throughput: {tokens_per_second:.2f} tokens/s, "+ f"Elapsed since last eval: {elapsed_since_eval:.2f}s"+ )+ print(get_peak_memory_report(config['device']))- # Decay the learning rate at the specified step.if step == config['t_lr_decay_step']:print('Decaying learning rate')for g in optimizer.param_groups:g['lr'] = config['t_lr_decayed']except StopIteration:- # Handle the case where the training data iterator ends early.print("Training data iterator finished early.")break-# --- Save Model and Final Evaluation -----# Create the output directory if it does not exist.os.makedirs(config['t_out_path'].split('/')[0], exist_ok=True)-# Perform a final evaluation of the model on training and development datasets.evaluation_losses = estimate_loss(200)train_loss = evaluation_losses['train']dev_loss = evaluation_losses['dev']-# Ensure unique model save path in case the file already exists.modified_model_out_path = config['t_out_path']save_tries = 0while os.path.exists(modified_model_out_path):@@ -140,7 +162,6 @@ while os.path.exists(modified_model_out_path):model_out_name = os.path.splitext(config['t_out_path'])[0]modified_model_out_path = model_out_name + f"_{save_tries}" + ".pt"-# Save the model's state dictionary, optimizer state, and training metadata.torch.save({'model_state_dict': model.state_dict(),@@ -149,8 +170,12 @@ torch.save('train_loss': train_loss,'dev_loss': dev_loss,'steps': len(losses),+ 'device': config['device'],+ 'pytorch_version': torch.__version__,+ 'cuda_version': torch.version.cuda,},modified_model_out_path)print(f"Saved model to {modified_model_out_path}")-print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")\ No newline at end of file+print(get_peak_memory_report(config['device']))+print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")