Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

Revert "Fix attention scaling factor"

Fareed Khan8e1fbb78a29a
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 54e6782..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-__pycache__/
-data/
\ No newline at end of file
diff --git a/PITFALLS.md b/PITFALLS.md
deleted file mode 100644
index 37d8362..0000000
--- a/PITFALLS.md
+++ /dev/null
@@ -1,7 +0,0 @@
-- No bias in Q/K/V
-- `attn = torch.tril(attn, diagonal=0) + torch.triu(torch.ones_like(attn) * (-torch.inf), diagonal=1)`
-- `k = torch.transpose(k, -1, -2)`
-- `x = x + self.attn(self.ln_attn(x)); x = x + self.mlp(self.ln_mlp(x))`
-- `x = self.lm_head(self.ln_lmhead(x))`
-- Send model & all tensors to device
-- Use `x = x + ...` not `x += ...` !
\ No newline at end of file
diff --git a/config/__init__.py b/config/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/config/config.py b/config/config.py
index 035396e..ae4bf50 100644
--- a/config/config.py
+++ b/config/config.py
@@ -1,15 +1,11 @@
# --- Configuration ---
-# Define vocabulary size and transformer configuration (0.8 Billion)
+# Define vocabulary size and transformer configuration (3 Billion)
VOCAB_SIZE = 50304 # Number of unique tokens in the vocabulary
CONTEXT_LENGTH = 512 # Maximum sequence length for the model
N_EMBED = 2048 # Dimension of the embedding space
N_HEAD = 16 # Number of attention heads in each transformer block
-N_BLOCKS = 16 # Number of transformer blocks in the model
-# CONTEXT_LENGTH = 16 # Maximum sequence length for the model
-# N_EMBED = 32 # Dimension of the embedding space
-# N_HEAD = 8 # Number of attention heads in each transformer block
-# N_BLOCKS = 1 # Number of transformer blocks in the model
+N_BLOCKS = 64 # Number of transformer blocks in the model
# Paths to training and development datasets
TRAIN_PATH = "data/train/pile_train.h5" # File path for the training dataset
@@ -22,14 +18,12 @@ T_TRAIN_STEPS = 200000 # Total number of training steps
T_EVAL_STEPS = 1000 # Frequency (in steps) to perform evaluation
T_EVAL_ITERS = 250 # Number of iterations to evaluate the model
T_LR_DECAY_STEP = 50000 # Step at which to decay the learning rate
-T_LR = 5e-5 # Initial learning rate for training
-T_LR_DECAYED = 5e-6 # Learning rate after decay
-T_LR_DECAY_RATE = 0.95
+T_LR = 5e-4 # Initial learning rate for training
+T_LR_DECAYED = 5e-5 # Learning rate after decay
T_OUT_PATH = "models/transformer_B.pt" # Path to save the trained model
# Device configuration
DEVICE = 'cuda'
-# DEVICE = 'cpu'
# Store all configurations in a dictionary for easy access and modification
default_config = {
diff --git a/data_loader/__init__.py b/data_loader/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/scripts/__init__.py b/scripts/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/scripts/train_transformer.py b/scripts/train_transformer.py
index ce89c01..8cbb14a 100644
--- a/scripts/train_transformer.py
+++ b/scripts/train_transformer.py
@@ -1,114 +1,156 @@
-from data_loader.data_loader import get_batch_iterator
-from src.models import Transformer
-from config import config
-from typing import Iterator, Literal, Tuple
-from tqdm import tqdm
import torch
+import torch.nn.functional as F
+import os
+from tqdm import tqdm
import numpy as np
+from config.config import default_config as config
+from src.models.transformer import Transformer
+from data_loader.data_loader import get_batch_iterator
+from typing import Dict
-torch.autograd.set_detect_anomaly(True)
-
-try:
- device = torch.device(config.DEVICE)
-except:
- device = torch.device('cpu')
+# --- Initialize the Model and Print Parameters ---
model = Transformer(
- text_dim=config.VOCAB_SIZE,
- ctx_len=config.CONTEXT_LENGTH,
- emb_dim=config.N_EMBED,
- num_heads=config.N_HEAD,
- num_blocks=config.N_BLOCKS,
- device=device,
-).to(device)
-
-criterion = torch.nn.CrossEntropyLoss()
-
-def create_dataloader(split: Literal["train", "val"]) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]:
- dataloader = get_batch_iterator(
- (config.TRAIN_PATH if split == "train" else config.DEV_PATH),
- config.T_BATCH_SIZE,
- config.CONTEXT_LENGTH,
- )
- return dataloader
-
-train_dataloader = create_dataloader("train")
-val_dataloader = create_dataloader("val")
-
-def compute_loss(logits: torch.Tensor, gt: torch.Tensor) -> torch.Tensor:
- gt = gt.type(torch.LongTensor).to(device)
-
- B, CL, VOCAB = logits.shape
- logits = torch.moveaxis(logits, 2, 1)
- assert logits.shape == (B, VOCAB, CL)
- assert gt.shape == (B, CL)
-
- loss = criterion(logits, gt)
- return loss
+ n_head=config['n_head'],
+ n_embed=config['n_embed'],
+ context_length=config['context_length'],
+ vocab_size=config['vocab_size'],
+ N_BLOCKS=config['n_blocks']
+).to(config['device'])
+
+# Print the total number of parameters
+total_params = sum(p.numel() for p in model.parameters())
+print(f"Total number of parameters in the model: {total_params:,}")
+
+# --- Optimizer Setup and Loss Tracking ---
+# Set up the AdamW optimizer with the specified learning rate.
+optimizer = torch.optim.AdamW(model.parameters(), lr=config['t_lr'])
+
+# List to track loss values during training.
+losses = []
+
+# Define a window size for averaging recent losses in the training loop.
+AVG_WINDOW = 64
+
+# Helper function to estimate the average loss for training and development data.
@torch.no_grad()
-def evaluate_model(model: Transformer, num_batches: int = config.T_EVAL_ITERS, split: Literal["train", "val"] = "val") -> float:
- if split == "val":
- dataloader = val_dataloader
- else:
- dataloader = create_dataloader(split)
-
- losses = []
- print("Evaluation start.")
- for _ in tqdm(list(range(num_batches))):
- x, y = next(dataloader)
- x = x.to(device)
- y = y.to(device)
- assert len(x.shape) == 2 and x.shape[0] == y.shape[0]
-
- losses.append(
- compute_loss(
- model(x, get_probs=False, keep_prompt_seg=True),
- y
- ).item()
+def estimate_loss(steps: int) -> Dict[str, float]:
+ """
+ Evaluate the model on training and development datasets and calculate average loss.
+
+ Args:
+ steps (int): Number of steps to evaluate.
+
+ Returns:
+ dict: Dictionary containing average losses for 'train' and 'dev' splits.
+ """
+ out = {}
+ model.eval() # Set the model to evaluation mode.
+
+ for split in ['train', 'dev']:
+ # Select the appropriate data path for the current split.
+ data_path = config['train_path'] if split == 'train' else config['dev_path']
+
+ # Create a batch iterator for evaluation.
+ batch_iterator_eval = get_batch_iterator(
+ data_path, config['t_batch_size'], config['t_context_length'], device=config['device']
)
-
- return np.mean(losses)
-optimizer = torch.optim.Adam(
- model.parameters(),
- config.T_LR,
+ # Initialize a tensor to track loss values for each evaluation step.
+ losses_eval = torch.zeros(steps)
+ for k in range(steps):
+ try:
+ # Fetch a batch and calculate the loss.
+ xb, yb = next(batch_iterator_eval)
+ _, loss = model(xb, yb)
+ losses_eval[k] = loss.item()
+ except StopIteration:
+ # Handle the case where the data iterator ends early.
+ print(f"Warning: Iterator for {split} ended early.")
+ break
+
+ # Compute the mean loss for the current split.
+ out[split] = losses_eval[:k + 1].mean()
+
+ model.train() # Restore the model to training mode.
+ return out
+
+# --- Training Loop ---
+
+# Create a batch iterator for the training data.
+batch_iterator = get_batch_iterator(
+ config['train_path'],
+ config['t_batch_size'],
+ config['t_context_length'],
+ device=config['device']
)
-print(f"All components count: {sum(1 for param in model.parameters())}")
-print(f"All parameter count: {sum(param.numel() for param in model.parameters())}")
-print(f"Trainable parameter count: {sum(param.numel() for param in model.parameters() if param.requires_grad)}")
-
-train_losses = []
-eval_losses = []
-
-for step in tqdm(list(range(config.T_TRAIN_STEPS))):
- x, y_gt = next(train_dataloader)
- x = x.to(device)
- y_gt = y_gt.to(device)
- logits = model(x, get_probs=False, keep_prompt_seg=True)
-
- optimizer.zero_grad(set_to_none=True)
- loss = compute_loss(logits, y_gt)
- train_losses.append(loss.item())
- loss.backward()
- optimizer.step()
-
- if step % config.T_EVAL_STEPS == 0:
- eval_loss = evaluate_model(model)
- print(f"Eval loss at step {step:7d}: {eval_loss:1.6f}")
- eval_losses.append((step, eval_loss))
-
- if step % config.T_LR_DECAY_STEP == 0:
- for gr in optimizer.param_groups:
- gr["lr"] *= config.T_LR_DECAY_RATE
+# Create a progress bar to monitor training progress.
+pbar = tqdm(range(config['t_train_steps']))
+for step in pbar:
+ try:
+ # Fetch a batch of input and target data.
+ xb, yb = next(batch_iterator)
+
+ # Perform a forward pass and compute the loss.
+ _, loss = model(xb, yb)
+
+ # Record the loss for tracking.
+ losses.append(loss.item())
+ pbar.set_description(f"Train loss: {np.mean(losses[-AVG_WINDOW:]):.4f}")
+
+ # Backpropagate the loss and update the model parameters.
+ optimizer.zero_grad(set_to_none=True)
+ loss.backward()
+ optimizer.step()
+ # Periodically evaluate the model on training and development data.
+ if step % config['t_eval_steps'] == 0:
+ evaluation_losses = estimate_loss(config['t_eval_iters'])
+ train_loss = evaluation_losses['train']
+ dev_loss = evaluation_losses['dev']
+ print(f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")
+
+ # Decay the learning rate at the specified step.
+ if step == config['t_lr_decay_step']:
+ print('Decaying learning rate')
+ for g in optimizer.param_groups:
+ g['lr'] = config['t_lr_decayed']
+ except StopIteration:
+ # Handle the case where the training data iterator ends early.
+ print("Training data iterator finished early.")
+ break
+
+# --- Save Model and Final Evaluation ---
+
+# Create the output directory if it does not exist.
+os.makedirs(config['t_out_path'].split('/')[0], exist_ok=True)
+
+# Perform a final evaluation of the model on training and development datasets.
+evaluation_losses = estimate_loss(200)
+train_loss = evaluation_losses['train']
+dev_loss = evaluation_losses['dev']
+
+# Ensure unique model save path in case the file already exists.
+modified_model_out_path = config['t_out_path']
+save_tries = 0
+while os.path.exists(modified_model_out_path):
+ save_tries += 1
+ model_out_name = os.path.splitext(config['t_out_path'])[0]
+ modified_model_out_path = model_out_name + f"_{save_tries}" + ".pt"
+
+# Save the model's state dictionary, optimizer state, and training metadata.
torch.save(
{
- "model_state": model.state_dict(),
- "optim_state": optimizer.state_dict(),
- "train_losses": train_losses,
- "eval_losses": eval_losses,
+ 'model_state_dict': model.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ 'losses': losses,
+ 'train_loss': train_loss,
+ 'dev_loss': dev_loss,
+ 'steps': len(losses),
},
- config.T_OUT_PATH
-)
\ No newline at end of file
+ modified_model_out_path
+)
+print(f"Saved model to {modified_model_out_path}")
+print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")
\ No newline at end of file
diff --git a/src/models/__init__.py b/src/models/__init__.py
index db69368..dab85fd 100644
--- a/src/models/__init__.py
+++ b/src/models/__init__.py
@@ -1,5 +1,5 @@
# This file makes the 'src.models' directory a Python package.
from .mlp import MLP
-from .attention import MultiHeadAttn
-from .transformer_block import TransformerBlock
+from .attention import Head, MultiHeadAttention
+from .transformer_block import Block
from .transformer import Transformer
\ No newline at end of file
diff --git a/src/models/attention.py b/src/models/attention.py
index 87f8327..c0f9c8f 100644
--- a/src/models/attention.py
+++ b/src/models/attention.py
@@ -1,63 +1,110 @@
import torch
-from torch import nn
-
-class Attention(nn.Module):
- emb_dim: int
- attn_dim: int
- q_head: nn.Linear
- k_head: nn.Linear
- v_head: nn.Linear
-
- def __init__(self, emb_dim: int, attn_dim: int):
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+class Head(nn.Module):
+ """
+ A single attention head.
+
+ This module calculates attention scores and applies them to the values.
+ It includes key, query, and value projections, and uses causal masking
+ to prevent attending to future tokens.
+
+ Args:
+ head_size (int): The dimensionality of the key, query, and value projections.
+ n_embed (int): The dimensionality of the input embedding.
+ context_length (int): The maximum length of the input sequence, used for causal masking.
+ """
+ def __init__(self, head_size: int, n_embed: int, context_length: int) -> None:
+ """
+ Initializes the attention head.
+
+ Args:
+ head_size (int): The dimensionality of the key, query, and value projections.
+ n_embed (int): The dimensionality of the input embedding.
+ context_length (int): The maximum length of the input sequence.
+ """
super().__init__()
- self.emb_dim = emb_dim
- self.attn_dim = attn_dim
- self.q_head = nn.Linear(emb_dim, attn_dim, bias=False)
- self.k_head = nn.Linear(emb_dim, attn_dim, bias=False)
- self.v_head = nn.Linear(emb_dim, attn_dim, bias=False)
-
+ self.key = nn.Linear(n_embed, head_size, bias=False) # Key projection
+ self.query = nn.Linear(n_embed, head_size, bias=False) # Query projection
+ self.value = nn.Linear(n_embed, head_size, bias=False) # Value projection
+ # Lower triangular matrix for causal masking
+ self.register_buffer('tril', torch.tril(torch.ones(context_length, context_length)))
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
- # x: (B, ctx_len, emb_dim)
- assert len(x.shape) == 3 and x.shape[-1] == self.emb_dim
-
- q: torch.Tensor = self.q_head(x) # (B, ctx_len, attn_dim)
- k: torch.Tensor = self.k_head(x)
- k = torch.transpose(k, -1, -2) # (B, attn_dim, ctx_len)
- attn = q @ k * (self.attn_dim ** -0.5)
- attn = torch.tril(attn, diagonal=0) + torch.triu(torch.ones_like(attn) * (-torch.inf), diagonal=1)
- attn = nn.functional.softmax(attn, dim=-1) # (B, ctx_len, ctx_len)
-
- v: torch.Tensor = self.v_head(x) # (B, ctx_len, attn_dim)
- return attn @ v
-
-
-class MultiHeadAttn(nn.Module):
- emb_dim: int
- num_heads: int
- heads: nn.ModuleList
-
- def __init__(self, emb_dim: int, num_heads: int):
+ """
+ Forward pass through the attention head.
+
+ Args:
+ x (torch.Tensor): Input tensor of shape (B, T, C).
+
+ Returns:
+ torch.Tensor: Output tensor after applying attention.
+ """
+ B, T, C = x.shape
+ k = self.key(x) # (B, T, head_size)
+ q = self.query(x) # (B, T, head_size)
+ scale_factor = 1 / math.sqrt(C)
+ # Calculate attention weights: (B, T, head_size) @ (B, head_size, T) -> (B, T, T)
+ attn_weights = q @ k.transpose(-2, -1) * scale_factor
+ # Apply causal masking
+ attn_weights = attn_weights.masked_fill(self.tril[:T, :T] == 0, float('-inf'))
+ attn_weights = F.softmax(attn_weights, dim=-1)
+ v = self.value(x) # (B, T, head_size)
+ # Apply attention weights to values
+ out = attn_weights @ v # (B, T, T) @ (B, T, head_size) -> (B, T, head_size)
+ return out
+
+class MultiHeadAttention(nn.Module):
+ """
+ Multi-Head Attention module.
+
+ This module combines multiple attention heads in parallel. The outputs of each head
+ are concatenated to form the final output.
+
+ Args:
+ n_head (int): The number of parallel attention heads.
+ n_embed (int): The dimensionality of the input embedding.
+ context_length (int): The maximum length of the input sequence.
+ """
+ def __init__(self, n_head: int, n_embed: int, context_length: int) -> None:
+ """
+ Initializes the multi-head attention module.
+
+ Args:
+ n_head (int): The number of parallel attention heads.
+ n_embed (int): The dimensionality of the input embedding.
+ context_length (int): The maximum length of the input sequence.
+ """
super().__init__()
- self.emb_dim = emb_dim
- self.num_heads = num_heads
-
- assert emb_dim % num_heads == 0
-
- self.heads = nn.ModuleList([
- Attention(emb_dim, emb_dim // num_heads)
- for _ in range(num_heads)
- ])
-
- def forward(self, x: torch.Tensor):
- return torch.cat(
- [attn(x) for attn in self.heads],
- dim = -1
- )
-
-
-if __name__ == "__main__":
- mha = MultiHeadAttn(16, 4)
- x = torch.randn((5, 64, 16))
- y = mha(x)
- print(x.shape)
- print(y.shape)
\ No newline at end of file
+ self.heads = nn.ModuleList([Head(n_embed // n_head, n_embed, context_length) for _ in range(n_head)])
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ """
+ Forward pass through the multi-head attention.
+
+ Args:
+ x (torch.Tensor): Input tensor of shape (B, T, C).
+
+ Returns:
+ torch.Tensor: Output tensor after concatenating the outputs of all heads.
+ """
+ # Concatenate the output of each head along the last dimension (C)
+ x = torch.cat([h(x) for h in self.heads], dim=-1)
+ return x
+
+if __name__ == '__main__':
+ # Example Usage (optional, for testing the module independently)
+ batch_size = 2
+ sequence_length = 5
+ embedding_dim = 32
+ num_heads = 4
+ context_len = 5
+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)
+
+ multihead_attn = MultiHeadAttention(n_head=num_heads, n_embed=embedding_dim, context_length=context_len)
+ output_tensor = multihead_attn(input_tensor)
+
+ print("MultiHeadAttention Input Shape:", input_tensor.shape)
+ print("MultiHeadAttention Output Shape:", output_tensor.shape)
\ No newline at end of file
diff --git a/src/models/mlp.py b/src/models/mlp.py
index 7f1b7bf..c7baa79 100644
--- a/src/models/mlp.py
+++ b/src/models/mlp.py
@@ -1,29 +1,80 @@
import torch
-from torch import nn
+import torch.nn as nn
+from torch import Tensor
class MLP(nn.Module):
- emb_dim: int
- relu: nn.Module
- in_linear: nn.Linear
- out_linear: nn.Linear
-
- def __init__(self, emb_dim: int):
+ """
+ A simple Multi-Layer Perceptron with one hidden layer.
+
+ This module is used within the Transformer block for feed-forward processing.
+ It expands the input embedding size, applies a ReLU activation, and then projects it back
+ to the original embedding size.
+
+ Args:
+ n_embed (int): The dimensionality of the input embedding.
+ """
+ def __init__(self, n_embed: int) -> None:
+ """
+ Initializes the MLP module.
+
+ Args:
+ n_embed (int): The dimensionality of the input embedding.
+ """
super().__init__()
- self.emb_dim = emb_dim
- self.in_linear = nn.Linear(emb_dim, emb_dim * 4)
- self.relu = nn.ReLU()
- self.out_linear = nn.Linear(emb_dim * 4, emb_dim)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x = self.in_linear(x)
- x = self.relu(x)
- x = self.out_linear(x)
+ self.hidden = nn.Linear(n_embed, 4 * n_embed) # Linear layer to expand embedding size
+ self.relu = nn.ReLU() # ReLU activation function
+ self.proj = nn.Linear(4 * n_embed, n_embed) # Linear layer to project back to original size
+
+ def forward(self, x: Tensor) -> Tensor:
+ """
+ Forward pass through the MLP.
+
+ Args:
+ x (torch.Tensor): Input tensor of shape (B, T, C), where B is batch size,
+ T is sequence length, and C is embedding size.
+
+ Returns:
+ torch.Tensor: Output tensor of the same shape as the input.
+ """
+ x = self.forward_embedding(x)
+ x = self.project_embedding(x)
return x
+ def forward_embedding(self, x: Tensor) -> Tensor:
+ """
+ Applies the hidden linear layer followed by ReLU activation.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ torch.Tensor: Output after the hidden layer and ReLU.
+ """
+ x = self.relu(self.hidden(x))
+ return x
+
+ def project_embedding(self, x: Tensor) -> Tensor:
+ """
+ Applies the projection linear layer.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ torch.Tensor: Output after the projection layer.
+ """
+ x = self.proj(x)
+ return x
+
+if __name__ == '__main__':
+ # Example Usage (optional, for testing the module independently)
+ batch_size = 2
+ sequence_length = 3
+ embedding_dim = 16
+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)
+
+ mlp_module = MLP(n_embed=embedding_dim)
+ output_tensor = mlp_module(input_tensor)
-if __name__ == "__main__":
- mlp = MLP(10)
- x = torch.randn((5, 64, 10))
- y = mlp(x)
- print(x.shape)
- print(y.shape)
\ No newline at end of file
+ print("MLP Input Shape:", input_tensor.shape)
+ print("MLP Output Shape:", output_tensor.shape)
\ No newline at end of file
diff --git a/src/models/transformer.py b/src/models/transformer.py
index 49d1140..4650753 100644
--- a/src/models/transformer.py
+++ b/src/models/transformer.py
@@ -1,66 +1,136 @@
import torch
-from torch import nn
-from src.models.transformer_block import TransformerBlock
+import torch.nn as nn
+import torch.nn.functional as F
+from src.models.transformer_block import Block
class Transformer(nn.Module):
- text_dim: int
- ctx_len: int
- emb_dim: int
- num_heads: int
- num_blocks: int
-
- text_emb: nn.Embedding
- pos_emb: nn.Embedding
- blocks: nn.ModuleList
- lm_head: nn.Linear
-
- model_device: torch.device
-
- def __init__(self, text_dim: int, ctx_len: int, emb_dim: int, num_heads: int, num_blocks: int, device: torch.device = None):
+ """
+ The main Transformer model.
+
+ This class combines token and position embeddings with a sequence of Transformer blocks
+ and a final linear layer for language modeling.
+
+ Args:
+ n_head (int): The number of attention heads in each transformer block.
+ n_embed (int): The dimensionality of the embedding space.
+ context_length (int): The maximum length of the input sequence.
+ vocab_size (int): The size of the vocabulary.
+ N_BLOCKS (int): The number of transformer blocks in the model.
+ """
+ def __init__(self, n_head: int, n_embed: int, context_length: int, vocab_size: int, N_BLOCKS: int) -> None:
+ """
+ Initializes the Transformer model.
+
+ Args:
+ n_head (int): Number of attention heads.
+ n_embed (int): Embedding dimension.
+ context_length (int): Maximum sequence length.
+ vocab_size (int): Size of the vocabulary.
+ N_BLOCKS (int): Number of transformer blocks.
+ """
super().__init__()
- self.text_dim = text_dim
- self.ctx_len = ctx_len
- self.emb_dim = emb_dim
- self.num_heads = num_heads
- self.num_blocks = num_blocks
- self.model_device = device
-
- self.text_emb = nn.Embedding(text_dim, emb_dim)
- self.pos_emb = nn.Embedding(ctx_len, emb_dim)
- self.blocks = nn.ModuleList([
- TransformerBlock(emb_dim, num_heads)
- for _ in range(num_blocks)
- ])
- self.ln_lmhead = nn.LayerNorm(emb_dim)
- self.lm_head = nn.Linear(emb_dim, text_dim)
-
- def forward(self, x: torch.Tensor, get_probs: bool = False, keep_prompt_seg: bool = False):
- """Transformer forward pass. Input 2-D tensor contains raw tokens, NOT one-hot encoded."""
- assert len(x.shape) == 2 and x.shape[-1] <= self.ctx_len
-
- x = self.text_emb(x) + self.pos_emb(torch.arange(x.shape[-1], device=self.model_device))
- assert len(x.shape) == 3 and x.shape[2] == self.emb_dim
-
- for block in self.blocks:
+ self.context_length = context_length
+ self.N_BLOCKS = N_BLOCKS
+ self.token_embed = nn.Embedding(vocab_size, n_embed)
+ self.position_embed = nn.Embedding(context_length, n_embed)
+ self.attn_blocks = nn.ModuleList([Block(n_head, n_embed, context_length) for _ in range(N_BLOCKS)])
+ self.layer_norm = nn.LayerNorm(n_embed)
+ self.lm_head = nn.Linear(n_embed, vocab_size)
+ self.register_buffer('pos_idxs', torch.arange(context_length))
+
+ def _pre_attn_pass(self, idx: torch.Tensor) -> torch.Tensor:
+ """
+ Combines token and position embeddings.
+
+ Args:
+ idx (torch.Tensor): Input token indices.
+
+ Returns:
+ torch.Tensor: Sum of token and position embeddings.
+ """
+ B, T = idx.shape
+ tok_embedding = self.token_embed(idx)
+ pos_embedding = self.position_embed(self.pos_idxs[:T])
+ return tok_embedding + pos_embedding
+
+ def forward(self, idx: torch.Tensor, targets: torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor | None]:
+ """
+ Forward pass through the Transformer.
+
+ Args:
+ idx (torch.Tensor): Input token indices.
+ targets (torch.Tensor, optional): Target token indices for loss calculation. Defaults to None.
+
+ Returns:
+ tuple: Logits and loss (if targets are provided).
+ """
+ x = self._pre_attn_pass(idx)
+ for block in self.attn_blocks:
x = block(x)
-
- x = self.lm_head(self.ln_lmhead(x))
- assert len(x.shape) == 3 and x.shape[2] == self.text_dim
-
- if get_probs:
- # Compute probs instead of logits
- x = nn.functional.softmax(x, dim=-1)
-
- if not keep_prompt_seg:
- # Only return the next-token results
- x = x[:, -1, :]
-
- return x
-
-if __name__ == "__main__":
- transformer = Transformer(8, 8, 32, 4, 5)
- x = torch.randint(8, (3, 7))
- y = transformer(x, get_probs = True, keep_prompt_seg = True)
- print(x.shape)
- print(y.shape)
- print(y)
\ No newline at end of file
+ x = self.layer_norm(x)
+ logits = self.lm_head(x)
+ loss = None
+ if targets is not None:
+ B, T, C = logits.shape
+ flat_logits = logits.view(B * T, C)
+ targets = targets.view(B * T).long()
+ loss = F.cross_entropy(flat_logits, targets)
+ return logits, loss
+
+ def forward_embedding(self, idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """
+ Forward pass focusing on the embedding and attention blocks.
+
+ Args:
+ idx (torch.Tensor): Input token indices.
+
+ Returns:
+ tuple: Output after attention blocks and the residual.
+ """
+ x = self._pre_attn_pass(idx)
+ residual = x
+ for block in self.attn_blocks:
+ x, residual = block.forward_embedding(x)
+ return x, residual
+
+ def generate(self, idx: torch.Tensor, max_new_tokens: int) -> torch.Tensor:
+ """
+ Generates new tokens given a starting sequence.
+
+ Args:
+ idx (torch.Tensor): Initial sequence of token indices.
+ max_new_tokens (int): Number of tokens to generate.
+
+ Returns:
+ torch.Tensor: The extended sequence of tokens.
+ """
+ for _ in range(max_new_tokens):
+ idx_cond = idx[:, -self.context_length:]
+ logits, _ = self(idx_cond)
+ logits = logits[:, -1, :]
+ probs = F.softmax(logits, dim=-1)
+ idx_next = torch.multinomial(probs, num_samples=1)
+ idx = torch.cat((idx, idx_next), dim=1)
+ return idx
+
+if __name__ == '__main__':
+ # Example Usage (optional, for testing the module independently)
+ batch_size = 2
+ sequence_length = 5
+ vocab_size = 100
+ embedding_dim = 32
+ num_heads = 4
+ num_blocks = 2
+ context_len = 5
+ input_indices = torch.randint(0, vocab_size, (batch_size, sequence_length))
+
+ transformer_model = Transformer(n_head=num_heads, n_embed=embedding_dim, context_length=context_len, vocab_size=vocab_size, N_BLOCKS=num_blocks)
+ logits, loss = transformer_model(input_indices, targets=input_indices) # Using input as target for simplicity
+
+ print("Transformer Logits Shape:", logits.shape)
+ print("Transformer Loss:", loss)
+
+ # Example of generating tokens
+ start_indices = input_indices[:, :1] # Take the first token of each sequence as start
+ generated_tokens = transformer_model.generate(start_indices, max_new_tokens=5)
+ print("Generated Tokens Shape:", generated_tokens.shape)
\ No newline at end of file
diff --git a/src/models/transformer_block.py b/src/models/transformer_block.py
index 4cb76ed..b6cfd29 100644
--- a/src/models/transformer_block.py
+++ b/src/models/transformer_block.py
@@ -1,23 +1,76 @@
import torch
-from torch import nn
+import torch.nn as nn
+from src.models.attention import MultiHeadAttention
from src.models.mlp import MLP
-from src.models.attention import MultiHeadAttn
-
-class TransformerBlock(nn.Module):
- emb_dim: int
- attn: MultiHeadAttn
- ln_attn: nn.LayerNorm
- mlp: MLP
- ln_mlp: nn.LayerNorm
-
- def __init__(self, emb_dim: int, num_heads: int):
+
+class Block(nn.Module):
+ """
+ A single Transformer block.
+
+ This block consists of a multi-head attention layer followed by an MLP,
+ with layer normalization and residual connections.
+
+ Args:
+ n_head (int): The number of attention heads in the multi-head attention layer.
+ n_embed (int): The dimensionality of the input embedding.
+ context_length (int): The maximum length of the input sequence.
+ """
+ def __init__(self, n_head: int, n_embed: int, context_length: int) -> None:
+ """
+ Initializes the Transformer block.
+
+ Args:
+ n_head (int): The number of attention heads.
+ n_embed (int): The dimensionality of the embedding space.
+ context_length (int): The maximum sequence length.
+ """
super().__init__()
- self.attn = MultiHeadAttn(emb_dim, num_heads)
- self.ln_attn = nn.LayerNorm(emb_dim)
- self.mlp = MLP(emb_dim)
- self.ln_mlp = nn.LayerNorm(emb_dim)
-
- def forward(self, x: torch.Tensor):
- x = x + self.attn(self.ln_attn(x))
- x = x + self.mlp(self.ln_mlp(x))
- return x
\ No newline at end of file
+ self.ln1 = nn.LayerNorm(n_embed)
+ self.attn = MultiHeadAttention(n_head, n_embed, context_length)
+ self.ln2 = nn.LayerNorm(n_embed)
+ self.mlp = MLP(n_embed)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ """
+ Forward pass through the Transformer block.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ torch.Tensor: Output tensor after the block.
+ """
+ # Apply multi-head attention with residual connection
+ x = x + self.attn(self.ln1(x))
+ # Apply MLP with residual connection
+ x = x + self.mlp(self.ln2(x))
+ return x
+
+ def forward_embedding(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """
+ Forward pass focusing on the embedding and attention parts.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ tuple: A tuple containing the output after MLP embedding and the residual.
+ """
+ res = x + self.attn(self.ln1(x))
+ x = self.mlp.forward_embedding(self.ln2(res))
+ return x, res
+
+if __name__ == '__main__':
+ # Example Usage (optional, for testing the module independently)
+ batch_size = 2
+ sequence_length = 5
+ embedding_dim = 32
+ num_heads = 4
+ context_len = 5
+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)
+
+ transformer_block = Block(n_head=num_heads, n_embed=embedding_dim, context_length=context_len)
+ output_tensor = transformer_block(input_tensor)
+
+ print("Transformer Block Input Shape:", input_tensor.shape)
+ print("Transformer Block Output Shape:", output_tensor.shape)
\ No newline at end of file