A straightforward method for training your LLM, from downloading data to generating text.
Revert "Fix attention scaling factor"
Fareed Khan8e1fbb78a29a
diff --git a/.gitignore b/.gitignoredeleted file mode 100644index 54e6782..0000000--- a/.gitignore+++ /dev/null@@ -1,2 +0,0 @@-__pycache__/-data/\ No newline at end of filediff --git a/PITFALLS.md b/PITFALLS.mddeleted file mode 100644index 37d8362..0000000--- a/PITFALLS.md+++ /dev/null@@ -1,7 +0,0 @@-- No bias in Q/K/V-- `attn = torch.tril(attn, diagonal=0) + torch.triu(torch.ones_like(attn) * (-torch.inf), diagonal=1)`-- `k = torch.transpose(k, -1, -2)`-- `x = x + self.attn(self.ln_attn(x)); x = x + self.mlp(self.ln_mlp(x))`-- `x = self.lm_head(self.ln_lmhead(x))`-- Send model & all tensors to device-- Use `x = x + ...` not `x += ...` !\ No newline at end of filediff --git a/config/__init__.py b/config/__init__.pydeleted file mode 100644index e69de29..0000000diff --git a/config/config.py b/config/config.pyindex 035396e..ae4bf50 100644--- a/config/config.py+++ b/config/config.py@@ -1,15 +1,11 @@# --- Configuration ----# Define vocabulary size and transformer configuration (0.8 Billion)+# Define vocabulary size and transformer configuration (3 Billion)VOCAB_SIZE = 50304 # Number of unique tokens in the vocabularyCONTEXT_LENGTH = 512 # Maximum sequence length for the modelN_EMBED = 2048 # Dimension of the embedding spaceN_HEAD = 16 # Number of attention heads in each transformer block-N_BLOCKS = 16 # Number of transformer blocks in the model-# CONTEXT_LENGTH = 16 # Maximum sequence length for the model-# N_EMBED = 32 # Dimension of the embedding space-# N_HEAD = 8 # Number of attention heads in each transformer block-# N_BLOCKS = 1 # Number of transformer blocks in the model+N_BLOCKS = 64 # Number of transformer blocks in the model# Paths to training and development datasetsTRAIN_PATH = "data/train/pile_train.h5" # File path for the training dataset@@ -22,14 +18,12 @@ T_TRAIN_STEPS = 200000 # Total number of training stepsT_EVAL_STEPS = 1000 # Frequency (in steps) to perform evaluationT_EVAL_ITERS = 250 # Number of iterations to evaluate the modelT_LR_DECAY_STEP = 50000 # Step at which to decay the learning rate-T_LR = 5e-5 # Initial learning rate for training-T_LR_DECAYED = 5e-6 # Learning rate after decay-T_LR_DECAY_RATE = 0.95+T_LR = 5e-4 # Initial learning rate for training+T_LR_DECAYED = 5e-5 # Learning rate after decayT_OUT_PATH = "models/transformer_B.pt" # Path to save the trained model# Device configurationDEVICE = 'cuda'-# DEVICE = 'cpu'# Store all configurations in a dictionary for easy access and modificationdefault_config = {diff --git a/data_loader/__init__.py b/data_loader/__init__.pydeleted file mode 100644index e69de29..0000000diff --git a/scripts/__init__.py b/scripts/__init__.pydeleted file mode 100644index e69de29..0000000diff --git a/scripts/train_transformer.py b/scripts/train_transformer.pyindex ce89c01..8cbb14a 100644--- a/scripts/train_transformer.py+++ b/scripts/train_transformer.py@@ -1,114 +1,156 @@-from data_loader.data_loader import get_batch_iterator-from src.models import Transformer-from config import config-from typing import Iterator, Literal, Tuple-from tqdm import tqdmimport torch+import torch.nn.functional as F+import os+from tqdm import tqdmimport numpy as np+from config.config import default_config as config+from src.models.transformer import Transformer+from data_loader.data_loader import get_batch_iterator+from typing import Dict-torch.autograd.set_detect_anomaly(True)--try:- device = torch.device(config.DEVICE)-except:- device = torch.device('cpu')+# --- Initialize the Model and Print Parameters ---model = Transformer(- text_dim=config.VOCAB_SIZE,- ctx_len=config.CONTEXT_LENGTH,- emb_dim=config.N_EMBED,- num_heads=config.N_HEAD,- num_blocks=config.N_BLOCKS,- device=device,-).to(device)--criterion = torch.nn.CrossEntropyLoss()--def create_dataloader(split: Literal["train", "val"]) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]:- dataloader = get_batch_iterator(- (config.TRAIN_PATH if split == "train" else config.DEV_PATH),- config.T_BATCH_SIZE,- config.CONTEXT_LENGTH,- )- return dataloader--train_dataloader = create_dataloader("train")-val_dataloader = create_dataloader("val")--def compute_loss(logits: torch.Tensor, gt: torch.Tensor) -> torch.Tensor:- gt = gt.type(torch.LongTensor).to(device)-- B, CL, VOCAB = logits.shape- logits = torch.moveaxis(logits, 2, 1)- assert logits.shape == (B, VOCAB, CL)- assert gt.shape == (B, CL)-- loss = criterion(logits, gt)- return loss+ n_head=config['n_head'],+ n_embed=config['n_embed'],+ context_length=config['context_length'],+ vocab_size=config['vocab_size'],+ N_BLOCKS=config['n_blocks']+).to(config['device'])++# Print the total number of parameters+total_params = sum(p.numel() for p in model.parameters())+print(f"Total number of parameters in the model: {total_params:,}")++# --- Optimizer Setup and Loss Tracking ---+# Set up the AdamW optimizer with the specified learning rate.+optimizer = torch.optim.AdamW(model.parameters(), lr=config['t_lr'])++# List to track loss values during training.+losses = []++# Define a window size for averaging recent losses in the training loop.+AVG_WINDOW = 64++# Helper function to estimate the average loss for training and development data.@torch.no_grad()-def evaluate_model(model: Transformer, num_batches: int = config.T_EVAL_ITERS, split: Literal["train", "val"] = "val") -> float:- if split == "val":- dataloader = val_dataloader- else:- dataloader = create_dataloader(split)-- losses = []- print("Evaluation start.")- for _ in tqdm(list(range(num_batches))):- x, y = next(dataloader)- x = x.to(device)- y = y.to(device)- assert len(x.shape) == 2 and x.shape[0] == y.shape[0]-- losses.append(- compute_loss(- model(x, get_probs=False, keep_prompt_seg=True),- y- ).item()+def estimate_loss(steps: int) -> Dict[str, float]:+ """+ Evaluate the model on training and development datasets and calculate average loss.++ Args:+ steps (int): Number of steps to evaluate.++ Returns:+ dict: Dictionary containing average losses for 'train' and 'dev' splits.+ """+ out = {}+ model.eval() # Set the model to evaluation mode.++ for split in ['train', 'dev']:+ # Select the appropriate data path for the current split.+ data_path = config['train_path'] if split == 'train' else config['dev_path']++ # Create a batch iterator for evaluation.+ batch_iterator_eval = get_batch_iterator(+ data_path, config['t_batch_size'], config['t_context_length'], device=config['device'])-- return np.mean(losses)-optimizer = torch.optim.Adam(- model.parameters(),- config.T_LR,+ # Initialize a tensor to track loss values for each evaluation step.+ losses_eval = torch.zeros(steps)+ for k in range(steps):+ try:+ # Fetch a batch and calculate the loss.+ xb, yb = next(batch_iterator_eval)+ _, loss = model(xb, yb)+ losses_eval[k] = loss.item()+ except StopIteration:+ # Handle the case where the data iterator ends early.+ print(f"Warning: Iterator for {split} ended early.")+ break++ # Compute the mean loss for the current split.+ out[split] = losses_eval[:k + 1].mean()++ model.train() # Restore the model to training mode.+ return out++# --- Training Loop ---++# Create a batch iterator for the training data.+batch_iterator = get_batch_iterator(+ config['train_path'],+ config['t_batch_size'],+ config['t_context_length'],+ device=config['device'])-print(f"All components count: {sum(1 for param in model.parameters())}")-print(f"All parameter count: {sum(param.numel() for param in model.parameters())}")-print(f"Trainable parameter count: {sum(param.numel() for param in model.parameters() if param.requires_grad)}")--train_losses = []-eval_losses = []--for step in tqdm(list(range(config.T_TRAIN_STEPS))):- x, y_gt = next(train_dataloader)- x = x.to(device)- y_gt = y_gt.to(device)- logits = model(x, get_probs=False, keep_prompt_seg=True)-- optimizer.zero_grad(set_to_none=True)- loss = compute_loss(logits, y_gt)- train_losses.append(loss.item())- loss.backward()- optimizer.step()-- if step % config.T_EVAL_STEPS == 0:- eval_loss = evaluate_model(model)- print(f"Eval loss at step {step:7d}: {eval_loss:1.6f}")- eval_losses.append((step, eval_loss))-- if step % config.T_LR_DECAY_STEP == 0:- for gr in optimizer.param_groups:- gr["lr"] *= config.T_LR_DECAY_RATE+# Create a progress bar to monitor training progress.+pbar = tqdm(range(config['t_train_steps']))+for step in pbar:+ try:+ # Fetch a batch of input and target data.+ xb, yb = next(batch_iterator)++ # Perform a forward pass and compute the loss.+ _, loss = model(xb, yb)++ # Record the loss for tracking.+ losses.append(loss.item())+ pbar.set_description(f"Train loss: {np.mean(losses[-AVG_WINDOW:]):.4f}")++ # Backpropagate the loss and update the model parameters.+ optimizer.zero_grad(set_to_none=True)+ loss.backward()+ optimizer.step()+ # Periodically evaluate the model on training and development data.+ if step % config['t_eval_steps'] == 0:+ evaluation_losses = estimate_loss(config['t_eval_iters'])+ train_loss = evaluation_losses['train']+ dev_loss = evaluation_losses['dev']+ print(f"Step: {step}, Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")++ # Decay the learning rate at the specified step.+ if step == config['t_lr_decay_step']:+ print('Decaying learning rate')+ for g in optimizer.param_groups:+ g['lr'] = config['t_lr_decayed']+ except StopIteration:+ # Handle the case where the training data iterator ends early.+ print("Training data iterator finished early.")+ break++# --- Save Model and Final Evaluation ---++# Create the output directory if it does not exist.+os.makedirs(config['t_out_path'].split('/')[0], exist_ok=True)++# Perform a final evaluation of the model on training and development datasets.+evaluation_losses = estimate_loss(200)+train_loss = evaluation_losses['train']+dev_loss = evaluation_losses['dev']++# Ensure unique model save path in case the file already exists.+modified_model_out_path = config['t_out_path']+save_tries = 0+while os.path.exists(modified_model_out_path):+ save_tries += 1+ model_out_name = os.path.splitext(config['t_out_path'])[0]+ modified_model_out_path = model_out_name + f"_{save_tries}" + ".pt"++# Save the model's state dictionary, optimizer state, and training metadata.torch.save({- "model_state": model.state_dict(),- "optim_state": optimizer.state_dict(),- "train_losses": train_losses,- "eval_losses": eval_losses,+ 'model_state_dict': model.state_dict(),+ 'optimizer_state_dict': optimizer.state_dict(),+ 'losses': losses,+ 'train_loss': train_loss,+ 'dev_loss': dev_loss,+ 'steps': len(losses),},- config.T_OUT_PATH-)\ No newline at end of file+ modified_model_out_path+)+print(f"Saved model to {modified_model_out_path}")+print(f"Finished training. Train loss: {train_loss:.4f}, Dev loss: {dev_loss:.4f}")\ No newline at end of filediff --git a/src/models/__init__.py b/src/models/__init__.pyindex db69368..dab85fd 100644--- a/src/models/__init__.py+++ b/src/models/__init__.py@@ -1,5 +1,5 @@# This file makes the 'src.models' directory a Python package.from .mlp import MLP-from .attention import MultiHeadAttn-from .transformer_block import TransformerBlock+from .attention import Head, MultiHeadAttention+from .transformer_block import Blockfrom .transformer import Transformer\ No newline at end of filediff --git a/src/models/attention.py b/src/models/attention.pyindex 87f8327..c0f9c8f 100644--- a/src/models/attention.py+++ b/src/models/attention.py@@ -1,63 +1,110 @@import torch-from torch import nn--class Attention(nn.Module):- emb_dim: int- attn_dim: int- q_head: nn.Linear- k_head: nn.Linear- v_head: nn.Linear-- def __init__(self, emb_dim: int, attn_dim: int):+import torch.nn as nn+import torch.nn.functional as F+import math++class Head(nn.Module):+ """+ A single attention head.++ This module calculates attention scores and applies them to the values.+ It includes key, query, and value projections, and uses causal masking+ to prevent attending to future tokens.++ Args:+ head_size (int): The dimensionality of the key, query, and value projections.+ n_embed (int): The dimensionality of the input embedding.+ context_length (int): The maximum length of the input sequence, used for causal masking.+ """+ def __init__(self, head_size: int, n_embed: int, context_length: int) -> None:+ """+ Initializes the attention head.++ Args:+ head_size (int): The dimensionality of the key, query, and value projections.+ n_embed (int): The dimensionality of the input embedding.+ context_length (int): The maximum length of the input sequence.+ """super().__init__()- self.emb_dim = emb_dim- self.attn_dim = attn_dim- self.q_head = nn.Linear(emb_dim, attn_dim, bias=False)- self.k_head = nn.Linear(emb_dim, attn_dim, bias=False)- self.v_head = nn.Linear(emb_dim, attn_dim, bias=False)-+ self.key = nn.Linear(n_embed, head_size, bias=False) # Key projection+ self.query = nn.Linear(n_embed, head_size, bias=False) # Query projection+ self.value = nn.Linear(n_embed, head_size, bias=False) # Value projection+ # Lower triangular matrix for causal masking+ self.register_buffer('tril', torch.tril(torch.ones(context_length, context_length)))+def forward(self, x: torch.Tensor) -> torch.Tensor:- # x: (B, ctx_len, emb_dim)- assert len(x.shape) == 3 and x.shape[-1] == self.emb_dim-- q: torch.Tensor = self.q_head(x) # (B, ctx_len, attn_dim)- k: torch.Tensor = self.k_head(x)- k = torch.transpose(k, -1, -2) # (B, attn_dim, ctx_len)- attn = q @ k * (self.attn_dim ** -0.5)- attn = torch.tril(attn, diagonal=0) + torch.triu(torch.ones_like(attn) * (-torch.inf), diagonal=1)- attn = nn.functional.softmax(attn, dim=-1) # (B, ctx_len, ctx_len)-- v: torch.Tensor = self.v_head(x) # (B, ctx_len, attn_dim)- return attn @ v---class MultiHeadAttn(nn.Module):- emb_dim: int- num_heads: int- heads: nn.ModuleList-- def __init__(self, emb_dim: int, num_heads: int):+ """+ Forward pass through the attention head.++ Args:+ x (torch.Tensor): Input tensor of shape (B, T, C).++ Returns:+ torch.Tensor: Output tensor after applying attention.+ """+ B, T, C = x.shape+ k = self.key(x) # (B, T, head_size)+ q = self.query(x) # (B, T, head_size)+ scale_factor = 1 / math.sqrt(C)+ # Calculate attention weights: (B, T, head_size) @ (B, head_size, T) -> (B, T, T)+ attn_weights = q @ k.transpose(-2, -1) * scale_factor+ # Apply causal masking+ attn_weights = attn_weights.masked_fill(self.tril[:T, :T] == 0, float('-inf'))+ attn_weights = F.softmax(attn_weights, dim=-1)+ v = self.value(x) # (B, T, head_size)+ # Apply attention weights to values+ out = attn_weights @ v # (B, T, T) @ (B, T, head_size) -> (B, T, head_size)+ return out++class MultiHeadAttention(nn.Module):+ """+ Multi-Head Attention module.++ This module combines multiple attention heads in parallel. The outputs of each head+ are concatenated to form the final output.++ Args:+ n_head (int): The number of parallel attention heads.+ n_embed (int): The dimensionality of the input embedding.+ context_length (int): The maximum length of the input sequence.+ """+ def __init__(self, n_head: int, n_embed: int, context_length: int) -> None:+ """+ Initializes the multi-head attention module.++ Args:+ n_head (int): The number of parallel attention heads.+ n_embed (int): The dimensionality of the input embedding.+ context_length (int): The maximum length of the input sequence.+ """super().__init__()- self.emb_dim = emb_dim- self.num_heads = num_heads-- assert emb_dim % num_heads == 0-- self.heads = nn.ModuleList([- Attention(emb_dim, emb_dim // num_heads)- for _ in range(num_heads)- ])-- def forward(self, x: torch.Tensor):- return torch.cat(- [attn(x) for attn in self.heads],- dim = -1- )---if __name__ == "__main__":- mha = MultiHeadAttn(16, 4)- x = torch.randn((5, 64, 16))- y = mha(x)- print(x.shape)- print(y.shape)\ No newline at end of file+ self.heads = nn.ModuleList([Head(n_embed // n_head, n_embed, context_length) for _ in range(n_head)])++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ """+ Forward pass through the multi-head attention.++ Args:+ x (torch.Tensor): Input tensor of shape (B, T, C).++ Returns:+ torch.Tensor: Output tensor after concatenating the outputs of all heads.+ """+ # Concatenate the output of each head along the last dimension (C)+ x = torch.cat([h(x) for h in self.heads], dim=-1)+ return x++if __name__ == '__main__':+ # Example Usage (optional, for testing the module independently)+ batch_size = 2+ sequence_length = 5+ embedding_dim = 32+ num_heads = 4+ context_len = 5+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)++ multihead_attn = MultiHeadAttention(n_head=num_heads, n_embed=embedding_dim, context_length=context_len)+ output_tensor = multihead_attn(input_tensor)++ print("MultiHeadAttention Input Shape:", input_tensor.shape)+ print("MultiHeadAttention Output Shape:", output_tensor.shape)\ No newline at end of filediff --git a/src/models/mlp.py b/src/models/mlp.pyindex 7f1b7bf..c7baa79 100644--- a/src/models/mlp.py+++ b/src/models/mlp.py@@ -1,29 +1,80 @@import torch-from torch import nn+import torch.nn as nn+from torch import Tensorclass MLP(nn.Module):- emb_dim: int- relu: nn.Module- in_linear: nn.Linear- out_linear: nn.Linear-- def __init__(self, emb_dim: int):+ """+ A simple Multi-Layer Perceptron with one hidden layer.++ This module is used within the Transformer block for feed-forward processing.+ It expands the input embedding size, applies a ReLU activation, and then projects it back+ to the original embedding size.++ Args:+ n_embed (int): The dimensionality of the input embedding.+ """+ def __init__(self, n_embed: int) -> None:+ """+ Initializes the MLP module.++ Args:+ n_embed (int): The dimensionality of the input embedding.+ """super().__init__()- self.emb_dim = emb_dim- self.in_linear = nn.Linear(emb_dim, emb_dim * 4)- self.relu = nn.ReLU()- self.out_linear = nn.Linear(emb_dim * 4, emb_dim)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- x = self.in_linear(x)- x = self.relu(x)- x = self.out_linear(x)+ self.hidden = nn.Linear(n_embed, 4 * n_embed) # Linear layer to expand embedding size+ self.relu = nn.ReLU() # ReLU activation function+ self.proj = nn.Linear(4 * n_embed, n_embed) # Linear layer to project back to original size++ def forward(self, x: Tensor) -> Tensor:+ """+ Forward pass through the MLP.++ Args:+ x (torch.Tensor): Input tensor of shape (B, T, C), where B is batch size,+ T is sequence length, and C is embedding size.++ Returns:+ torch.Tensor: Output tensor of the same shape as the input.+ """+ x = self.forward_embedding(x)+ x = self.project_embedding(x)return x+ def forward_embedding(self, x: Tensor) -> Tensor:+ """+ Applies the hidden linear layer followed by ReLU activation.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ torch.Tensor: Output after the hidden layer and ReLU.+ """+ x = self.relu(self.hidden(x))+ return x++ def project_embedding(self, x: Tensor) -> Tensor:+ """+ Applies the projection linear layer.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ torch.Tensor: Output after the projection layer.+ """+ x = self.proj(x)+ return x++if __name__ == '__main__':+ # Example Usage (optional, for testing the module independently)+ batch_size = 2+ sequence_length = 3+ embedding_dim = 16+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)++ mlp_module = MLP(n_embed=embedding_dim)+ output_tensor = mlp_module(input_tensor)-if __name__ == "__main__":- mlp = MLP(10)- x = torch.randn((5, 64, 10))- y = mlp(x)- print(x.shape)- print(y.shape)\ No newline at end of file+ print("MLP Input Shape:", input_tensor.shape)+ print("MLP Output Shape:", output_tensor.shape)\ No newline at end of filediff --git a/src/models/transformer.py b/src/models/transformer.pyindex 49d1140..4650753 100644--- a/src/models/transformer.py+++ b/src/models/transformer.py@@ -1,66 +1,136 @@import torch-from torch import nn-from src.models.transformer_block import TransformerBlock+import torch.nn as nn+import torch.nn.functional as F+from src.models.transformer_block import Blockclass Transformer(nn.Module):- text_dim: int- ctx_len: int- emb_dim: int- num_heads: int- num_blocks: int-- text_emb: nn.Embedding- pos_emb: nn.Embedding- blocks: nn.ModuleList- lm_head: nn.Linear-- model_device: torch.device-- def __init__(self, text_dim: int, ctx_len: int, emb_dim: int, num_heads: int, num_blocks: int, device: torch.device = None):+ """+ The main Transformer model.++ This class combines token and position embeddings with a sequence of Transformer blocks+ and a final linear layer for language modeling.++ Args:+ n_head (int): The number of attention heads in each transformer block.+ n_embed (int): The dimensionality of the embedding space.+ context_length (int): The maximum length of the input sequence.+ vocab_size (int): The size of the vocabulary.+ N_BLOCKS (int): The number of transformer blocks in the model.+ """+ def __init__(self, n_head: int, n_embed: int, context_length: int, vocab_size: int, N_BLOCKS: int) -> None:+ """+ Initializes the Transformer model.++ Args:+ n_head (int): Number of attention heads.+ n_embed (int): Embedding dimension.+ context_length (int): Maximum sequence length.+ vocab_size (int): Size of the vocabulary.+ N_BLOCKS (int): Number of transformer blocks.+ """super().__init__()- self.text_dim = text_dim- self.ctx_len = ctx_len- self.emb_dim = emb_dim- self.num_heads = num_heads- self.num_blocks = num_blocks- self.model_device = device-- self.text_emb = nn.Embedding(text_dim, emb_dim)- self.pos_emb = nn.Embedding(ctx_len, emb_dim)- self.blocks = nn.ModuleList([- TransformerBlock(emb_dim, num_heads)- for _ in range(num_blocks)- ])- self.ln_lmhead = nn.LayerNorm(emb_dim)- self.lm_head = nn.Linear(emb_dim, text_dim)-- def forward(self, x: torch.Tensor, get_probs: bool = False, keep_prompt_seg: bool = False):- """Transformer forward pass. Input 2-D tensor contains raw tokens, NOT one-hot encoded."""- assert len(x.shape) == 2 and x.shape[-1] <= self.ctx_len-- x = self.text_emb(x) + self.pos_emb(torch.arange(x.shape[-1], device=self.model_device))- assert len(x.shape) == 3 and x.shape[2] == self.emb_dim-- for block in self.blocks:+ self.context_length = context_length+ self.N_BLOCKS = N_BLOCKS+ self.token_embed = nn.Embedding(vocab_size, n_embed)+ self.position_embed = nn.Embedding(context_length, n_embed)+ self.attn_blocks = nn.ModuleList([Block(n_head, n_embed, context_length) for _ in range(N_BLOCKS)])+ self.layer_norm = nn.LayerNorm(n_embed)+ self.lm_head = nn.Linear(n_embed, vocab_size)+ self.register_buffer('pos_idxs', torch.arange(context_length))++ def _pre_attn_pass(self, idx: torch.Tensor) -> torch.Tensor:+ """+ Combines token and position embeddings.++ Args:+ idx (torch.Tensor): Input token indices.++ Returns:+ torch.Tensor: Sum of token and position embeddings.+ """+ B, T = idx.shape+ tok_embedding = self.token_embed(idx)+ pos_embedding = self.position_embed(self.pos_idxs[:T])+ return tok_embedding + pos_embedding++ def forward(self, idx: torch.Tensor, targets: torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor | None]:+ """+ Forward pass through the Transformer.++ Args:+ idx (torch.Tensor): Input token indices.+ targets (torch.Tensor, optional): Target token indices for loss calculation. Defaults to None.++ Returns:+ tuple: Logits and loss (if targets are provided).+ """+ x = self._pre_attn_pass(idx)+ for block in self.attn_blocks:x = block(x)-- x = self.lm_head(self.ln_lmhead(x))- assert len(x.shape) == 3 and x.shape[2] == self.text_dim-- if get_probs:- # Compute probs instead of logits- x = nn.functional.softmax(x, dim=-1)-- if not keep_prompt_seg:- # Only return the next-token results- x = x[:, -1, :]-- return x--if __name__ == "__main__":- transformer = Transformer(8, 8, 32, 4, 5)- x = torch.randint(8, (3, 7))- y = transformer(x, get_probs = True, keep_prompt_seg = True)- print(x.shape)- print(y.shape)- print(y)\ No newline at end of file+ x = self.layer_norm(x)+ logits = self.lm_head(x)+ loss = None+ if targets is not None:+ B, T, C = logits.shape+ flat_logits = logits.view(B * T, C)+ targets = targets.view(B * T).long()+ loss = F.cross_entropy(flat_logits, targets)+ return logits, loss++ def forward_embedding(self, idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Forward pass focusing on the embedding and attention blocks.++ Args:+ idx (torch.Tensor): Input token indices.++ Returns:+ tuple: Output after attention blocks and the residual.+ """+ x = self._pre_attn_pass(idx)+ residual = x+ for block in self.attn_blocks:+ x, residual = block.forward_embedding(x)+ return x, residual++ def generate(self, idx: torch.Tensor, max_new_tokens: int) -> torch.Tensor:+ """+ Generates new tokens given a starting sequence.++ Args:+ idx (torch.Tensor): Initial sequence of token indices.+ max_new_tokens (int): Number of tokens to generate.++ Returns:+ torch.Tensor: The extended sequence of tokens.+ """+ for _ in range(max_new_tokens):+ idx_cond = idx[:, -self.context_length:]+ logits, _ = self(idx_cond)+ logits = logits[:, -1, :]+ probs = F.softmax(logits, dim=-1)+ idx_next = torch.multinomial(probs, num_samples=1)+ idx = torch.cat((idx, idx_next), dim=1)+ return idx++if __name__ == '__main__':+ # Example Usage (optional, for testing the module independently)+ batch_size = 2+ sequence_length = 5+ vocab_size = 100+ embedding_dim = 32+ num_heads = 4+ num_blocks = 2+ context_len = 5+ input_indices = torch.randint(0, vocab_size, (batch_size, sequence_length))++ transformer_model = Transformer(n_head=num_heads, n_embed=embedding_dim, context_length=context_len, vocab_size=vocab_size, N_BLOCKS=num_blocks)+ logits, loss = transformer_model(input_indices, targets=input_indices) # Using input as target for simplicity++ print("Transformer Logits Shape:", logits.shape)+ print("Transformer Loss:", loss)++ # Example of generating tokens+ start_indices = input_indices[:, :1] # Take the first token of each sequence as start+ generated_tokens = transformer_model.generate(start_indices, max_new_tokens=5)+ print("Generated Tokens Shape:", generated_tokens.shape)\ No newline at end of filediff --git a/src/models/transformer_block.py b/src/models/transformer_block.pyindex 4cb76ed..b6cfd29 100644--- a/src/models/transformer_block.py+++ b/src/models/transformer_block.py@@ -1,23 +1,76 @@import torch-from torch import nn+import torch.nn as nn+from src.models.attention import MultiHeadAttentionfrom src.models.mlp import MLP-from src.models.attention import MultiHeadAttn--class TransformerBlock(nn.Module):- emb_dim: int- attn: MultiHeadAttn- ln_attn: nn.LayerNorm- mlp: MLP- ln_mlp: nn.LayerNorm-- def __init__(self, emb_dim: int, num_heads: int):++class Block(nn.Module):+ """+ A single Transformer block.++ This block consists of a multi-head attention layer followed by an MLP,+ with layer normalization and residual connections.++ Args:+ n_head (int): The number of attention heads in the multi-head attention layer.+ n_embed (int): The dimensionality of the input embedding.+ context_length (int): The maximum length of the input sequence.+ """+ def __init__(self, n_head: int, n_embed: int, context_length: int) -> None:+ """+ Initializes the Transformer block.++ Args:+ n_head (int): The number of attention heads.+ n_embed (int): The dimensionality of the embedding space.+ context_length (int): The maximum sequence length.+ """super().__init__()- self.attn = MultiHeadAttn(emb_dim, num_heads)- self.ln_attn = nn.LayerNorm(emb_dim)- self.mlp = MLP(emb_dim)- self.ln_mlp = nn.LayerNorm(emb_dim)-- def forward(self, x: torch.Tensor):- x = x + self.attn(self.ln_attn(x))- x = x + self.mlp(self.ln_mlp(x))- return x\ No newline at end of file+ self.ln1 = nn.LayerNorm(n_embed)+ self.attn = MultiHeadAttention(n_head, n_embed, context_length)+ self.ln2 = nn.LayerNorm(n_embed)+ self.mlp = MLP(n_embed)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ """+ Forward pass through the Transformer block.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ torch.Tensor: Output tensor after the block.+ """+ # Apply multi-head attention with residual connection+ x = x + self.attn(self.ln1(x))+ # Apply MLP with residual connection+ x = x + self.mlp(self.ln2(x))+ return x++ def forward_embedding(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Forward pass focusing on the embedding and attention parts.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ tuple: A tuple containing the output after MLP embedding and the residual.+ """+ res = x + self.attn(self.ln1(x))+ x = self.mlp.forward_embedding(self.ln2(res))+ return x, res++if __name__ == '__main__':+ # Example Usage (optional, for testing the module independently)+ batch_size = 2+ sequence_length = 5+ embedding_dim = 32+ num_heads = 4+ context_len = 5+ input_tensor = torch.randn(batch_size, sequence_length, embedding_dim)++ transformer_block = Block(n_head=num_heads, n_embed=embedding_dim, context_length=context_len)+ output_tensor = transformer_block(input_tensor)++ print("Transformer Block Input Shape:", input_tensor.shape)+ print("Transformer Block Output Shape:", output_tensor.shape)\ No newline at end of file