Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

2.1 KB
Raw Blame
"""
Actor-critic wrapper for PPO.

PPO needs a per-token *value* estimate V(s_t) alongside the policy logits. We get both
from a single backbone: the wrapper reuses the :class:`Transformer`'s ``forward_hidden``
and ``lm_head`` for the policy, and adds a small scalar value head on top of the same
hidden states. Sharing the backbone is the standard, memory-cheap choice; the value head
is initialized to output ~0 so early training is stable.
"""

from __future__ import annotations

import torch
import torch.nn as nn

from src.models.factory import LanguageModel


class TransformerWithValueHead(nn.Module):
    """Wrap a :class:`Transformer` and add a scalar value head for PPO.

    ``forward(idx)`` returns ``(logits, values)`` where ``logits`` are the policy logits
    (B, T, vocab) and ``values`` are per-token value estimates (B, T).
    """

    def __init__(self, transformer: LanguageModel) -> None:
        super().__init__()
        self.transformer = transformer
        n_embed = transformer.lm_head.in_features
        hidden, out = nn.Linear(n_embed, n_embed), nn.Linear(n_embed, 1)
        # Start with values ~0 so the critic does not destabilize the policy early on.
        nn.init.zeros_(out.weight)
        nn.init.zeros_(out.bias)
        self.value_head = nn.Sequential(hidden, nn.ReLU(), out)

    # Expose the backbone's context length so rollout's cap detection works.
    @property
    def context_length(self) -> int:
        return self.transformer.context_length

    def forward(self, idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
        hidden = self.transformer.forward_hidden(idx)        # (B, T, n_embed)
        logits = self.transformer.lm_head(hidden)            # (B, T, vocab)
        values = self.value_head(hidden).squeeze(-1)         # (B, T)
        return logits, values

    @torch.no_grad()
    def value_only(self, idx: torch.Tensor) -> torch.Tensor:
        """Per-token values without computing the (large) vocab logits -- used when
        scoring rollouts where only V(s_t) is needed."""
        hidden = self.transformer.forward_hidden(idx)
        return self.value_head(hidden).squeeze(-1)