Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

7.9 KB
Raw Blame
"""
Generate text with a model trained by scripts/train_transformer.py.

Everything the model needs is read from the checkpoint: its size, its architecture
(classic or modern), its tokenizer (r50k_base or the BPE trained by prepare_tiny_data.py),
and the window it was trained on. So you only pass the checkpoint path:

    python scripts/generate_text.py --model_path models/tiny.pt
    python scripts/generate_text.py --model_path models/student.pt --input_text "The little dog" \
        --max_new_tokens 200 --temperature 0.7 --top_k 40 --num_samples 3
    python scripts/generate_text.py --model_path models/tiny.pt --top_k 0 --min_p 0.05   # min-p sampling
    python scripts/generate_text.py --model_path models/tiny.pt --int8   # int8 linear weights
    python scripts/generate_text.py --model_path models/student.pt --draft_model models/tiny.pt
        # speculative decoding: the tiny model guesses, the student model checks

Very old checkpoints that stored only the weights fall back to the sizes in config/config.py.
"""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

import torch

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # run from the repo without installing

from config.config import default_config as config  # noqa: E402
from src.checkpoint import (  # noqa: E402
    load_checkpoint,
    load_model_weights,
    model_config_from_checkpoint,
    model_state_from_checkpoint,
)
from src.device import resolve_device  # noqa: E402
from src.inference.quantize import Int8Linear, quantize_int8  # noqa: E402
from src.inference.speculative import speculative_generate  # noqa: E402
from src.models.factory import build_model  # noqa: E402
from src.tokenizer import safe_decode, tokenizer_from_spec, tokenizer_spec  # noqa: E402


def load_trained_model(model_path: str, device: str):
    """Rebuild the model, its tokenizer and its training window from a checkpoint."""
    checkpoint = load_checkpoint(model_path, map_location="cpu")
    model_cfg = {**config, **model_config_from_checkpoint(checkpoint)}
    model = build_model(model_cfg)
    load_model_weights(model, model_state_from_checkpoint(checkpoint), source=model_path)
    tokenizer = tokenizer_from_spec(model_cfg.get("tokenizer"))
    # The legacy trainer may train on windows shorter than the model's context (t_context_length).
    # Positions it never saw have untrained embeddings, so generation stays inside that window.
    window = min(model_cfg.get("t_context_length") or model_cfg["context_length"], model_cfg["context_length"])
    return model.to(device).eval(), tokenizer, window, model_cfg


def generate_text(model_path: str, input_text: str, max_new_tokens: int = 100, device: str = "auto",
                  temperature: float = 0.8, top_k: int | None = 50, top_p: float | None = None,
                  min_p: float | None = None) -> str:
    """
    Generates text using a pre-trained Transformer model.

    Args:
        model_path (str): Path to the saved model checkpoint.
        input_text (str): The initial text to start generation from.
        max_new_tokens (int): The maximum number of new tokens to generate.
        device (str): "auto", "cuda", "mps" or "cpu".
        temperature (float): Below 1 is safer and more repetitive, above 1 more random.
        top_k (int, optional): Sample only from the k most likely next tokens.
        top_p (float, optional): Sample from the most likely tokens that add up to this probability.
        min_p (float, optional): Drop tokens less than min_p times as likely as the top token.

    Returns:
        str: The prompt followed by the generated continuation.
    """
    device = resolve_device(device)
    model, tokenizer, window, _ = load_trained_model(model_path, device)
    start_ids = tokenizer.encode_ordinary(input_text)
    context = torch.tensor([start_ids], dtype=torch.long, device=device)
    with torch.no_grad():
        out = model.generate(context, max_new_tokens=max_new_tokens, temperature=temperature,
                             top_k=top_k, top_p=top_p, min_p=min_p, context_window=window)
    return safe_decode(tokenizer, out[0].tolist())


def main() -> None:
    parser = argparse.ArgumentParser(description="Generate text using a pre-trained Transformer model.")
    parser.add_argument("--model_path", type=str, required=True, help="Path to the saved model checkpoint.")
    parser.add_argument("--input_text", type=str, default="Once upon a time", help="The text to continue.")
    parser.add_argument("--max_new_tokens", type=int, default=100, help="Maximum number of new tokens to generate.")
    parser.add_argument("--temperature", type=float, default=0.8, help="Sampling temperature.")
    parser.add_argument("--top_k", type=int, default=50, help="Sample from the k most likely tokens (0 = all).")
    parser.add_argument("--top_p", type=float, default=None, help="Nucleus sampling: keep tokens up to this total probability.")
    parser.add_argument("--min_p", type=float, default=None,
                        help="Keep tokens at least this fraction as likely as the top token, e.g. 0.05.")
    parser.add_argument("--num_samples", type=int, default=1, help="How many continuations to print.")
    parser.add_argument("--device", default="auto", help="auto, cuda, mps or cpu.")
    parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducible samples.")
    parser.add_argument("--int8", action="store_true", help="Store the linear weights as int8 (about 4x smaller).")
    parser.add_argument("--draft_model", default=None,
                        help="A smaller checkpoint with the same tokenizer; turns on speculative decoding.")
    parser.add_argument("--draft_k", type=int, default=4, help="Tokens the draft model guesses per check.")
    args = parser.parse_args()

    if args.seed is not None:
        torch.manual_seed(args.seed)
    device = resolve_device(args.device)
    model, tokenizer, window, model_cfg = load_trained_model(args.model_path, device)
    if args.int8:
        model = quantize_int8(model)
    n_params = sum(p.numel() for p in model.parameters()) + sum(
        m.weight_int8.numel() for m in model.modules() if isinstance(m, Int8Linear))
    print(f"Loaded {args.model_path}: {n_params:,} parameters, {model_cfg.get('arch', 'classic')} "
          f"architecture, window {window} tokens, on {device}")

    draft = None
    if args.draft_model:
        draft, draft_tokenizer, draft_window, _ = load_trained_model(args.draft_model, device)
        if tokenizer_spec(draft_tokenizer) != tokenizer_spec(tokenizer):
            parser.error("--draft_model must use the same tokenizer as --model_path")
        if args.int8:
            draft = quantize_int8(draft)
        print(f"Draft model {args.draft_model} guesses {args.draft_k} tokens at a time")

    context = torch.tensor([tokenizer.encode_ordinary(args.input_text)], dtype=torch.long, device=device)
    for i in range(args.num_samples):
        if draft is None:
            with torch.no_grad():
                out = model.generate(context, max_new_tokens=args.max_new_tokens, temperature=args.temperature,
                                     top_k=args.top_k or None, top_p=args.top_p, min_p=args.min_p,
                                     context_window=window)
            stats = ""
        else:
            out, spec = speculative_generate(model, draft, context, args.max_new_tokens, k=args.draft_k,
                                             temperature=args.temperature, top_k=args.top_k or None,
                                             top_p=args.top_p, min_p=args.min_p,
                                             target_window=window, draft_window=draft_window)
            stats = (f" ({spec.acceptance_rate:.0%} of the draft's guesses kept, "
                     f"{spec.tokens_per_target_call:.2f} tokens per pass of the big model)")
        print(f"\n--- sample {i + 1}{stats} ---\n{safe_decode(tokenizer, out[0].tolist())}")


if __name__ == "__main__":
    main()