Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

2.8 KB
Raw Blame
"""
Tokenizers: OpenAI's tiktoken encodings, or a byte-level BPE trained from scratch.

Every tokenizer here is described by a small JSON-friendly dict, so it can travel inside an
HDF5 data file and inside a checkpoint. That keeps a trained model self-contained: the
generation script reads the tokenizer from the checkpoint instead of guessing.

    {"type": "tiktoken", "name": "r50k_base"}          # GPT-2/3 tokenizer, the repo default
    {"type": "bpe", "merges": [...], ...}                # one you trained with BPETokenizer
"""

from __future__ import annotations

import json
from functools import lru_cache
from typing import Any, Protocol

import tiktoken

from src.tokenizer.bpe import EOT, BPETokenizer

DEFAULT_TOKENIZER = {"type": "tiktoken", "name": "r50k_base"}


class Tokenizer(Protocol):
    """The tokenizer interface the scripts rely on (tiktoken.Encoding and BPETokenizer both fit)."""

    @property
    def n_vocab(self) -> int: ...

    @property
    def eot_token(self) -> int: ...

    def encode_ordinary(self, text: str) -> list[int]: ...

    def decode(self, tokens: list[int]) -> str: ...


@lru_cache(maxsize=8)
def _tiktoken(name: str) -> tiktoken.Encoding:
    return tiktoken.get_encoding(name)


def tokenizer_from_spec(spec: dict[str, Any] | str | None) -> Any:
    """Build a tokenizer from a spec dict (or its JSON string, or a tiktoken name)."""
    if spec is None:
        spec = DEFAULT_TOKENIZER
    if isinstance(spec, str):
        spec = json.loads(spec) if spec.lstrip().startswith("{") else {"type": "tiktoken", "name": spec}
    if spec.get("type") == "tiktoken":
        return _tiktoken(str(spec.get("name", "r50k_base")))
    if spec.get("type") == "bpe":
        return BPETokenizer.from_dict(spec)
    raise ValueError(f"unknown tokenizer spec: {spec!r}")


def tokenizer_spec(tokenizer: Any) -> dict[str, Any]:
    """The JSON-friendly description of ``tokenizer`` (the inverse of :func:`tokenizer_from_spec`)."""
    if isinstance(tokenizer, BPETokenizer):
        return tokenizer.to_dict()
    if isinstance(tokenizer, tiktoken.Encoding):
        return {"type": "tiktoken", "name": tokenizer.name}
    raise TypeError(f"cannot describe tokenizer of type {type(tokenizer).__name__}")


def safe_decode(tokenizer: Any, ids: list[int]) -> str:
    """Decode while skipping ids the tokenizer does not know.

    Models pad their vocabulary (50257 -> 50304 for r50k_base) so the embedding table is a
    multiple of 64. An under-trained model can sample one of those padding ids, and tiktoken
    raises on them. Dropping them keeps generation from crashing.
    """
    return tokenizer.decode([int(t) for t in ids if 0 <= int(t) < tokenizer.n_vocab])


__all__ = [
    "BPETokenizer",
    "DEFAULT_TOKENIZER",
    "EOT",
    "Tokenizer",
    "safe_decode",
    "tokenizer_from_spec",
    "tokenizer_spec",
]