Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

14.1 KB
Raw Blame
from __future__ import annotations

import functools
import math
from typing import Any, Mapping, Tuple

import torch

from .base import StateLessOP


class RotaryEmbedding(StateLessOP):
    def __init__(
        self,
        head_size: int,
        rotary_dim: int,
        cos_sin_cache: torch.Tensor,
        is_neox: bool = True,
    ) -> None:
        super().__init__()
        assert 0 < rotary_dim <= head_size
        assert rotary_dim % 2 == 0
        assert head_size in [64, 128, 256, 512]
        self.head_size = head_size
        self.rotary_dim = rotary_dim
        # NeoX (half-rotation, HF default) vs GPT-J interleaved (adjacent pairs,
        # ``rope_interleave`` models: GLM MLA lineage). Both underlying kernels
        # accept the flag; the cos/sin cache layout is identical.
        self.is_neox = is_neox
        # shared with every rope of the same frequencies (rope_cos_sin_table), so never written
        self._cos_sin_cache = cos_sin_cache

        from freetoken.kernel.backend import is_flashinfer_installed

        if is_flashinfer_installed():
            from flashinfer import apply_rope_with_cos_sin_cache_inplace
        else:
            from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace

        self.apply_rope_with_cos_sin_cache_inplace = apply_rope_with_cos_sin_cache_inplace

    def forward(
        self,
        positions: torch.Tensor,
        query: torch.Tensor,
        key: torch.Tensor,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        self.apply_rope_with_cos_sin_cache_inplace(
            positions=positions,
            query=query,
            key=key,
            head_size=self.head_size,
            cos_sin_cache=self._cos_sin_cache,
            is_neox=self.is_neox,
        )
        return query, key


def mrope_cos_sin_rows(
    cos_sin_cache: torch.Tensor, positions: torch.Tensor, section_table: torch.Tensor
) -> torch.Tensor:
    """Per-token [cos | sin] rows for [3, n] positions: frequency slot i reads the cache row of axis section_table[i]."""
    half = cos_sin_cache.shape[1] // 2
    pos = positions[section_table.long(), :].transpose(0, 1).long()  # [n, half]
    idx = torch.arange(half, device=cos_sin_cache.device)
    return torch.cat((cos_sin_cache[pos, idx], cos_sin_cache[pos, half + idx]), dim=1)


def _mrope_torch(
    positions: torch.Tensor,
    query: torch.Tensor,
    key: torch.Tensor,
    head_size: int,
    cos_sin_cache: torch.Tensor,
    section_table: torch.Tensor,
) -> None:
    """Pure-torch mrope for triton-less installs; same in-place NeoX contract as the kernel."""
    nnz = query.shape[0]
    if nnz == 0:
        return
    rotary_dim = cos_sin_cache.shape[1]
    half = rotary_dim // 2
    rows = mrope_cos_sin_rows(cos_sin_cache, positions, section_table)
    cos = rows[:, :half].unsqueeze(1).float()
    sin = rows[:, half:].unsqueeze(1).float()
    for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
        v = t.view(nnz, heads, head_size)
        lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
        # Compute both halves before writing: float() aliases FP32 inputs.
        lo_new = lo * cos - hi * sin
        hi_new = hi * cos + lo * sin
        v[..., :half] = lo_new.to(v.dtype)
        v[..., half:rotary_dim] = hi_new.to(v.dtype)


MROPE_LAYOUTS = ("contiguous", "interleaved", "interleaved_glm")


def build_section_table(mrope_section: tuple[int, int, int], layout: str) -> torch.Tensor:
    """Axis id (0=t, 1=h, 2=w) per frequency slot for a [t, h, w] section split.

    contiguous: T|H|W blocks. interleaved: H at slots 1,4,.. and W at 2,5,.. until each share is used, T takes the rest.
    interleaved_glm: round-robin over the three axes, skipping an axis once its share is used up.
    """
    st, sh, sw = mrope_section
    half = st + sh + sw
    sec = torch.zeros(half, dtype=torch.int32)
    if layout == "contiguous":
        sec[st : st + sh] = 1
        sec[st + sh :] = 2
    elif layout == "interleaved":
        sec[1 : 3 * sh : 3] = 1
        sec[2 : 3 * sw : 3] = 2
        if int((sec == 1).sum()) != sh or int((sec == 2).sum()) != sw:
            raise ValueError(f"mrope_section {mrope_section} is not representable in the interleaved layout")
    elif layout == "interleaved_glm":
        counts = [0, 0, 0]
        for i in range(half):
            ax = i % 3
            while counts[ax] >= mrope_section[ax]:
                ax = (ax + 1) % 3
            sec[i] = ax
            counts[ax] += 1
    else:
        raise ValueError(f"unknown mrope layout {layout!r}; expected one of {MROPE_LAYOUTS}")
    return sec


class MRotaryEmbedding(RotaryEmbedding):
    """3-axis (t/h/w) rope over the parent's cos_sin_cache; section_table picks the axis row per frequency slot. Consumes positions [3, n]."""

    def __init__(
        self, *args, mrope_section: tuple, layout: str = "interleaved", **kwargs
    ) -> None:
        super().__init__(*args, **kwargs)
        assert self.is_neox, "mrope is defined on the NeoX half-rotation layout"
        assert self._cos_sin_cache.shape[1] == self.rotary_dim, "mrope reads a rotary_dim-wide table, not a proportional rope's"
        half = self.rotary_dim // 2
        assert sum(mrope_section) == half, (mrope_section, half)
        self._section_table = build_section_table(tuple(mrope_section), layout)
        try:
            from freetoken.kernel.triton.rope import apply_mrope_with_cos_sin_cache_inplace

            self._kernel = apply_mrope_with_cos_sin_cache_inplace
        except ImportError:
            self._kernel = None

    def forward(
        self,
        positions: torch.Tensor,
        query: torch.Tensor,
        key: torch.Tensor,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        assert positions.dim() == 2, "mrope models feed [3, n] positions"
        if self._section_table.device != query.device:
            self._section_table = self._section_table.to(query.device)
        positions = positions.contiguous()
        if self._kernel is not None:
            self._kernel(
                positions=positions, query=query, key=key, head_size=self.head_size,
                cos_sin_cache=self._cos_sin_cache, section_table=self._section_table,
            )
        else:
            _mrope_torch(
                positions, query, key, self.head_size,
                self._cos_sin_cache, self._section_table,
            )
        return query, key


def rope_table_positions(max_position: int, rope_scaling: Mapping[str, Any] | None) -> int:
    """Rows the cos/sin table needs: YaRN serves ``original_max_position_embeddings * factor``,
    above ``max_position_embeddings`` when only the rope parameters are overridden."""
    if rope_scaling is None or rope_scaling.get("rope_type") != "yarn":
        return max_position
    extended = int(rope_scaling["original_max_position_embeddings"] * rope_scaling["factor"])
    return max(max_position, extended)


def _rope_frequencies(
    head_size: int, rotary_dim: int, base: float, rope_scaling: Mapping[str, Any] | None
) -> Tuple[torch.Tensor, float]:
    """``inv_freq`` over the table's half width, and the factor cos and sin are both scaled by."""
    rope_type = "default" if rope_scaling is None else rope_scaling["rope_type"]
    if rope_type == "proportional":
        # Frequencies spaced over the whole head; the pairs past rotary_dim do not rotate.
        inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2, dtype=torch.float) / head_size))
        inv_freq[rotary_dim // 2 :] = 0.0
        return inv_freq, 1.0
    # Standard (NeoX) rope. Supports partial rotary (rotary_dim < head_size): rope is applied
    # to the first ``rotary_dim`` dims of each head, the rest pass through, and frequencies
    # are spaced over ``rotary_dim`` (HF default partial rope, e.g. Qwen3.5
    # partial_rotary_factor, MiniMax-M2's ``apply_rotary_pos_emb``).
    inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
    match rope_type:
        case "default":
            return inv_freq, 1.0

        case "llama3":
            scaling_factor: float = rope_scaling["factor"]
            low_freq_factor: float = rope_scaling["low_freq_factor"]
            high_freq_factor: float = rope_scaling["high_freq_factor"]
            original_max_position: int = rope_scaling["original_max_position_embeddings"]
            wave_len = 2 * math.pi / inv_freq
            # no smooth if low_freq_factor == high_freq_factor
            if low_freq_factor == high_freq_factor:
                return torch.where(
                    wave_len < original_max_position / high_freq_factor,
                    inv_freq,
                    inv_freq / scaling_factor,
                ), 1.0
            delta = high_freq_factor - low_freq_factor
            smooth = torch.clamp((original_max_position / wave_len - low_freq_factor) / delta, 0, 1)
            return ((1 - smooth) / scaling_factor + smooth) * inv_freq, 1.0

        case "yarn":
            factor: float = rope_scaling["factor"]
            beta_fast: float = rope_scaling.get("beta_fast", 32.0)
            beta_slow: float = rope_scaling.get("beta_slow", 1.0)
            orig_max_pos: int = rope_scaling["original_max_position_embeddings"]

            def get_mscale(scale: float, mscale: float = 1.0) -> float:
                if scale <= 1:
                    return 1.0
                return 0.1 * mscale * math.log(scale) + 1.0

            attention_factor = rope_scaling.get("attention_factor")
            if attention_factor is None:
                mscale = rope_scaling.get("mscale")
                mscale_all_dim = rope_scaling.get("mscale_all_dim")
                # Truthiness, not presence: HF falls back to get_mscale(factor) when
                # mscale_all_dim is 0 (a real DeepSeek-lineage default).
                if mscale and mscale_all_dim:
                    attention_factor = get_mscale(factor, mscale) / get_mscale(
                        factor, mscale_all_dim
                    )
                else:
                    attention_factor = get_mscale(factor)

            def _find_correction_dim(num_rotations: float) -> float:
                return (
                    rotary_dim
                    * math.log(orig_max_pos / (num_rotations * 2 * math.pi))
                    / (2 * math.log(base))
                )

            low = _find_correction_dim(beta_fast)
            high = _find_correction_dim(beta_slow)
            if rope_scaling.get("truncate", True):
                low = math.floor(low)
                high = math.ceil(high)
            low = max(low, 0)
            # rotary_dim - 1, per HF's find_correction_range and this repo's own faithful copy in
            # models/deepseek_v4/ops.py. Clamping to rotary_dim//2 - 1 instead forces the ramp to
            # reach 1.0 at the last entry, fully interpolating the longest-wavelength dims that
            # the reference deliberately leaves partly extrapolated.
            high = min(high, rotary_dim - 1)
            if low == high:  # HF nudges instead of flooring the gap at 1 ("truncate": false)
                high += 0.001
            ramp = torch.clamp(
                (torch.arange(rotary_dim // 2, dtype=torch.float32) - low) / (high - low), 0, 1
            )
            return (inv_freq / factor) * ramp + inv_freq * (1 - ramp), float(attention_factor)

    raise ValueError(f"Unsupported {rope_scaling = }")


@functools.cache
def _cos_sin_table(
    head_size: int,
    rotary_dim: int,
    positions: int,
    base: float,
    rope_scaling: Tuple[Tuple[str, Any], ...] | None,
    device: torch.device,
) -> torch.Tensor:
    inv_freq, attention_factor = _rope_frequencies(
        head_size, rotary_dim, base, dict(rope_scaling) if rope_scaling is not None else None
    )
    freqs = torch.arange(positions, dtype=torch.float)[:, None] * inv_freq[None, :]
    return torch.cat((freqs.cos() * attention_factor, freqs.sin() * attention_factor), dim=-1)


def rope_cos_sin_table(
    head_size: int,
    rotary_dim: int,
    max_position: int,
    base: float,
    rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,
) -> torch.Tensor:
    """The ``[positions, width]`` cos|sin table, one per distinct rope: only proportional rope
    depends on the head size, so an indexer whose heads differ in width reads the attention's table."""
    rope_map = dict(rope_scaling) if rope_scaling is not None else None
    proportional = rope_map is not None and rope_map["rope_type"] == "proportional"
    device = torch.get_default_device()
    if device.type == "cuda" and device.index is None:
        device = torch.device("cuda", torch.cuda.current_device())
    return _cos_sin_table(
        head_size if proportional else rotary_dim,
        rotary_dim,
        rope_table_positions(max_position, rope_map),
        base,
        rope_scaling,
        device,
    )


_ROPE_DEVICE: torch.device | None = None


def set_rope_device(device: torch.device):
    global _ROPE_DEVICE
    _ROPE_DEVICE = device


@functools.cache
def get_rope(
    head_dim: int,
    rotary_dim: int,
    max_position: int,
    base: float,
    rope_scaling: Tuple[Tuple[str, Any], ...] | None = None,
    is_neox: bool = True,
    mrope_section: Tuple[int, ...] | None = None,
    mrope_layout: str = "interleaved",
) -> RotaryEmbedding:
    def build() -> RotaryEmbedding:
        table = rope_cos_sin_table(head_dim, rotary_dim, max_position, base, rope_scaling)
        if mrope_section is not None:
            return MRotaryEmbedding(
                head_dim, rotary_dim, table,
                is_neox=is_neox, mrope_section=tuple(mrope_section),
                layout=mrope_layout,
            )
        return RotaryEmbedding(head_dim, rotary_dim, table, is_neox=is_neox)

    t = torch.tensor([])
    if t.device == torch.device("meta"):
        # we cannot use meta device for rope
        if _ROPE_DEVICE is None:
            raise RuntimeError(
                "We cannot use meta device for rope. Please call set_rope_device() first."
            )
        with torch.device(_ROPE_DEVICE):
            return build()
    return build()


__all__ = [
    "MROPE_LAYOUTS",
    "MRotaryEmbedding",
    "RotaryEmbedding",
    "build_section_table",
    "get_rope",
    "rope_cos_sin_table",
    "rope_table_positions",
    "set_rope_device",
]