Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

10.1 KB
Raw Blame
"""Tiled Triton RoPE (provenance: rope<-lightllm).

Adapted from lightllm's rotary_emb kernel (models/llama/triton_kernel/rotary_emb.py):
grid = (cdiv(nnz, BLOCK_SEQ), cdiv(num_heads, BLOCK_HEAD)); each program processes a
BLOCK_SEQ x BLOCK_HEAD x (rotary_dim/2) tile. Changed vs upstream:

  * upstream takes pre-gathered cos/sin (nnz, dim/2); FreeToken passes a
    cos_sin_cache (max_pos, rotary_dim) plus positions -> we gather
    cache[positions[token]] inside the kernel (cos = first half, sin = second
    half over rotary_dim).
  * Q and K are rotated in a single kernel launch (one grid over max head
    count, K stores masked past HEAD_K) to cut launch overhead for tiny decode
    batches.
  * rotation math in fp32 like the vendored/flashinfer kernel; is_neox and
    interleave both supported as constexpr.

Optional pure-triton drop-in for
freetoken.kernel.rope.apply_rope_with_cos_sin_cache_inplace.
"""

from __future__ import annotations

import torch
import triton
import triton.language as tl


@triton.jit(do_not_specialize=["nnz"])
def _rope_tiled(
    Q, K, POS, CACHE,
    stride_qbs, stride_qh, stride_qd,
    stride_kbs, stride_kh, stride_kd,
    nnz,
    HEAD_Q, HEAD_K, rotary_dim, half,
    HAS_K: tl.constexpr,
    INTERLEAVE: tl.constexpr,
    BLOCK_SEQ: tl.constexpr,
    BLOCK_HEAD: tl.constexpr,
    BLOCK_DHALF: tl.constexpr,
):
    seq_pid = tl.program_id(0)
    head_pid = tl.program_id(1)

    seq_range = seq_pid * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ)      # (S,)
    head_range = head_pid * BLOCK_HEAD + tl.arange(0, BLOCK_HEAD)  # (H,)
    d = tl.arange(0, BLOCK_DHALF)                                  # (D,)

    seq_mask = seq_range < nnz
    dmask = d < half

    pos = tl.load(POS + seq_range, mask=seq_mask, other=0).to(tl.int64)   # (S,)
    cache_row = CACHE + pos[:, None] * rotary_dim
    cs_mask = seq_mask[:, None] & dmask[None, :]
    cos = tl.load(cache_row + d[None, :], mask=cs_mask, other=0.0)         # (S,D) fp32
    sin = tl.load(cache_row + half + d[None, :], mask=cs_mask, other=0.0)  # (S,D) fp32
    cos = cos[:, None, :]   # (S,1,D)
    sin = sin[:, None, :]

    if INTERLEAVE:
        d0 = 2 * d
        d1 = 2 * d + 1
    else:
        d0 = d
        d1 = half + d
    d0 = d0[None, None, :]
    d1 = d1[None, None, :]

    # --- Q ---
    qmask = (seq_mask[:, None, None]
             & (head_range[None, :, None] < HEAD_Q)
             & dmask[None, None, :])
    base_q = seq_range[:, None, None] * stride_qbs + head_range[None, :, None] * stride_qh
    q0 = tl.load(Q + base_q + d0 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
    q1 = tl.load(Q + base_q + d1 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
    o0 = q0 * cos - q1 * sin
    o1 = q1 * cos + q0 * sin
    tl.store(Q + base_q + d0 * stride_qd, o0.to(Q.dtype.element_ty), mask=qmask)
    tl.store(Q + base_q + d1 * stride_qd, o1.to(Q.dtype.element_ty), mask=qmask)

    # --- K ---
    if HAS_K:
        kmask = (seq_mask[:, None, None]
                 & (head_range[None, :, None] < HEAD_K)
                 & dmask[None, None, :])
        base_k = seq_range[:, None, None] * stride_kbs + head_range[None, :, None] * stride_kh
        k0 = tl.load(K + base_k + d0 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
        k1 = tl.load(K + base_k + d1 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
        ok0 = k0 * cos - k1 * sin
        ok1 = k1 * cos + k0 * sin
        tl.store(K + base_k + d0 * stride_kd, ok0.to(K.dtype.element_ty), mask=kmask)
        tl.store(K + base_k + d1 * stride_kd, ok1.to(K.dtype.element_ty), mask=kmask)


def apply_rope_with_cos_sin_cache_inplace(
    positions: torch.Tensor,
    query: torch.Tensor,
    key: torch.Tensor,
    head_size: int,
    cos_sin_cache: torch.Tensor,
    is_neox: bool = True,
) -> None:
    """Drop-in for the vendored op; rotates query/key in place."""
    if str(cos_sin_cache.dtype) != "torch.float32":
        raise ValueError("cos_sin_cache should be float32")
    assert query.is_cuda and key.is_cuda and positions.is_cuda
    assert cos_sin_cache.is_contiguous()

    nnz = query.shape[0]
    if nnz == 0:
        return
    rotary_dim = cos_sin_cache.shape[1]
    half = rotary_dim // 2
    block_dhalf = triton.next_power_of_2(half)

    head_q = query.shape[1] // head_size
    head_k = key.shape[1] // head_size
    qv = query.view(nnz, head_q, head_size)
    kv = key.view(nnz, head_k, head_size)

    max_head = max(head_q, head_k)

    grid = lambda META: (
        triton.cdiv(nnz, META["BLOCK_SEQ"]),
        triton.cdiv(max_head, META["BLOCK_HEAD"]),
    )
    # Fixed via H100 sweep (27-config grid; 16/1/w4 within 5% of the winner at
    # every nnz 1..4096, faster than tuned on average).
    _rope_tiled[grid](
        qv, kv, positions, cos_sin_cache,
        qv.stride(0), qv.stride(1), qv.stride(2),
        kv.stride(0), kv.stride(1), kv.stride(2),
        nnz, head_q, head_k, rotary_dim, half,
        HAS_K=True,
        INTERLEAVE=not is_neox,
        BLOCK_DHALF=block_dhalf,
        BLOCK_SEQ=16,
        BLOCK_HEAD=1,
        num_warps=4,
        num_stages=1,
    )


__all__ = ["apply_rope_with_cos_sin_cache_inplace"]


@triton.jit(do_not_specialize=["nnz"])
def _mrope_tiled(
    Q, K, POS, CACHE, SEC,
    stride_qbs, stride_qh, stride_qd,
    stride_kbs, stride_kh, stride_kd,
    nnz,
    HEAD_Q, HEAD_K, rotary_dim, half,
    BLOCK_SEQ: tl.constexpr,
    BLOCK_HEAD: tl.constexpr,
    BLOCK_DHALF: tl.constexpr,
):
    # _rope_tiled with 3-axis positions: POS is [3, nnz], SEC[half] picks the axis each frequency slot reads
    seq_pid = tl.program_id(0)
    head_pid = tl.program_id(1)

    seq_range = seq_pid * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ)
    head_range = head_pid * BLOCK_HEAD + tl.arange(0, BLOCK_HEAD)
    d = tl.arange(0, BLOCK_DHALF)

    seq_mask = seq_range < nnz
    dmask = d < half

    sec = tl.load(SEC + d, mask=dmask, other=0).to(tl.int64)  # (D,) in {0,1,2}
    pos_ptr = POS + sec[None, :] * nnz + seq_range[:, None]
    cs_mask = seq_mask[:, None] & dmask[None, :]
    pos = tl.load(pos_ptr, mask=cs_mask, other=0).to(tl.int64)  # (S,D)
    cache_row = CACHE + pos * rotary_dim
    cos = tl.load(cache_row + d[None, :], mask=cs_mask, other=0.0)
    sin = tl.load(cache_row + half + d[None, :], mask=cs_mask, other=0.0)
    cos = cos[:, None, :]
    sin = sin[:, None, :]

    d0 = d[None, None, :]
    d1 = (half + d)[None, None, :]

    qmask = (seq_mask[:, None, None]
             & (head_range[None, :, None] < HEAD_Q)
             & dmask[None, None, :])
    base_q = seq_range[:, None, None] * stride_qbs + head_range[None, :, None] * stride_qh
    q0 = tl.load(Q + base_q + d0 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
    q1 = tl.load(Q + base_q + d1 * stride_qd, mask=qmask, other=0.0).to(tl.float32)
    tl.store(Q + base_q + d0 * stride_qd, (q0 * cos - q1 * sin).to(Q.dtype.element_ty), mask=qmask)
    tl.store(Q + base_q + d1 * stride_qd, (q1 * cos + q0 * sin).to(Q.dtype.element_ty), mask=qmask)

    kmask = (seq_mask[:, None, None]
             & (head_range[None, :, None] < HEAD_K)
             & dmask[None, None, :])
    base_k = seq_range[:, None, None] * stride_kbs + head_range[None, :, None] * stride_kh
    k0 = tl.load(K + base_k + d0 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
    k1 = tl.load(K + base_k + d1 * stride_kd, mask=kmask, other=0.0).to(tl.float32)
    tl.store(K + base_k + d0 * stride_kd, (k0 * cos - k1 * sin).to(K.dtype.element_ty), mask=kmask)
    tl.store(K + base_k + d1 * stride_kd, (k1 * cos + k0 * sin).to(K.dtype.element_ty), mask=kmask)


def apply_mrope_with_cos_sin_cache_inplace(
    positions: torch.Tensor,
    query: torch.Tensor,
    key: torch.Tensor,
    head_size: int,
    cos_sin_cache: torch.Tensor,
    section_table: torch.Tensor,
) -> None:
    """3-axis rope: positions is [3, nnz], section_table [rotary_dim//2] maps each frequency slot to its axis row; NeoX layout only."""
    if str(cos_sin_cache.dtype) != "torch.float32":
        raise ValueError("cos_sin_cache should be float32")
    assert positions.dim() == 2 and positions.shape[0] == 3
    assert query.is_cuda and key.is_cuda and positions.is_cuda
    assert cos_sin_cache.is_contiguous() and positions.is_contiguous()

    nnz = query.shape[0]
    if nnz == 0:
        return
    rotary_dim = cos_sin_cache.shape[1]
    half = rotary_dim // 2

    head_q = query.shape[1] // head_size
    head_k = key.shape[1] // head_size
    qv = query.view(nnz, head_q, head_size)
    kv = key.view(nnz, head_k, head_size)
    max_head = max(head_q, head_k)

    grid = lambda META: (
        triton.cdiv(nnz, META["BLOCK_SEQ"]),
        triton.cdiv(max_head, META["BLOCK_HEAD"]),
    )
    _mrope_tiled[grid](
        qv, kv, positions, cos_sin_cache, section_table,
        qv.stride(0), qv.stride(1), qv.stride(2),
        kv.stride(0), kv.stride(1), kv.stride(2),
        nnz, head_q, head_k, rotary_dim, half,
        BLOCK_DHALF=triton.next_power_of_2(half),
        BLOCK_SEQ=16,
        BLOCK_HEAD=1,
        num_warps=4,
        num_stages=1,
    )


def apply_mrope_torch_fallback(
    positions: torch.Tensor,
    query: torch.Tensor,
    key: torch.Tensor,
    head_size: int,
    cos_sin_cache: torch.Tensor,
    section_table: torch.Tensor,
) -> None:
    """Pure-torch mrope (Triton unavailable); same in-place contract as the kernel."""
    nnz = query.shape[0]
    if nnz == 0:
        return
    rotary_dim = cos_sin_cache.shape[1]
    half = rotary_dim // 2
    pos = positions[section_table.long(), :].transpose(0, 1)  # [nnz, half]
    cs = cos_sin_cache[pos]  # [nnz, half, 2*half] rows gathered per slot
    idx = torch.arange(half, device=query.device)
    cos = cs[:, idx, idx].unsqueeze(1).float()
    sin = cs[:, idx, half + idx].unsqueeze(1).float()
    for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
        v = t.view(nnz, heads, head_size)
        lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
        # Compute both halves before writing: float() aliases FP32 inputs.
        lo_new = lo * cos - hi * sin
        hi_new = hi * cos + lo * sin
        v[..., :half] = lo_new.to(v.dtype)
        v[..., half:rotary_dim] = hi_new.to(v.dtype)


__all__ = ["apply_rope_with_cos_sin_cache_inplace", "apply_mrope_with_cos_sin_cache_inplace"]