Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.9 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
# Adapted from SGLang (kernels/ops/elementwise.py, ``_fused_gate_sigmoid_mul_add``)
"""Gated shared-expert epilogue: the gate reduction and the sigmoid-mul-add.

The routed experts may write into ``hidden_states`` in place, so the gate reduction runs
before them and the mul-add after.
"""

from __future__ import annotations

import functools

import torch
import triton
import triton.language as tl

from freetoken.utils.arch import is_sm90_supported


@functools.cache
def _pdl_supported() -> bool:
    return is_sm90_supported()


def _reduction_warps(hidden_dim: int, num_tokens: int) -> int:
    warps = max(min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), 32), 4)
    return min(warps, 8) if num_tokens >= 1024 else warps


@triton.jit
def _gate_sigmoid_kernel(
    hidden_ptr,
    weight_ptr,
    gate_ptr,
    stride_h,
    HIDDEN: tl.constexpr,
    BLOCK_SIZE: tl.constexpr,
    launch_pdl: tl.constexpr,
) -> None:
    # row * stride can overflow int32 for large token counts.
    row = tl.program_id(0).to(tl.int64)
    offs = tl.arange(0, BLOCK_SIZE)
    mask = offs < HIDDEN

    w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32)

    if launch_pdl:
        tl.extra.cuda.gdc_wait()

    h = tl.load(hidden_ptr + row * stride_h + offs, mask=mask, other=0.0).to(tl.float32)

    if launch_pdl:
        tl.extra.cuda.gdc_launch_dependents()

    tl.store(gate_ptr + row, tl.sigmoid(tl.sum(h * w, axis=0)))


def shared_gate_sigmoid(hidden_states: torch.Tensor, gate_weight: torch.Tensor) -> torch.Tensor:
    """Per-token ``sigmoid(hidden_states @ gate_weight)`` as fp32 [num_tokens]."""
    num_tokens, hidden_dim = hidden_states.shape
    assert hidden_states.stride(1) == 1, "shared gate requires unit inner stride"
    assert gate_weight.shape == (hidden_dim,) and gate_weight.is_contiguous()

    gate = torch.empty(num_tokens, dtype=torch.float32, device=hidden_states.device)
    _gate_sigmoid_kernel[(num_tokens,)](
        hidden_states,
        gate_weight,
        gate,
        hidden_states.stride(0),
        HIDDEN=hidden_dim,
        BLOCK_SIZE=triton.next_power_of_2(hidden_dim),
        launch_pdl=_pdl_supported(),
        num_warps=_reduction_warps(hidden_dim, num_tokens),
    )
    return gate


@triton.jit
def _gate_mul_add_kernel(
    routed_ptr,
    shared_ptr,
    gate_ptr,
    out_ptr,
    stride_r,
    stride_s,
    stride_o,
    HIDDEN: tl.constexpr,
    BLOCK_SIZE: tl.constexpr,
    launch_pdl: tl.constexpr,
) -> None:
    row = tl.program_id(0).to(tl.int64)
    block = tl.program_id(1)
    offs = block * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
    mask = offs < HIDDEN

    if launch_pdl:
        tl.extra.cuda.gdc_wait()

    gate = tl.load(gate_ptr + row)
    routed = tl.load(routed_ptr + row * stride_r + offs, mask=mask, other=0.0).to(tl.float32)
    shared = tl.load(shared_ptr + row * stride_s + offs, mask=mask, other=0.0).to(tl.float32)

    if launch_pdl:
        tl.extra.cuda.gdc_launch_dependents()

    tl.store(out_ptr + row * stride_o + offs, routed + gate * shared, mask=mask)


def shared_gate_mul_add(
    routed: torch.Tensor, shared: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
    """``routed + gate[:, None] * shared`` into a fresh tensor."""
    num_tokens, hidden_dim = routed.shape
    assert shared.shape == routed.shape
    assert routed.stride(1) == 1 and shared.stride(1) == 1
    assert gate.shape == (num_tokens,)

    out = torch.empty_like(routed)
    block_size = min(triton.next_power_of_2(hidden_dim), 2048)
    _gate_mul_add_kernel[(num_tokens, triton.cdiv(hidden_dim, block_size))](
        routed,
        shared,
        gate,
        out,
        routed.stride(0),
        shared.stride(0),
        out.stride(0),
        HIDDEN=hidden_dim,
        BLOCK_SIZE=block_size,
        launch_pdl=_pdl_supported(),
        num_warps=4,
    )
    return out


__all__ = ["shared_gate_mul_add", "shared_gate_sigmoid"]