Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.3 KB
Raw Blame
from __future__ import annotations

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    import torch


def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool:
    """flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes."""
    import torch

    from freetoken.kernel.backend import is_flashinfer_installed

    return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed()


def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
    if _use_flashinfer(x, out):
        from flashinfer import silu_and_mul
    else:
        from freetoken.kernel.triton.activation import silu_and_mul

    return silu_and_mul(x, out=out)


def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
    if _use_flashinfer(x, out):
        from flashinfer import gelu_and_mul
    else:
        from freetoken.kernel.triton.activation import gelu_and_mul

    return gelu_and_mul(x, out=out)


def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
    """tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul."""
    if _use_flashinfer(x, out):
        from flashinfer import gelu_tanh_and_mul
    else:
        from freetoken.kernel.triton.activation import gelu_tanh_and_mul

    return gelu_tanh_and_mul(x, out=out)


def swigluoai_and_mul(
    x: torch.Tensor,
    out: torch.Tensor | None = None,
    *,
    alpha: float = 1.702,
    limit: float = 7.0,
):
    """SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves
    (gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) *
    sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton
    kernel (flashinfer ships no clamped-swiglu *_and_mul)."""
    from freetoken.kernel.triton.activation import swigluoai_and_mul

    return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)


def swiglu_clamp_and_mul(
    x, out=None, *, alpha: float = 1.0, limit: float = 10.0
):
    """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``."""
    from freetoken.kernel.triton.activation import swiglu_clamp_and_mul

    return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)


GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp")


def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")):
    """The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``."""
    if activation == "silu":
        return silu_and_mul(x, out)
    if activation == "gelu":
        return gelu_and_mul(x, out)
    if activation == "gelu_tanh":
        return gelu_tanh_and_mul(x, out)
    if activation == "swigluoai":
        return swigluoai_and_mul(x, out, alpha=alpha, limit=limit)
    if activation == "swiglu_clamp":
        return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit)
    raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}")


__all__ = [
    "silu_and_mul",
    "gelu_and_mul",
    "gelu_tanh_and_mul",
    "swigluoai_and_mul",
    "swiglu_clamp_and_mul",
    "gated_act_and_mul",
    "GATED_ACTIVATIONS",
]