Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.0 KB
Raw Blame
"""fp8 e4m3 experts with 128x128 block scales."""

from __future__ import annotations

import torch

from ..registry import LayerKind, register_method
from ..scheme import FP8_BLOCK as BLOCK, QuantKind, fp8_block_size
from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, limit_or_inf, MoEConfig, MoEKernel, MoEMethod

FP8 = torch.float8_e4m3fn


def _pad_scale(rows: int, cols: int) -> int:
    from freetoken.kernel.aot_models import fp8_block_scale_pad

    return fp8_block_scale_pad(rows, cols)


class TritonFp8BlockMoEKernel(MoEKernel):
    name = "triton"
    cpu_format = "fp8_block"
    supports_inactive_slots = True

    def unusable_reason(self, cfg: MoEConfig) -> str | None:
        reason = self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)
        if reason:
            return reason
        if cfg.scheme is not None and fp8_block_size(cfg.scheme) != BLOCK:
            return f"fp8 block MoE kernel serves {BLOCK}x{BLOCK} expert blocks only"
        reason = gated_epilogue_reason(cfg)
        if reason:
            return f"fp8 block MoE kernel: {reason}"
        if cfg.apply_router_weight_on_input:
            return "fp8 block MoE kernel cannot apply the router weight on the input"
        return None

    def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
        i, h, b = cfg.intermediate, cfg.hidden, BLOCK
        return {
            "gate_up": BankSpec((2 * i, h), FP8),
            "gate_up_scale": BankSpec((2 * i // b, _pad_scale(2 * i // b, h // b)), torch.bfloat16),
            "down": BankSpec((h, i), FP8),
            "down_scale": BankSpec((h // b, _pad_scale(h // b, i // b)), torch.bfloat16),
        }

    def pack(self, pieces, cfg: MoEConfig, out):
        out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
        out["down"].copy_(pieces["down"])
        # the scale banks carry 16-byte padding on the last dim
        gs = fused_piece(pieces, "gate_up_scale")
        out["gate_up_scale"][:, :, : gs.shape[2]].copy_(gs)
        ds = pieces["down_scale"]
        out["down_scale"][:, :, : ds.shape[2]].copy_(ds)
        return {}

    def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
        from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block, fused_experts_fp8_block

        t = view.tensors
        alpha, limit = float(layer.alpha), limit_or_inf(layer)
        if is_prefill:
            n = view.n if view.n is not None else layer.num_experts
            return fused_experts_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
        return fused_experts_decode_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)


@register_method(QuantKind.FP8_BLOCK, LayerKind.MOE)
class Fp8BlockMoEMethod(MoEMethod):
    candidates = (TritonFp8BlockMoEKernel,)