Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

1.7 KB
Raw Blame
"""fp8 e4m3 weight with e8m0 scales per 32 input elements (OCP MXFP8)."""

from __future__ import annotations

from typing import Any

import torch

from ..registry import LayerKind, register_method
from ..scheme import MX_GROUP as BLOCK, QuantKind
from .base import LinearKernel, LinearMethod

FP8 = torch.float8_e4m3fn


class TritonMxfp8LinearKernel(LinearKernel):
    """W8A16: split-K GEMV up to M=256, bf16 dequant + cuBLAS above."""

    name = "triton"

    def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
        from freetoken.kernel.triton.mxfp8_linear import mxfp8_linear

        return mxfp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)


class EmulationMxfp8LinearKernel(LinearKernel):
    name = "emulation"

    def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
        from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant

        *lead, K = x.shape
        w = mxfp8_dequant(layer.weight, layer.weight_scale_inv, dtype=torch.float32)
        out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])
        return out + layer.bias.to(out.dtype) if layer.bias is not None else out


@register_method(QuantKind.MXFP8, LayerKind.LINEAR)
class Mxfp8LinearMethod(LinearMethod):
    candidates = (TritonMxfp8LinearKernel, EmulationMxfp8LinearKernel)

    def create_weights(self, layer: Any) -> None:
        g = self.cfg
        if g.in_features % BLOCK:
            raise ValueError(f"mxfp8 needs in_features divisible by {BLOCK}, got {g.in_features}")
        layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
        layer.weight_scale_inv = torch.empty(g.out_features, g.in_features // BLOCK, dtype=torch.uint8)