FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""fp8 e4m3 weight with square block scales: 128x128 (DeepSeek-V3 style) or 32x32 (DeepSeek-V4.1)."""
from __future__ import annotations
from typing import Any
import torch
from ..registry import LayerKind, register_method
from ..scheme import FP8_BLOCK, QuantKind, fp8_block_size
from .base import LinearConfig, LinearKernel, LinearMethod
FP8 = torch.float8_e4m3fn
E8M0 = torch.float8_e8m0fnu
def _e8m0(cfg: LinearConfig) -> bool:
return cfg.scheme is not None and cfg.scheme.weight.scale == "e8m0"
def layer_block_size(layer: Any) -> int:
"""The block edge a finalized layer was declared with: ``K / (scale columns)``, exact by construction."""
return layer.weight.shape[1] // layer.weight_scale_inv.shape[1]
class Dsv4Fp8BlockLinearKernel(LinearKernel):
"""DeepSeek-V4's reference path: activations quantized to fp8 with power-of-two block scales, e8m0 weight scales read as codes."""
name = "dsv4"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
return None if _e8m0(cfg) else "serves e8m0 block scales only"
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear
return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias, block=layer_block_size(layer))
class TritonFp8BlockLinearKernel(LinearKernel):
"""W8A16 GEMV at M=1, dynamic 1x128 W8A8 GEMM above."""
name = "triton"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
if _e8m0(cfg):
return "reads float block scales; e8m0 codes go to the dsv4 kernel"
if fp8_block_size(cfg.scheme) != FP8_BLOCK:
return f"float-scale kernel serves {FP8_BLOCK}x{FP8_BLOCK} blocks only"
return None
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.fp8_block_linear import block_fp8_linear
return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
@register_method(QuantKind.FP8_BLOCK, LayerKind.LINEAR)
class Fp8BlockLinearMethod(LinearMethod):
candidates = (Dsv4Fp8BlockLinearKernel, TritonFp8BlockLinearKernel)
def create_weights(self, layer: Any) -> None:
g = self.cfg
block = fp8_block_size(g.scheme)
if g.in_features % block or any(o % block for o in g.output_sizes):
raise ValueError(f"block-fp8 needs in/out sizes divisible by {block}, got K={g.in_features} N={g.output_sizes}")
layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
# e8m0 codes stay codes for the dsv4 kernel; float scales are bf16 as the readers push them today
scale_dtype = E8M0 if _e8m0(g) else torch.bfloat16
layer.weight_scale_inv = torch.empty(g.out_features // block, g.in_features // block, dtype=scale_dtype)