Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

7.1 KB
Raw Blame
"""e4m3 (fp8e4nv) emulation for pre-sm_89 CUDA GPUs.

Triton rejects the fp8e4nv type anywhere in a kernel compiled for sm < 89 -- the
check sits in ``dtype.to_ir``, so even an fp8 *pointer argument* is illegal.
Affected kernels branch on :func:`e4m3_native_cx` (a compile-time constexpr): the
native branch casts fp32 through :func:`f32_to_e4m3`, the emulated branch is dead-code
eliminated there. When the emulated branch is active, wrappers must pass e4m3
tensors as ``.view(torch.uint8)`` and allocate act-quant outputs as bf16 -- use
the host-side twin :func:`e4m3_native` for those decisions.

``FREETOKEN_FORCE_E4M3_EMU=1`` (or true/yes/on) forces the emulated path on any
GPU (for A/B validation against the native fp8 unit). The flag is read ONCE at
import and is deliberately NOT part of triton's compilation cache key, so:
flipping it later in the same process raises (see :func:`e4m3_native`), and when
it is set without an explicit ``TRITON_CACHE_DIR`` the default cache dir is
salted so a warm native cache can never serve the other branch's binary for the
signature-invariant kernels (``_act_quant_inplace_kernel``).

The primitives are validated bit-exact against the native fp8 unit on H100 (all
254 non-NaN codes; 2.1M adversarial rounding samples covering every grid
midpoint, the subnormal range and the 2^-6 boundary). Known deviations: the NaN
codes 0x7F/0xFF decode to +-480 (checkpoints never store NaN weights), and
``round_e4m3(-0.0)`` returns +0.0.
"""

from __future__ import annotations

import os

import torch
import triton.language as tl
from triton import jit
from triton.language import target_info
from triton.runtime.jit import constexpr_function

def _env_force() -> bool:
    return os.environ.get("FREETOKEN_FORCE_E4M3_EMU", "").lower() in ("1", "true", "yes", "on")


FORCE_EMU = _env_force()

if FORCE_EMU and "TRITON_CACHE_DIR" not in os.environ:
    os.environ["TRITON_CACHE_DIR"] = os.path.join(
        os.path.expanduser("~/.triton"), "cache-e4m3emu")

_native: bool | None = None


def e4m3_native() -> bool:
    """Host-side twin of :func:`e4m3_native_cx`: True when kernels take fp8e4nv
    tensors directly. False: pass ``.view(torch.uint8)`` and bf16 act buffers."""
    global _native
    if _env_force() != FORCE_EMU:
        raise RuntimeError(
            "FREETOKEN_FORCE_E4M3_EMU changed after import: the flag is read once at "
            "import and is not part of triton's compile cache key -- set it before "
            "the process starts (with its own TRITON_CACHE_DIR)"
        )
    if _native is None:
        if FORCE_EMU:
            _native = False
        elif torch.version.hip is not None:
            # ROCm reports gfx1101 as capability (11, 0), which is not a CUDA
            # compute capability and must not select the native fp8e4nv path.
            _native = False
        else:
            from freetoken.gpu_select import assigned_visible_gpu

            # one process runs on one GPU, so its convention is that GPU's; None (-> the current device) only before the process binds
            _native = torch.cuda.get_device_capability(assigned_visible_gpu()) >= (8, 9)
    return _native


def e4m3_kernel_view(t: torch.Tensor) -> torch.Tensor:
    """An e4m3 tensor as the branched kernels expect it: unchanged when native,
    the uint8 view otherwise (the fp8 pointer type is illegal pre-sm_89)."""
    return t if e4m3_native() else t.view(torch.uint8)


def e4m3_act_dtype() -> torch.dtype:
    """Buffer dtype for quantized activations: fp8 when native, else bf16 (every
    e4m3 grid value is exactly representable)."""
    return torch.float8_e4m3fn if e4m3_native() else torch.bfloat16


@constexpr_function
def _f32_cast_rounds_once_cx():
    return not target_info.is_cuda() or target_info.cuda_capability_geq(9, 0)


@constexpr_function
def e4m3_native_cx():
    """Compile-time: does the compilation target have native fp8e4nv (sm_89+)?
    Delegates to ``target_info`` (reads the active driver's target, so
    cross-compilation tests that patch ``driver.active.get_current_target``
    resolve consistently)."""
    return not FORCE_EMU and target_info.cuda_capability_geq(8, 9)


@jit
def e4m3_u8_to_f32(v):
    """Decode e4m3 bits (uint8) to fp32: place exp+mantissa in the fp16 field
    (exact, incl. e4m3 subnormals) and rescale by 2^(15-7). NaN codes -> +-480."""
    h = ((v & 0x80).to(tl.uint16) << 8) | ((v & 0x7F).to(tl.uint16) << 7)
    return h.to(tl.float16, bitcast=True).to(tl.float32) * 256.0


@jit
def e4m3_u8_to_f16_x128(v):
    """Decode e4m3 bits (uint8) to fp16 pre-scaled by 128 (the nvfp4 GEMM form:
    native is ``.to(tl.float16) * 128``). (val/256) * 2^15 stays in fp16 range
    (max 448*128 = 57344) and is exact (power-of-two scaling)."""
    h = ((v & 0x80).to(tl.uint16) << 8) | ((v & 0x7F).to(tl.uint16) << 7)
    return h.to(tl.float16, bitcast=True) * 32768.0


@jit
def e4m3_f32_to_u8(x):
    """Encode an fp32 value that already sits on the e4m3 grid (see :func:`round_e4m3`, |x| <= 448)
    into its e4m3 bit pattern (uint8). Normals: fp32 exponent rebased 127 -> 7 with the top three
    mantissa bits; subnormals (|x| < 2^-6, grid 2^-9): mantissa = |x| / 2^-9, exponent field 0."""
    b = x.to(tl.uint32, bitcast=True)
    sign = ((b >> 31) & 1).to(tl.uint8) << 7
    exp = ((b >> 23) & 0xFF).to(tl.int32) - 127 + 7
    man = ((b >> 20) & 0x7).to(tl.uint8)
    normal = (exp.to(tl.uint8) << 3) | man
    sub = (tl.abs(x) * 512.0).to(tl.uint8)  # 0..7 on the 2^-9 grid
    return sign | tl.where(tl.abs(x) >= 0.015625, normal, sub)


@jit
def round_e4m3(x):
    """Round fp32 onto the e4m3 value grid (RNE), fp32 -> fp32, in a SINGLE
    rounding step -- an fp32 -> fp16 -> 3-bit chain double-rounds when the fp16
    result lands exactly on an e4m3 tie. Caller clamps to +-448 first.

    Normal range: RNE-truncate the fp32 mantissa 23 -> 3 bits with the integer
    round-half-to-even trick (carry into the exponent rounds up correctly; it
    cannot reach the sign bit for |x| <= 448). Subnormal range (|x| < 2^-6, grid
    fixed at 2^-9): quantize via the add-magic trick -- at magnitude 2^14 the
    fp32 ulp is exactly 2^-9, so the add rounds RNE onto the grid and the
    subtract is exact."""
    b = x.to(tl.uint32, bitcast=True)
    lsb = (b >> 20) & 1
    y_norm = ((b + 524287 + lsb) & 0xFFF00000).to(tl.float32, bitcast=True)
    y_sub = (x + 24576.0) - 24576.0
    return tl.where(tl.abs(x) >= 0.015625, y_norm, y_sub)


@jit
def f32_to_e4m3(x):
    """Native-branch fp32 -> fp8e4nv cast with one RNE rounding (satfinite).

    Triton 3.8 lowers this cast on sm_89 as fp32 -RTZ-> fp16 -RNE-> e4m3, which double-rounds a
    value just above an e4m3 midpoint (triton-lang/triton#11753); emit the direct cvt there."""
    if _f32_cast_rounds_once_cx():
        return x.to(tl.float8e4nv)
    # FIXME: remove the sm_89 inline asm once the minimum Triton is 3.9 (triton-lang/triton#11924).
    # cvt packs its first source into the high byte, so the low byte is fp8(x)
    h = tl.inline_asm_elementwise(
        "cvt.rn.satfinite.e4m3x2.f32 $0, 0f00000000, $1;", "=h,r", [x], dtype=tl.uint16, is_pure=True, pack=1)
    return h.to(tl.uint8).to(tl.float8e4nv, bitcast=True)