FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""NVFP4 helpers shared by the Gluon kernels."""
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
from triton.experimental.gluon.language import _core as ttgl
from triton.experimental.gluon.language._layouts import (
DotOperandLayout,
NVMMADistributedLayout,
)
@gluon.jit
def bf16x2_mul(a, b):
# sm_90+; sm_80-89 need fma.rn.bf16x2 with a -0.0 addend
return gl.inline_asm_elementwise("mul.rn.bf16x2 $0, $1, $2;", "=r,r,r", [a, b], gl.uint32, True, 1)
@gluon.jit
def prmt(a, b, sel: gl.constexpr):
return gl.inline_asm_elementwise(f"prmt.b32 $0, $1, $2, {sel};", "=r,r,r", [a, b], gl.uint32, True, 1)
@gluon.jit
def scale_bf16x2(s):
"""e4m3 scale bytes -> bf16x2 words holding scale * 2^118 twice."""
# 2^118 * the 2^-126 of e2m1x8_bf16x2 leaves 2^-8 for the caller to undo; every step is exact in bf16
s = (s.to(gl.float8e4nv, bitcast=True).to(gl.float32) * 3.3230699894622794e35).to(gl.bfloat16)
s = s.to(gl.uint16, bitcast=True).to(gl.uint32)
return s | (s << 16)
@gluon.jit
def e2m1x8_bf16x2(q, s2):
"""8 e2m1 codes in a uint32 -> 4 bf16x2 words in k order, each value times the bf16x2 scale s2 and 2^-126."""
# each 16-bit half's top nibble lands in a bf16 as sign + exponent/mantissa bits, a value of e2m1 * 2^-126
o0 = (q & 0x80008000) | ((q >> 6) & 0x01C001C0)
q1 = q << 4
o1 = (q1 & 0x80008000) | ((q1 >> 6) & 0x01C001C0)
q2 = q << 8
o2 = (q2 & 0x80008000) | ((q2 >> 6) & 0x01C001C0)
q3 = q << 12
o3 = (q3 & 0x80008000) | ((q3 >> 6) & 0x01C001C0)
# o0..o3 hold the k pairs (3, 7), (2, 6), (1, 5), (0, 4)
w0 = bf16x2_mul(prmt(o3, o2, 0x5410), s2)
w1 = bf16x2_mul(prmt(o1, o0, 0x5410), s2)
w2 = bf16x2_mul(prmt(o3, o2, 0x7632), s2)
w3 = bf16x2_mul(prmt(o1, o0, 0x7632), s2)
return w0, w1, w2, w3
def scale_layout(op_idx: int, rows: int, kgroups: int, mma_warps: list[int]) -> gl.DistributedLinearLayout:
"""Layout of a [rows, K / 16] e4m3 scale tile for operand op_idx of an NVFP4 mma_v2; mirrors Triton's getSM120DotScaledScaleLayout."""
def bits(n):
return n.bit_length() - 1
w0, w1 = mma_warps
reg = [[0, 1 << i] for i in range(bits(kgroups))]
if op_idx == 0:
lane = [[8, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
warp = [[0, 0]] * bits(w1) + [[16 << i, 0] for i in range(bits(w0))]
cover = 16 * w0
else:
lane = [[0, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
warp = [[8 << i, 0] for i in range(bits(w1))] + [[0, 0]] * bits(w0)
cover = 8 * w1
reg += [[cover << i, 0] for i in range(max(0, bits(rows) - bits(cover)))]
# bases past the tile's rows broadcast
lane = [[b[0] if b[0] < rows else 0, b[1]] for b in lane]
warp = [[b[0] if b[0] < rows else 0, b[1]] for b in warp]
return gl.DistributedLinearLayout(reg, lane, warp, [], [rows, kgroups])
@ttgl.builtin
def mma_nvf4(a, a_scale, b, b_scale, acc, _semantic=None):
"""acc += (a * a_scale) @ (b * b_scale) on sm_120 (mma kind::mxf4nvf4, scale_vec::4X).
a [M, K/2] and b [K/2, N] are packed e2m1 in DotOperandLayout(k_width=4) of acc's mma_v2 layout; the e4m3 scales are [M, K/16] / [N, K/16] in scale_layout.
"""
assert isinstance(acc.type.layout, NVMMADistributedLayout) and acc.type.layout.version == [2, 0]
assert isinstance(a.type.layout, DotOperandLayout) and isinstance(b.type.layout, DotOperandLayout)
out = _semantic.dot_scaled(a, a_scale, "e2m1", b, b_scale, "e2m1", acc, False, True, True, ttgl.float32)
return ttgl.tensor(out.handle, acc.type)