FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(moe): prefill nvfp4 moe experts on nvfp4 activations on sm_120 (#736)
Xiaoze Fane4f69d9e2324
diff --git a/python/freetoken/kernel/gluon/nvfp4/moe/sm120_w4a4.py b/python/freetoken/kernel/gluon/nvfp4/moe/sm120_w4a4.pynew file mode 100644index 00000000..2a2c1430--- /dev/null+++ b/python/freetoken/kernel/gluon/nvfp4/moe/sm120_w4a4.py@@ -0,0 +1,536 @@+"""NVFP4 W4A4 MoE prefill GEMMs for sm_120: NVFP4 weights times NVFP4 activations on mma.sync kind::mxf4nvf4.++The activations are quantized under static global scales (the checkpoint's input_scale), so each GEMM input is e2m1 codes, per-16 e4m3 scales and one fp32 scalar.+:func:`gate_up_gemm` applies the gated activation in its epilogue and writes the down GEMM's NVFP4 input; :func:`down_gemm` scales by the routing weight and adds each route into its token's output row with bf16 atomics (the sum order, and so the result, varies from run to run).+Routing follows the W4A16 kernels (moe_align sorted ids, expert per M block, swap-AB accumulator [BN, BM]); :mod:`freetoken.kernel.nvfp4_moe` picks the constexprs.+"""++import torch+import triton+from triton.experimental import gluon+from triton.experimental.gluon import language as gl+from triton.experimental.gluon.language.nvidia.ampere import async_copy+from triton.experimental.gluon.language.nvidia.hopper import mbarrier, tma+from triton.experimental.gluon.nvidia.hopper import TensorDescriptor++from ..utils import mma_nvf4, scale_layout++# gated activations of the gate_up epilogue, as gated_act_and_mul computes them+ACTIVATIONS = {"silu": 0, "gelu_tanh": 1, "swigluoai": 2, "swiglu_clamp": 3}++SCALE_SMEM_LAYOUT = gl.constexpr(gl.SwizzledSharedLayout(1, 1, 1, [1, 0]))+++def _tile(t, num_pid_m, num_pid_n, GROUP_SIZE_M: gl.constexpr, N_GROUP: gl.constexpr):+ if N_GROUP > 0:+ # N-outer strips of N_GROUP tiles: concurrent CTAs add into a narrow output strip that stays in L2+ per_strip = num_pid_m * N_GROUP+ strip = t // per_strip+ rem = t % per_strip+ width = min(N_GROUP, num_pid_n - strip * N_GROUP)+ return rem // width, strip * N_GROUP + rem % width+ num_pid_in_group = GROUP_SIZE_M * num_pid_n+ first_pid_m = (t // num_pid_in_group) * GROUP_SIZE_M+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)+ return first_pid_m + ((t % num_pid_in_group) % group_size_m), (t % num_pid_in_group) // group_size_m+++def _scale_ptrs(ws_ptr, xs_ptr, ids_ptr, w_row, w_row_up, x_div, pid_m, num_valid, ks4, n_rows,+ BM: gl.constexpr, BN: gl.constexpr, K_TAIL: gl.constexpr, GATED: gl.constexpr, NWL: gl.constexpr):+ # scale rows are K / 16 bytes: 16-byte copies when K % 256 == 0, else 4-byte copies zeroed past the row end+ SV: gl.constexpr = 4 if K_TAIL else 16+ sll: gl.constexpr = gl.BlockedLayout([1, SV], [32 * SV // 16, 16 // SV], [NWL, 1], [1, 0])+ s_cols = gl.arange(0, 16, layout=gl.SliceLayout(0, sll))+ # both forms equal K // 16; the second tells the compiler the row is a multiple of 16 bytes+ row_len = ks4 * 4 if K_TAIL else (ks4 // 4) * 16+ r = gl.arange(0, BN, layout=gl.SliceLayout(1, sll))+ if GATED:+ sw_rows = gl.where(r < BN // 2, w_row + r, w_row_up + r - BN // 2)+ else:+ sw_rows = w_row + r+ sw_ptrs = ws_ptr + sw_rows.to(gl.int64)[:, None] * row_len + s_cols[None, :]+ sw_ok = (sw_rows < n_rows)[:, None]+ sx_tok = gl.load(ids_ptr + pid_m * BM + gl.arange(0, BM, layout=gl.SliceLayout(1, sll)))+ # the x scale rows are contiguous K / 16 bytes too; row_len keeps the divisibility a stride argument loses+ sx_ptrs = xs_ptr + ((sx_tok // x_div).to(gl.int64) * row_len)[:, None] + s_cols[None, :]+ sx_ok = (sx_tok < num_valid)[:, None]+ return sw_ptrs, sw_ok, sx_ptrs, sx_ok, s_cols, row_len+++def _load_tile(xq_ptr, xs_ptr, ids_ptr, eid_ptr, w_desc, ws_ptr, bufs, bars, pid_m, pid_n, N, nk, it, jt, num_valid,+ stride_xq, ks4, n_rows, x_div, K_TAIL: gl.constexpr, GATED: gl.constexpr):+ x_bufs, w_bufs, sw_bufs, sx_bufs = bufs+ ready, empty, s_ready, s_empty = bars+ S: gl.constexpr = x_bufs.type.shape[0]+ BM: gl.constexpr = x_bufs.type.shape[1]+ KB: gl.constexpr = x_bufs.type.shape[2]+ BN: gl.constexpr = w_bufs.type.shape[1]+ SPC: gl.constexpr = 128 // KB+ NWL: gl.constexpr = gl.num_warps()++ if GATED:+ # rows [0, BN/2) of the tile are gate rows, [BN/2, BN) the matching up rows N/2 further down+ w_row = gl.load(eid_ptr + pid_m) * N + pid_n * (BN // 2)+ w_row_up = w_row + N // 2+ else:+ w_row = gl.load(eid_ptr + pid_m) * N + pid_n * BN+ w_row_up = w_row++ # x rows: 16 bytes per cp.async, KB // 16 lanes per row+ xl: gl.constexpr = gl.BlockedLayout([1, 16], [32 * 16 // KB, KB // 16], [NWL, 1], [1, 0])+ tok = gl.load(ids_ptr + pid_m * BM + gl.arange(0, BM, layout=gl.SliceLayout(1, xl)))+ x_ptrs = xq_ptr + ((tok // x_div).to(gl.int64) * stride_xq)[:, None] + gl.arange(0, KB, layout=gl.SliceLayout(0, xl))[None, :]+ x_mask = (tok < num_valid)[:, None]+ sw_ptrs, sw_ok, sx_ptrs, sx_ok, s_cols, row_len = _scale_ptrs(+ ws_ptr, xs_ptr, ids_ptr, w_row, w_row_up, x_div, pid_m, num_valid, ks4, n_rows, BM, BN, K_TAIL, GATED, NWL)++ for kk in range(nk):+ if kk % SPC == 0:+ jl = kk // SPC+ j = jt + jl+ ss = j % 2+ mbarrier.wait(s_empty.index(ss), (j // 2 & 1) ^ 1)+ col_ok = (jl * 16 + s_cols < row_len)[None, :]+ async_copy.async_load(sw_bufs.index(ss), sw_ptrs + jl * 16, mask=sw_ok & col_ok)+ async_copy.async_load(sx_bufs.index(ss), sx_ptrs + jl * 16, mask=sx_ok & col_ok)+ async_copy.mbarrier_arrive(s_ready.index(ss))+ mbarrier.arrive(s_ready.index(ss))+ c = it + kk+ st = c % S+ mbarrier.wait(empty.index(st), (c // S & 1) ^ 1)+ # the cp.async arrive goes first: it raises the pending count before the TMA expect can complete the phase+ async_copy.async_load(x_bufs.index(st), x_ptrs + kk * KB, mask=x_mask)+ async_copy.mbarrier_arrive(ready.index(st))+ if GATED:+ mbarrier.expect(ready.index(st), 2 * w_desc.block_type.nbytes)+ tma.async_load(w_desc, [w_row, kk * KB], ready.index(st), w_bufs.index(st).slice(0, BN // 2))+ tma.async_load(w_desc, [w_row_up, kk * KB], ready.index(st), w_bufs.index(st).slice(BN // 2, BN // 2))+ else:+ mbarrier.expect(ready.index(st), w_desc.block_type.nbytes)+ tma.async_load(w_desc, [w_row, kk * KB], ready.index(st), w_bufs.index(st))+++def _load_partition(xq_ptr, xs_ptr, ids_ptr, eid_ptr, w_desc, ws_ptr, bufs, bars, N, nk, num_pid_m, num_valid,+ stride_xq, ks4, n_rows, x_div, K_TAIL: gl.constexpr, GROUP_SIZE_M: gl.constexpr,+ GATED: gl.constexpr, N_GROUP: gl.constexpr):+ BN: gl.constexpr = bufs[1].type.shape[1]+ pid_m, pid_n = _tile(gl.program_id(0), num_pid_m, gl.cdiv(N, BN), GROUP_SIZE_M, N_GROUP)+ _load_tile(xq_ptr, xs_ptr, ids_ptr, eid_ptr, w_desc, ws_ptr, bufs, bars, pid_m, pid_n, N, nk, 0, 0, num_valid,+ stride_xq, ks4, n_rows, x_div, K_TAIL, GATED)+++def _mma_partition(bufs, bars, nk, mma: gl.constexpr, wsl: gl.constexpr, xsl: gl.constexpr, K_TAIL: gl.constexpr):+ x_bufs, w_bufs, sw_bufs, sx_bufs = bufs+ ready, empty, s_ready, s_empty = bars+ S: gl.constexpr = x_bufs.type.shape[0]+ BM: gl.constexpr = x_bufs.type.shape[1]+ KB: gl.constexpr = x_bufs.type.shape[2]+ BN: gl.constexpr = w_bufs.type.shape[1]+ SB: gl.constexpr = KB // 8+ SPC: gl.constexpr = 128 // KB+ al: gl.constexpr = gl.DotOperandLayout(0, mma, 4)+ bl: gl.constexpr = gl.DotOperandLayout(1, mma, 4)+ acc = gl.zeros([BN, BM], gl.float32, mma)+ # with K % 256 != 0 the last scale block covers fewer than SPC chunks+ for jb in range(gl.cdiv(nk, SPC)):+ ss = jb % 2+ mbarrier.wait(s_ready.index(ss), jb // 2 & 1)+ for c in gl.static_range(SPC):+ kk = jb * SPC + c+ if not K_TAIL or kk < nk:+ st = kk % S+ mbarrier.wait(ready.index(st), kk // S & 1)+ a = w_bufs.index(st).load(al)+ b = x_bufs.index(st).permute((1, 0)).load(bl)+ sw = sw_bufs.index(ss).slice(c * SB, SB, dim=1).load(wsl).to(gl.float8e4nv, bitcast=True)+ sx = sx_bufs.index(ss).slice(c * SB, SB, dim=1).load(xsl).to(gl.float8e4nv, bitcast=True)+ mbarrier.arrive(empty.index(st))+ acc = mma_nvf4(a, sw, b, sx, acc)+ mbarrier.arrive(s_empty.index(ss))+ return acc+++def _e2m1x2(hi, lo):+ # cvt packs its first source into the high nibble+ return gl.inline_asm_elementwise("{ .reg .b8 t; cvt.rn.satfinite.e2m1x2.f32 t, $1, $2; cvt.u16.u8 $0, t; }",+ "=h,r,r", [hi, lo], gl.uint16, True, 1)+++def _gated_epilogue(acc, slot, pid_m, pid_n, wg_ptr, g_in, ids_ptr, q_ptr, qs_ptr, g_out, N, num_valid, stride_ge,+ ACT: gl.constexpr, act_alpha, act_limit):+ """act(gate, up) of the tile's BN / 2 columns as NVFP4 under the global g_out: q [routes, N / 4], qs [routes, N / 32]."""+ BN: gl.constexpr = acc.type.shape[0]+ BM: gl.constexpr = acc.type.shape[1]+ H2: gl.constexpr = BN // 2+ G: gl.constexpr = H2 // 16+ NW: gl.constexpr = gl.num_warps()+ out_l: gl.constexpr = acc.type.layout+ r = gl.arange(0, BN, layout=gl.SliceLayout(1, out_l))+ col = gl.where(r < H2, pid_n * H2 + r, N // 2 + pid_n * H2 + r - H2)+ wg = gl.load(wg_ptr + slot.to(gl.int64) * stride_ge + col).to(gl.float32) * g_in+ # bf16 like the W4A16 gate_up output; n-contiguous 16-byte runs, swizzled so the lanes of a warp spread over banks+ h_l: gl.constexpr = gl.SwizzledSharedLayout(8, 1, 8, [0, 1])+ h_s = gl.allocate_shared_memory(gl.bfloat16, [BN, BM], h_l, (acc * wg[:, None]).to(gl.bfloat16))+ # one thread holds 16 consecutive columns of one route+ TG: gl.constexpr = min(32, G)+ ql2: gl.constexpr = gl.BlockedLayout([16, 1], [TG, 32 // TG], [1, NW], [0, 1])+ l4: gl.constexpr = gl.BlockedLayout([1, 8, 2, 1], [TG, 1, 1, 32 // TG], [1, 1, 1, NW], [2, 1, 0, 3])+ gate = gl.convert_layout(gl.reshape(h_s.slice(0, H2).load(ql2), [G, 8, 2, BM]), l4, assert_trivial=True).to(gl.float32)+ up = gl.convert_layout(gl.reshape(h_s.slice(H2, H2).load(ql2), [G, 8, 2, BM]), l4, assert_trivial=True).to(gl.float32)+ if ACT == 1:+ inner = 0.7978845608028654 * (gate + 0.044715 * gate * gate * gate)+ # tanh(y) = 1 - 2 / (exp(2y) + 1)+ h = 0.5 * gate * (2.0 - 2.0 / (gl.exp(2.0 * inner) + 1.0)) * up+ elif ACT == 2 or ACT == 3:+ gate = gl.minimum(gate, act_limit)+ up = gl.minimum(gl.maximum(up, -act_limit), act_limit)+ if ACT == 2:+ up = up + 1.0+ h = gate / (1.0 + gl.exp(-gate * act_alpha)) * up+ else:+ h = gate / (1.0 + gl.exp(-gate)) * up+ h = h.to(gl.bfloat16).to(gl.float32)+ amax = gl.max(gl.max(gl.abs(h), axis=2), axis=1)+ sc = (amax / (6.0 * g_out)).to(gl.float8e4nv)+ sf = sc.to(gl.float32) * g_out+ inv = gl.where(sf > 0, 1.0 / sf, 0.0)+ v = h * gl.expand_dims(gl.expand_dims(inv, 1), 2)+ lo, hi = gl.split(gl.permute(v, (0, 1, 3, 2)))+ q = _e2m1x2(hi, lo).to(gl.uint8)+ ql: gl.constexpr = q.type.layout+ g_i = gl.arange(0, G, layout=gl.SliceLayout(1, gl.SliceLayout(2, ql)))+ b_i = gl.arange(0, 8, layout=gl.SliceLayout(0, gl.SliceLayout(2, ql)))+ rq = gl.load(ids_ptr + pid_m * BM + gl.arange(0, BM, layout=gl.SliceLayout(0, gl.SliceLayout(1, ql))))+ q_cols = pid_n * (H2 // 2) + g_i[:, None] * 8 + b_i[None, :]+ q_ptrs = q_ptr + (rq.to(gl.int64) * (N // 4))[None, None, :] + q_cols[:, :, None]+ gl.store(q_ptrs, q, mask=(rq < num_valid)[None, None, :])+ sl: gl.constexpr = sc.type.layout+ sg = gl.arange(0, G, layout=gl.SliceLayout(1, sl))+ rs = gl.load(ids_ptr + pid_m * BM + gl.arange(0, BM, layout=gl.SliceLayout(0, sl)))+ s_ptrs = qs_ptr + (rs.to(gl.int64) * (N // 32))[None, :] + (pid_n * G + sg)[:, None]+ gl.store(s_ptrs, sc.to(gl.uint8, bitcast=True), mask=(rs < num_valid)[None, :])+++def _w4a4_moe_kernel(+ xq_ptr, xs_ptr, g_in_ptr, w_desc, ws_ptr, wg_ptr, out_ptr, topk_weights_ptr,+ sorted_token_ids_ptr, expert_ids_ptr, num_tokens_post_padded_ptr,+ q_ptr, qs_ptr, g_out_ptr, act_alpha, act_limit,+ N, K, EM, num_valid_tokens, ks4, n_rows,+ stride_xq, stride_ge, stride_out, stride_tw,+ BLOCK_SIZE_M: gl.constexpr, BLOCK_SIZE_N: gl.constexpr, BLOCK_SIZE_K: gl.constexpr,+ GROUP_SIZE_M: gl.constexpr, N_GROUP: gl.constexpr, STAGES: gl.constexpr, MMA_WARPS_N: gl.constexpr,+ LOAD_WARPS: gl.constexpr, LOAD_REGS: gl.constexpr, K_TAIL: gl.constexpr, GATED: gl.constexpr,+ ACT: gl.constexpr, top_k: gl.constexpr, wsl: gl.constexpr, xsl: gl.constexpr,+):+ BM: gl.constexpr = BLOCK_SIZE_M+ BN: gl.constexpr = BLOCK_SIZE_N+ KB: gl.constexpr = BLOCK_SIZE_K // 2+ num_pid_m = gl.cdiv(EM, BM)+ pid_m, pid_n = _tile(gl.program_id(0), num_pid_m, gl.cdiv(N, BN), GROUP_SIZE_M, N_GROUP)+ if pid_m >= gl.cdiv(gl.load(num_tokens_post_padded_ptr), BM):+ return+ nk = K // BLOCK_SIZE_K+ # gate_up reads one x row per token, down one row per route+ x_div = top_k if GATED else 1++ x_bufs = gl.allocate_shared_memory(gl.uint8, [STAGES, BM, KB], gl.NVMMASharedLayout.get_default_for([BM, KB], gl.uint8))+ w_bufs = gl.allocate_shared_memory(gl.uint8, [STAGES, BN, KB], w_desc.layout)+ sw_bufs = gl.allocate_shared_memory(gl.uint8, [2, BN, 16], SCALE_SMEM_LAYOUT)+ sx_bufs = gl.allocate_shared_memory(gl.uint8, [2, BM, 16], SCALE_SMEM_LAYOUT)+ ready = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout())+ empty = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout())+ s_ready = gl.allocate_shared_memory(gl.int64, [2, 1], mbarrier.MBarrierLayout())+ s_empty = gl.allocate_shared_memory(gl.int64, [2, 1], mbarrier.MBarrierLayout())+ for i in gl.static_range(STAGES):+ mbarrier.init(ready.index(i), count=1)+ mbarrier.init(empty.index(i), count=1)+ for i in gl.static_range(2):+ mbarrier.init(s_ready.index(i), count=1)+ mbarrier.init(s_empty.index(i), count=1)++ NW: gl.constexpr = gl.num_warps()+ mma: gl.constexpr = gl.NVMMADistributedLayout(version=[2, 0], warps_per_cta=[MMA_WARPS_N, NW // MMA_WARPS_N],+ instr_shape=[16, 8])+ # tuples go inline: a tuple stored in a variable turns its constexprs into runtime values+ acc = gl.warp_specialize([+ (_mma_partition, ((x_bufs, w_bufs, sw_bufs, sx_bufs), (ready, empty, s_ready, s_empty), nk, mma, wsl, xsl, K_TAIL)),+ (_load_partition, (xq_ptr, xs_ptr, sorted_token_ids_ptr, expert_ids_ptr, w_desc, ws_ptr,+ (x_bufs, w_bufs, sw_bufs, sx_bufs), (ready, empty, s_ready, s_empty), N, nk, num_pid_m,+ num_valid_tokens, stride_xq, ks4, n_rows, x_div, K_TAIL, GROUP_SIZE_M, GATED, N_GROUP)),+ ], [LOAD_WARPS], [LOAD_REGS])++ out_l: gl.constexpr = gl.BlockedLayout([8, 1], [BN // 8, 32 * 8 // BN], [1, NW], [0, 1])+ acc = gl.convert_layout(acc, out_l)+ slot = gl.load(expert_ids_ptr + pid_m)+ g_in = gl.load(g_in_ptr)+ if GATED:+ _gated_epilogue(acc, slot, pid_m, pid_n, wg_ptr, g_in, sorted_token_ids_ptr, q_ptr, qs_ptr, gl.load(g_out_ptr),+ N, num_valid_tokens, stride_ge, ACT, act_alpha, act_limit)+ else:+ offs_n = pid_n * BN + gl.arange(0, BN, layout=gl.SliceLayout(1, out_l))+ n_mask = offs_n < N+ route = gl.load(sorted_token_ids_ptr + pid_m * BM + gl.arange(0, BM, layout=gl.SliceLayout(0, out_l)))+ route_mask = route < num_valid_tokens+ wg = gl.load(wg_ptr + slot.to(gl.int64) * stride_ge + offs_n, mask=n_mask, other=0.0).to(gl.float32) * g_in+ tw = gl.load(topk_weights_ptr + route * stride_tw, mask=route_mask, other=0.0)+ acc = acc * wg[:, None] * tw[None, :]+ out_ptrs = out_ptr + (route // top_k).to(gl.int64)[None, :] * stride_out + offs_n[:, None]+ gl.atomic_add(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=route_mask[None, :] & n_mask[:, None], sem="relaxed")+++# Persistent down GEMM: the 256 x 128 tile and 8 MMA warps of the kernel above, with the routing-weight scale and the atomics on+# separate epilogue warps. The accumulator is two 128 x 128 halves that go to the epilogue one at a time through one 32 KB buffer,+# so the next tile's mainloop overlaps the atomics of the last half.+++def _num_tiles(ntpp_ptr, N, BM: gl.constexpr, BN: gl.constexpr):+ num_pid_m = gl.cdiv(gl.load(ntpp_ptr), BM)+ num_pid_n = gl.cdiv(N, BN)+ return num_pid_m, num_pid_n, num_pid_m * num_pid_n+++def _down_load_partition(xq_ptr, xs_ptr, ids_ptr, eid_ptr, ntpp_ptr, w_desc, ws_ptr, bufs, bars, N, nk, num_valid,+ stride_xq, ks4, n_rows, K_TAIL: gl.constexpr, N_GROUP: gl.constexpr, GROUP_SIZE_M: gl.constexpr):+ BM: gl.constexpr = bufs[0].type.shape[1]+ KB: gl.constexpr = bufs[0].type.shape[2]+ BN: gl.constexpr = bufs[1].type.shape[1]+ SPC: gl.constexpr = 128 // KB+ num_pid_m, num_pid_n, num_tiles = _num_tiles(ntpp_ptr, N, BM, BN)+ # it / jt count chunks and scale blocks across tiles, so the rings keep their phase from one tile to the next+ it = 0+ jt = 0+ for t in range(gl.program_id(0), num_tiles, gl.num_programs(0)):+ pid_m, pid_n = _tile(t, num_pid_m, num_pid_n, GROUP_SIZE_M, N_GROUP)+ _load_tile(xq_ptr, xs_ptr, ids_ptr, eid_ptr, w_desc, ws_ptr, (bufs[0], bufs[1], bufs[2], bufs[3]),+ (bars[0], bars[1], bars[2], bars[3]), pid_m, pid_n, N, nk, it, jt, num_valid, stride_xq, ks4,+ n_rows, 1, K_TAIL, False)+ it += nk+ jt += gl.cdiv(nk, SPC)+++def _down_epi_partition(ids_ptr, ntpp_ptr, out_ptr, bufs, bars, N, num_valid, stride_out, top_k: gl.constexpr,+ N_GROUP: gl.constexpr, GROUP_SIZE_M: gl.constexpr):+ h_buf = bufs[4]+ acc_ready, acc_empty = bars[4], bars[5]+ HB: gl.constexpr = h_buf.type.shape[0]+ BM: gl.constexpr = h_buf.type.shape[1]+ BN: gl.constexpr = 2 * HB+ CH: gl.constexpr = 64+ NWE: gl.constexpr = gl.num_warps()+ # 8 consecutive n of one route per thread: one 16-byte bf16 atomic; 64 rows per pass keep the registers low+ el: gl.constexpr = gl.BlockedLayout([8, 1], [CH // 8, 32 * 8 // CH], [1, NWE], [0, 1])+ offs_cl = gl.arange(0, CH, layout=gl.SliceLayout(1, el))+ offs_tl = gl.arange(0, BM, layout=gl.SliceLayout(0, el))+ num_pid_m, num_pid_n, num_tiles = _num_tiles(ntpp_ptr, N, BM, BN)+ hc = 0+ for t in range(gl.program_id(0), num_tiles, gl.num_programs(0)):+ pid_m, pid_n = _tile(t, num_pid_m, num_pid_n, GROUP_SIZE_M, N_GROUP)+ route = gl.load(ids_ptr + pid_m * BM + offs_tl)+ rmask = (route < num_valid)[None, :]+ rows = (route // top_k).to(gl.int64)[None, :] * stride_out+ for half in gl.static_range(2):+ mbarrier.wait(acc_ready.index(0), hc & 1)+ for ch in gl.static_range(HB // CH):+ offs_n = pid_n * BN + half * HB + ch * CH + offs_cl+ h = h_buf.slice(ch * CH, CH).load(el)+ gl.atomic_add(out_ptr + rows + offs_n[:, None], h, mask=rmask & (offs_n < N)[:, None], sem="relaxed")+ mbarrier.arrive(acc_empty.index(0))+ hc += 1+++def _down_mma_partition(bufs, bars, ids_ptr, eid_ptr, ntpp_ptr, wg_ptr, g_in_ptr, tw_ptr, N, nk, num_valid, stride_ge,+ stride_tw, mma: gl.constexpr, wsl: gl.constexpr, xsl: gl.constexpr, K_TAIL: gl.constexpr,+ N_GROUP: gl.constexpr, GROUP_SIZE_M: gl.constexpr):+ x_bufs, w_bufs, sw_bufs, sx_bufs, h_buf = bufs+ ready, empty, s_ready, s_empty, acc_ready, acc_empty = bars+ S: gl.constexpr = x_bufs.type.shape[0]+ BM: gl.constexpr = x_bufs.type.shape[1]+ KB: gl.constexpr = x_bufs.type.shape[2]+ BN: gl.constexpr = w_bufs.type.shape[1]+ HB: gl.constexpr = BN // 2+ SB: gl.constexpr = KB // 8+ SPC: gl.constexpr = 128 // KB+ al: gl.constexpr = gl.DotOperandLayout(0, mma, 4)+ bl: gl.constexpr = gl.DotOperandLayout(1, mma, 4)+ offs_nl = gl.arange(0, HB, layout=gl.SliceLayout(1, mma))+ offs_tl = gl.arange(0, BM, layout=gl.SliceLayout(0, mma))+ g_in = gl.load(g_in_ptr)+ num_pid_m, num_pid_n, num_tiles = _num_tiles(ntpp_ptr, N, BM, BN)+ it = 0+ jt = 0+ hc = 0+ for t in range(gl.program_id(0), num_tiles, gl.num_programs(0)):+ pid_m, pid_n = _tile(t, num_pid_m, num_pid_n, GROUP_SIZE_M, N_GROUP)+ acc0 = gl.zeros([HB, BM], gl.float32, mma)+ acc1 = gl.zeros([HB, BM], gl.float32, mma)+ for jb in range(gl.cdiv(nk, SPC)):+ j = jt + jb+ ss = j % 2+ mbarrier.wait(s_ready.index(ss), j // 2 & 1)+ for c in gl.static_range(SPC):+ kk = jb * SPC + c+ if not K_TAIL or kk < nk:+ i = it + kk+ st = i % S+ mbarrier.wait(ready.index(st), i // S & 1)+ b = x_bufs.index(st).permute((1, 0)).load(bl)+ sx = sx_bufs.index(ss).slice(c * SB, SB, dim=1).load(xsl).to(gl.float8e4nv, bitcast=True)+ a0 = w_bufs.index(st).slice(0, HB).load(al)+ a1 = w_bufs.index(st).slice(HB, HB).load(al)+ sws = sw_bufs.index(ss).slice(c * SB, SB, dim=1)+ sw0 = sws.slice(0, HB).load(wsl).to(gl.float8e4nv, bitcast=True)+ sw1 = sws.slice(HB, HB).load(wsl).to(gl.float8e4nv, bitcast=True)+ mbarrier.arrive(empty.index(st))+ acc0 = mma_nvf4(a0, sw0, b, sx, acc0)+ acc1 = mma_nvf4(a1, sw1, b, sx, acc1)+ mbarrier.arrive(s_empty.index(ss))+ it += nk+ jt += gl.cdiv(nk, SPC)+ slot = gl.load(eid_ptr + pid_m)+ route = gl.load(ids_ptr + pid_m * BM + offs_tl)+ rmask = route < num_valid+ xs_ = gl.load(tw_ptr + route * stride_tw, mask=rmask, other=0.0) * g_in+ wrow = wg_ptr + slot.to(gl.int64) * stride_ge+ n0 = pid_n * BN + offs_nl+ wg0 = gl.load(wrow + n0, mask=n0 < N, other=0.0).to(gl.float32)+ h0 = (acc0 * wg0[:, None] * xs_[None, :]).to(gl.bfloat16)+ mbarrier.wait(acc_empty.index(0), (hc & 1) ^ 1)+ h_buf.store(h0)+ mbarrier.arrive(acc_ready.index(0))+ hc += 1+ n1 = n0 + HB+ wg1 = gl.load(wrow + n1, mask=n1 < N, other=0.0).to(gl.float32)+ h1 = (acc1 * wg1[:, None] * xs_[None, :]).to(gl.bfloat16)+ mbarrier.wait(acc_empty.index(0), (hc & 1) ^ 1)+ h_buf.store(h1)+ mbarrier.arrive(acc_ready.index(0))+ hc += 1+++def _w4a4_down_persistent_kernel(+ xq_ptr, xs_ptr, g_in_ptr, w_desc, ws_ptr, wg_ptr, out_ptr, topk_weights_ptr,+ sorted_token_ids_ptr, expert_ids_ptr, num_tokens_post_padded_ptr,+ N, K, num_valid_tokens, ks4, n_rows, stride_xq, stride_ge, stride_out, stride_tw,+ BLOCK_SIZE_M: gl.constexpr, BLOCK_SIZE_N: gl.constexpr, BLOCK_SIZE_K: gl.constexpr, STAGES: gl.constexpr,+ K_TAIL: gl.constexpr, N_GROUP: gl.constexpr, GROUP_SIZE_M: gl.constexpr, MMA_WARPS_N: gl.constexpr,+ LOAD_WARPS: gl.constexpr, LOAD_REGS: gl.constexpr, EPI_WARPS: gl.constexpr, EPI_REGS: gl.constexpr,+ top_k: gl.constexpr, wsl: gl.constexpr, xsl: gl.constexpr,+):+ BM: gl.constexpr = BLOCK_SIZE_M+ BN: gl.constexpr = BLOCK_SIZE_N+ KB: gl.constexpr = BLOCK_SIZE_K // 2+ nk = K // BLOCK_SIZE_K+ x_bufs = gl.allocate_shared_memory(gl.uint8, [STAGES, BM, KB], gl.NVMMASharedLayout.get_default_for([BM, KB], gl.uint8))+ w_bufs = gl.allocate_shared_memory(gl.uint8, [STAGES, BN, KB], w_desc.layout)+ sw_bufs = gl.allocate_shared_memory(gl.uint8, [2, BN, 16], SCALE_SMEM_LAYOUT)+ sx_bufs = gl.allocate_shared_memory(gl.uint8, [2, BM, 16], SCALE_SMEM_LAYOUT)+ # n-contiguous and swizzled, so the epilogue's 8-row reads and the MMA-layout stores spread over banks+ h_buf = gl.allocate_shared_memory(gl.bfloat16, [BN // 2, BM], gl.SwizzledSharedLayout(8, 1, 8, [0, 1]))+ ready = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout())+ empty = gl.allocate_shared_memory(gl.int64, [STAGES, 1], mbarrier.MBarrierLayout())+ s_ready = gl.allocate_shared_memory(gl.int64, [2, 1], mbarrier.MBarrierLayout())+ s_empty = gl.allocate_shared_memory(gl.int64, [2, 1], mbarrier.MBarrierLayout())+ acc_ready = gl.allocate_shared_memory(gl.int64, [1, 1], mbarrier.MBarrierLayout())+ acc_empty = gl.allocate_shared_memory(gl.int64, [1, 1], mbarrier.MBarrierLayout())+ for i in gl.static_range(STAGES):+ mbarrier.init(ready.index(i), count=1)+ mbarrier.init(empty.index(i), count=1)+ for i in gl.static_range(2):+ mbarrier.init(s_ready.index(i), count=1)+ mbarrier.init(s_empty.index(i), count=1)+ mbarrier.init(acc_ready.index(0), count=1)+ mbarrier.init(acc_empty.index(0), count=1)++ NW: gl.constexpr = gl.num_warps()+ mma: gl.constexpr = gl.NVMMADistributedLayout(version=[2, 0], warps_per_cta=[MMA_WARPS_N, NW // MMA_WARPS_N],+ instr_shape=[16, 8])+ gl.warp_specialize([+ (_down_mma_partition, ((x_bufs, w_bufs, sw_bufs, sx_bufs, h_buf), (ready, empty, s_ready, s_empty, acc_ready, acc_empty),+ sorted_token_ids_ptr, expert_ids_ptr, num_tokens_post_padded_ptr, wg_ptr, g_in_ptr,+ topk_weights_ptr, N, nk, num_valid_tokens, stride_ge, stride_tw, mma, wsl, xsl, K_TAIL,+ N_GROUP, GROUP_SIZE_M)),+ (_down_load_partition, (xq_ptr, xs_ptr, sorted_token_ids_ptr, expert_ids_ptr, num_tokens_post_padded_ptr, w_desc,+ ws_ptr, (x_bufs, w_bufs, sw_bufs, sx_bufs, h_buf),+ (ready, empty, s_ready, s_empty, acc_ready, acc_empty), N, nk, num_valid_tokens,+ stride_xq, ks4, n_rows, K_TAIL, N_GROUP, GROUP_SIZE_M)),+ (_down_epi_partition, (sorted_token_ids_ptr, num_tokens_post_padded_ptr, out_ptr,+ (x_bufs, w_bufs, sw_bufs, sx_bufs, h_buf), (ready, empty, s_ready, s_empty, acc_ready, acc_empty),+ N, num_valid_tokens, stride_out, top_k, N_GROUP, GROUP_SIZE_M)),+ ], [LOAD_WARPS, EPI_WARPS], [LOAD_REGS, EPI_REGS])+++def supports(H: int, I: int) -> bool:+ """Whether the GEMMs take a layer of this hidden / intermediate size."""+ # K is a whole number of 64-wide k blocks for both GEMMs; the gated tile pairs 64 gate rows with 64 up rows at least+ return H % 64 == 0 and I % 64 == 0+++def _weight_desc(packed, box_n, BK):+ S, N, KB = packed.shape+ layout = gl.NVMMASharedLayout.get_default_for([box_n, BK // 2], gl.uint8)+ return TensorDescriptor.from_tensor(packed.view(S * N, KB), [box_n, BK // 2], layout)+++def _scale_layouts(cfg, w_rows):+ nw, wn = cfg["num_warps"], cfg["MMA_WARPS_N"]+ kg = cfg["BLOCK_SIZE_K"] // 16+ return scale_layout(0, w_rows, kg, [wn, nw // wn]), scale_layout(1, cfg["BLOCK_SIZE_M"], kg, [wn, nw // wn])+++def gate_up_gemm(xq, xs, g_in, packed, scale, glob, q, qs, g_out, sorted_ids, expert_ids, num_tokens_post_padded,+ num_valid_tokens, top_k, activation, act_alpha, act_limit, cfg):+ """q, qs = NVFP4 under g_out of act(gate, up), where [gate; up] = (xq * xs * g_in)[route // top_k] W4A4 packed[expert]^T * glob."""+ S, N, KB = packed.shape+ K = KB * 2+ BM, BN, BK = cfg["BLOCK_SIZE_M"], cfg["BLOCK_SIZE_N"], cfg["BLOCK_SIZE_K"]+ assert K % BK == 0 and (N // 2) % (BN // 2) == 0 and xs.stride(0) == K // 16+ assert q.stride(0) == N // 4 and qs.stride(0) == N // 32+ wsl, xsl = _scale_layouts(cfg, BN)+ grid = triton.cdiv(sorted_ids.shape[0], BM) * triton.cdiv(N, BN)+ _w4a4_moe_kernel[(grid,)](+ xq, xs.view(torch.uint8), g_in, _weight_desc(packed, BN // 2, BK), scale.view(torch.uint8), glob, None, None,+ sorted_ids, expert_ids, num_tokens_post_padded, q, qs.view(torch.uint8), g_out, act_alpha, act_limit,+ N, K, sorted_ids.shape[0], num_valid_tokens, K // 64, S * N, xq.stride(0), glob.stride(0), 0, 0,+ K_TAIL=K % 256 != 0, GATED=True, ACT=ACTIVATIONS[activation], top_k=top_k, wsl=wsl, xsl=xsl, **cfg,+ )+++def down_gemm(xq, xs, g_in, packed, scale, glob, out, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded,+ num_valid_tokens, top_k, cfg):+ """out[route // top_k] += (xq * xs * g_in)[route] W4A4 packed[expert]^T * glob * topk_weights_flat[route]; out starts zeroed."""+ S, N, KB = packed.shape+ K = KB * 2+ cfg = dict(cfg)+ persistent = cfg.pop("PERSISTENT")+ BM, BN, BK = cfg["BLOCK_SIZE_M"], cfg["BLOCK_SIZE_N"], cfg["BLOCK_SIZE_K"]+ assert K % BK == 0 and xs.stride(0) == K // 16+ args = (xq, xs.view(torch.uint8), g_in, _weight_desc(packed, BN, BK), scale.view(torch.uint8), glob, out,+ topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded)+ if persistent:+ wsl, xsl = _scale_layouts(cfg, BN // 2)+ grid = torch.cuda.get_device_properties(out.device).multi_processor_count+ _w4a4_down_persistent_kernel[(grid,)](+ *args, N, K, num_valid_tokens, K // 64, S * N, xq.stride(0), glob.stride(0), out.stride(0),+ topk_weights_flat.stride(0), K_TAIL=K % 256 != 0, top_k=top_k, wsl=wsl, xsl=xsl, **cfg,+ )+ return+ wsl, xsl = _scale_layouts(cfg, BN)+ grid = triton.cdiv(sorted_ids.shape[0], BM) * triton.cdiv(N, BN)+ _w4a4_moe_kernel[(grid,)](+ *args, None, None, None, 1.0, 1.0,+ N, K, sorted_ids.shape[0], num_valid_tokens, K // 64, S * N, xq.stride(0), glob.stride(0), out.stride(0),+ topk_weights_flat.stride(0), K_TAIL=K % 256 != 0, GATED=False, ACT=0, top_k=top_k, wsl=wsl, xsl=xsl, **cfg,+ )diff --git a/python/freetoken/kernel/gluon/nvfp4/utils.py b/python/freetoken/kernel/gluon/nvfp4/utils.pyindex 430c146d..11426ec5 100644--- a/python/freetoken/kernel/gluon/nvfp4/utils.py+++ b/python/freetoken/kernel/gluon/nvfp4/utils.py@@ -2,6 +2,11 @@from triton.experimental import gluonfrom triton.experimental.gluon import language as gl+from triton.experimental.gluon.language import _core as ttgl+from triton.experimental.gluon.language._layouts import (+ DotOperandLayout,+ NVMMADistributedLayout,+)@gluon.jit@@ -41,3 +46,37 @@ def e2m1x8_bf16x2(q, s2):w2 = bf16x2_mul(prmt(o3, o2, 0x7632), s2)w3 = bf16x2_mul(prmt(o1, o0, 0x7632), s2)return w0, w1, w2, w3+++def scale_layout(op_idx: int, rows: int, kgroups: int, mma_warps: list[int]) -> gl.DistributedLinearLayout:+ """Layout of a [rows, K / 16] e4m3 scale tile for operand op_idx of an NVFP4 mma_v2; mirrors Triton's getSM120DotScaledScaleLayout."""+ def bits(n):+ return n.bit_length() - 1++ w0, w1 = mma_warps+ reg = [[0, 1 << i] for i in range(bits(kgroups))]+ if op_idx == 0:+ lane = [[8, 0], [0, 0], [1, 0], [2, 0], [4, 0]]+ warp = [[0, 0]] * bits(w1) + [[16 << i, 0] for i in range(bits(w0))]+ cover = 16 * w0+ else:+ lane = [[0, 0], [0, 0], [1, 0], [2, 0], [4, 0]]+ warp = [[8 << i, 0] for i in range(bits(w1))] + [[0, 0]] * bits(w0)+ cover = 8 * w1+ reg += [[cover << i, 0] for i in range(max(0, bits(rows) - bits(cover)))]+ # bases past the tile's rows broadcast+ lane = [[b[0] if b[0] < rows else 0, b[1]] for b in lane]+ warp = [[b[0] if b[0] < rows else 0, b[1]] for b in warp]+ return gl.DistributedLinearLayout(reg, lane, warp, [], [rows, kgroups])+++def mma_nvf4(a, a_scale, b, b_scale, acc, _semantic=None):+ """acc += (a * a_scale) @ (b * b_scale) on sm_120 (mma kind::mxf4nvf4, scale_vec::4X).++ a [M, K/2] and b [K/2, N] are packed e2m1 in DotOperandLayout(k_width=4) of acc's mma_v2 layout; the e4m3 scales are [M, K/16] / [N, K/16] in scale_layout.+ """+ assert isinstance(acc.type.layout, NVMMADistributedLayout) and acc.type.layout.version == [2, 0]+ assert isinstance(a.type.layout, DotOperandLayout) and isinstance(b.type.layout, DotOperandLayout)+ out = _semantic.dot_scaled(a, a_scale, "e2m1", b, b_scale, "e2m1", acc, False, True, True, ttgl.float32)+ return ttgl.tensor(out.handle, acc.type)diff --git a/python/freetoken/kernel/nvfp4_moe.py b/python/freetoken/kernel/nvfp4_moe.pyindex 0aa49422..df894107 100644--- a/python/freetoken/kernel/nvfp4_moe.py+++ b/python/freetoken/kernel/nvfp4_moe.py@@ -1,6 +1,6 @@-"""Kernel dispatch for the inline-dequant NVFP4 MoE GEMMs: which kernel runs, with which tiles and switches, and its launch.+"""Kernel dispatch for the NVFP4 MoE GEMMs: which kernel runs, with which tiles and switches, and its launch.-The kernels are templates (:mod:`freetoken.kernel.triton.nvfp4_fused_moe`, :mod:`freetoken.kernel.gluon.nvfp4.moe.sm120`);+The kernels are templates (:mod:`freetoken.kernel.triton.nvfp4_fused_moe`, :mod:`freetoken.kernel.gluon.nvfp4.moe.sm120`, :mod:`freetoken.kernel.gluon.nvfp4.moe.sm120_w4a4`);:mod:`freetoken.moe.fused_nvfp4` strings the GEMMs into the MoE layer."""@@ -388,3 +388,73 @@ def prefill_gemm(SCALE_WORDS=scale_words,**cfg,)++[email protected]_cache(maxsize=None)+def _w4a4_prefill(device_index: int):+ """The Gluon module of the W4A4 prefill GEMMs on this GPU, or None."""+ from freetoken.kernel import gluon++ if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available():+ return None+ from freetoken.kernel.gluon.nvfp4.moe import sm120_w4a4++ return sm120_w4a4+++def w4a4_available(device_index: int) -> bool:+ return _w4a4_prefill(device_index) is not None+++# Below this many tokens per expert both prefills stream the weights at the same speed (RTX PRO 6000 and RTX 5090 sweeps over the+# NVFP4 MoE shapes); W4A16 keeps the exact activations there+_W4A4_MIN_TOKENS_PER_EXPERT = 4+# (min rho, tokens per expert where BLOCK_SIZE_M becomes 128), from the same sweeps+_W4A4_BLOCK_M_STEPS = ((3.0, 128), (0.0, 64))++[email protected]_cache(maxsize=None)+def _w4a4_device(device_index: int) -> Tuple[float, int]:+ props = torch.cuda.get_device_properties(device_index)+ rho = _tensor_rho(props.name, props.major, props.minor)+ return next(t for min_rho, t in _W4A4_BLOCK_M_STEPS if rho >= min_rho), props.L2_cache_size+++def w4a4_prefill_config(M: int, top_k: int, num_experts: int, H: int, I: int, activation: str, dtype: torch.dtype,+ device_index: int) -> Dict[str, Any] | None:+ """Tiles of the W4A4 prefill GEMMs, or None where the W4A16 prefill runs instead."""+ w4a4 = _w4a4_prefill(device_index)+ if w4a4 is None or dtype != torch.bfloat16 or activation not in w4a4.ACTIVATIONS or not w4a4.supports(H, I):+ return None+ to128, l2_bytes = _w4a4_device(device_index)+ per_expert = M * top_k / num_experts+ if per_expert < _W4A4_MIN_TOKENS_PER_EXPERT:+ return None+ block_m = 128 if per_expert >= to128 else 64+ common = dict(BLOCK_SIZE_M=block_m, GROUP_SIZE_M=8, MMA_WARPS_N=4, LOAD_WARPS=4, num_warps=8)+ # the gated tile holds BN / 2 gate rows and their up rows, so I must split into whole half tiles+ gate_up = dict(common, BLOCK_SIZE_N=256 if I % 128 == 0 else 128, BLOCK_SIZE_K=128 if H % 128 == 0 else 64,+ N_GROUP=0, STAGES=3, LOAD_REGS=88)+ if I < 1024:+ # short K: the persistent kernel overlaps a tile's atomics with the next mainloop; N strips keep the [M, H] output+ # in L2 once it outgrows half of it+ down = dict(common, BLOCK_SIZE_N=256, BLOCK_SIZE_K=64, STAGES=4, N_GROUP=2 if M * H * 2 > l2_bytes // 2 else 0,+ LOAD_REGS=56, EPI_WARPS=4, EPI_REGS=72, PERSISTENT=True)+ else:+ down = dict(common, BLOCK_SIZE_N=256 if H % 256 == 0 else 128, BLOCK_SIZE_K=128 if I % 128 == 0 else 64,+ N_GROUP=0, STAGES=3, LOAD_REGS=88, PERSISTENT=False)+ return dict(BLOCK_SIZE_M=block_m, gate_up=gate_up, down=down)+++def w4a4_gate_up_gemm(xq, xs, g_in, packed, scale, glob, q, qs, g_out, sorted_ids, expert_ids, num_tokens_post_padded,+ num_valid_tokens, top_k, activation, act_alpha, act_limit, cfg):+ _w4a4_prefill(xq.device.index).gate_up_gemm(+ xq, xs, g_in, packed, scale, glob, q, qs, g_out, sorted_ids, expert_ids, num_tokens_post_padded,+ num_valid_tokens, top_k, activation, act_alpha, act_limit, cfg)+++def w4a4_down_gemm(xq, xs, g_in, packed, scale, glob, out, topk_weights_flat, sorted_ids, expert_ids,+ num_tokens_post_padded, num_valid_tokens, top_k, cfg):+ _w4a4_prefill(xq.device.index).down_gemm(+ xq, xs, g_in, packed, scale, glob, out, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded,+ num_valid_tokens, top_k, cfg)diff --git a/python/freetoken/kernel/triton/nvfp4_act_quant.py b/python/freetoken/kernel/triton/nvfp4_act_quant.pynew file mode 100644index 00000000..b52cfbe8--- /dev/null+++ b/python/freetoken/kernel/triton/nvfp4_act_quant.py@@ -0,0 +1,42 @@+"""NVFP4 activation quantization under a static global scale: x ~= e2m1 * e4m3 block scale (per 16) * g."""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _e2m1x2(hi, lo):+ # sm_100+ / sm_120a; cvt packs its first source into the high nibble+ return tl.inline_asm_elementwise(+ "{ .reg .b8 t; cvt.rn.satfinite.e2m1x2.f32 t, $1, $2; cvt.u16.u8 $0, t; }",+ "=h,r,r", [hi, lo], dtype=tl.uint16, is_pure=True, pack=1)+++def _quant_kernel(x_ptr, q_ptr, s_ptr, g_ptr, K, stride_x, BLOCK_K: tl.constexpr):+ row = tl.program_id(0).to(tl.int64)+ offs = tl.arange(0, BLOCK_K)+ x = tl.load(x_ptr + row * stride_x + offs, mask=offs < K, other=0.0).to(tl.float32)+ g = tl.load(g_ptr)+ xb = tl.reshape(x, [BLOCK_K // 16, 16])+ s = (tl.max(tl.abs(xb), 1) / (6.0 * g)).to(tl.float8e4nv)+ sf = s.to(tl.float32) * g+ inv = tl.where(sf > 0, 1.0 / sf, 0.0)+ lo, hi = tl.split(tl.reshape(xb * inv[:, None], [BLOCK_K // 2, 2]))+ oq = tl.arange(0, BLOCK_K // 2)+ tl.store(q_ptr + row * (K // 2) + oq, _e2m1x2(hi, lo).to(tl.uint8), mask=oq < K // 2)+ os_ = tl.arange(0, BLOCK_K // 16)+ tl.store(s_ptr + row * (K // 16) + os_, s.to(tl.uint8, bitcast=True), mask=os_ < K // 16)+++def quantize_nvfp4(x: torch.Tensor, g: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ """x [R, K] -> (codes [R, K / 2] uint8, low nibble first; block scales [R, K / 16] e4m3) under the fp32 global g [1]; K % 16 == 0."""+ R, K = x.shape+ q = torch.empty(R, K // 2, dtype=torch.uint8, device=x.device)+ s = torch.empty(R, K // 16, dtype=torch.uint8, device=x.device)+ if R:+ _quant_kernel[(R,)](x, q, s, g, K, x.stride(0), BLOCK_K=triton.next_power_of_2(K), num_warps=4)+ return q, s.view(torch.float8_e4m3fn)diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.pyindex 1cd1a1ac..0dce3995 100644--- a/python/freetoken/layers/quantization/moe/nvfp4.py+++ b/python/freetoken/layers/quantization/moe/nvfp4.py@@ -78,6 +78,56 @@ class TritonNvfp4MoEKernel(MoEKernel):return fused_experts_decode_nvfp4_marlin(x, *banks, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)+class W4a4Nvfp4MoEKernel(TritonNvfp4MoEKernel):+ """The Triton kernels' banks and decode, with the prefill GEMMs on NVFP4 activations (W4A4) where the GPU and the batch favour it.++ Needs a checkpoint whose experts declare static activation scales; the GPU-resident alphas carry them per expert (gate_up_alpha: the larger of gate's and up's input_scale).+ """++ name = "w4a4"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ reason = super().unusable_reason(cfg)+ if reason:+ return reason+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):+ return "the experts declare no static activation scale (W4A16 checkpoint)"+ if cfg.apply_router_weight_on_input:+ return "the routing weight scales the input, which is quantized once per token"+ return None++ def worth_it(self, cfg: MoEConfig) -> bool:+ from freetoken.kernel import nvfp4_moe++ return not backend.is_rocm() and torch.cuda.is_available() and nvfp4_moe.w4a4_available(torch.cuda.current_device())++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ return {+ **super().layout(cfg),+ "gate_up_alpha": BankSpec((), torch.float32, resident=True),+ "down_alpha": BankSpec((), torch.float32, resident=True),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ super().pack(pieces, cfg, out)+ if not all(f"{p}_input" in pieces for p in ("gate", "up", "down")):+ raise ValueError("w4a4 nvfp4 MoE kernel: the checkpoint has no expert input_scale tensors")+ gate, up, down = (pieces[f"{p}_input"].reshape(len(pieces[f"{p}_input"]), -1)[:, 0].float() for p in ("gate", "up", "down"))+ return {"gate_up_alpha": torch.maximum(gate, up), "down_alpha": down}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ if not is_prefill:+ return super().apply(layer, x, topk_weights, topk_ids, view, is_prefill=False)+ from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4_w4a4++ t = view.tensors+ assert view.alphas is not None+ return fused_experts_nvfp4_w4a4(+ x, t["gate_up"], t["gate_up_scale"], t["gate_up_global"], t["down"], t["down_scale"], t["down_global"],+ view.alphas[0], view.alphas[1], topk_weights, topk_ids, view.n, layer.activation, float(layer.alpha), limit_or_inf(layer),+ )++# ---------------------------------------------------------------------------# Marlin: vLLM's fused Marlin MoE (W4A16, dequant in kernel) over Marlin-tiled banks.# The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the@@ -570,4 +620,4 @@ class B12xNvfp4MoEKernel(MoEKernel):@register_method(QuantKind.NVFP4, LayerKind.MOE)class Nvfp4MoEMethod(MoEMethod):- candidates = (TritonNvfp4MoEKernel, MarlinNvfp4MoEKernel, B12xNvfp4MoEKernel)+ candidates = (W4a4Nvfp4MoEKernel, TritonNvfp4MoEKernel, MarlinNvfp4MoEKernel, B12xNvfp4MoEKernel)diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.pyindex 04da0509..e30429c0 100644--- a/python/freetoken/models/nvfp4_banks.py+++ b/python/freetoken/models/nvfp4_banks.py@@ -39,6 +39,28 @@ def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torchreturn tensor.to(torch.float16)+def _activation_scale():+ """The checkpoint tensor of the NVFP4 static activation scale under the installed dialect, or None."""+ from freetoken.layers.quantization import QuantKind, get_quant_config++ try:+ quant = get_quant_config()+ except RuntimeError:+ return None+ if quant is None or QuantKind.NVFP4 not in quant.STORAGE:+ return None+ return quant.stored_tensors(QuantKind.NVFP4).get("input_scale")+++def _ingest(spec: "Nvfp4ExpertSourceSpec", act, role: str, tensor: torch.Tensor) -> torch.Tensor:+ if role.endswith("_global"):+ return _ingest_global(spec, tensor)+ if role.endswith("_input"):+ tensor = tensor.float()+ return 1.0 / tensor if act.reciprocal else tensor+ return tensor++def _num_moe_layers(config) -> int:value = getattr(config, "num_moe_layers", None)if value is not None:@@ -77,7 +99,8 @@ def iter_nvfp4_expert_pieces():"""One piece per routed expert: ``gate`` / ``up`` / ``down`` codes plus their ``_scale``(fp8 block scales) and ``_global`` (the per-tensor scale, reciprocal for quant-side dialects,- fp16) companions, straight from the safetensors shards.+ fp16) companions, straight from the safetensors shards. When every expert also stores the+ dialect's static activation scale, the pieces carry it as fp32 ``_input`` (dequant side).Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Eitherway tensors of one expert may span shards, so they are grouped by (layer, expert) as they land.@@ -92,6 +115,7 @@ def iter_nvfp4_expert_pieces(weight_map = safetensors_weight_map(folder)keep = None if layers is None else set(layers)+ act = _activation_scale()wanted: dict[str, tuple[int, int, str]] = {}for name in weight_map:match = spec.key_pattern.match(name)@@ -107,9 +131,20 @@ def iter_nvfp4_expert_pieces(if kind not in ("weight", "weight_scale", "weight_scale_2"):raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")wanted[name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + _kind_suffix(kind))- expected = (_num_moe_layers(config) if keep is None else len(keep)) * config.num_experts * 9- if len(wanted) != expected:- raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {expected}")+ if kind == "weight" and act is not None:+ act_name = name[: match.start("kind")] + act.name+ if act_name in weight_map:+ wanted[act_name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + "_input")+ experts = (_num_moe_layers(config) if keep is None else len(keep)) * config.num_experts+ acts = [name for name, (_, _, role) in wanted.items() if role.endswith("_input")]+ # a partial set serves no W4A4 kernel; the pieces then carry none+ if len(acts) != experts * 3:+ for name in acts:+ del wanted[name]+ acts = []+ per_expert = 12 if acts else 9+ if len(wanted) != experts * per_expert:+ raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {experts * per_expert}")def _serial():by_shard: dict[str, list[str]] = collections.defaultdict(list)@@ -121,21 +156,16 @@ def iter_nvfp4_expert_pieces(drop(path)with safetensors.safe_open(path, framework="pt", device="cpu") as f:for name in by_shard[shard]:- tensor = f.get_tensor(name)- if wanted[name][2].endswith("_global"):- tensor = _ingest_global(spec, tensor)- yield name, tensor+ yield name, _ingest(spec, act, wanted[name][2], f.get_tensor(name))drop(path)def _parallel():from freetoken.models.weight import iter_expert_tensors_parallelfor name, tensor in iter_expert_tensors_parallel(folder, lambda n: n in wanted, workers=workers, chunk=chunk):- if wanted[name][2].endswith("_global"):- tensor = _ingest_global(spec, tensor)- yield name, tensor+ yield name, _ingest(spec, act, wanted[name][2], tensor)- return per_expert_pieces(_parallel() if parallel else _serial(), wanted.get, tensors_per_expert=9)+ return per_expert_pieces(_parallel() if parallel else _serial(), wanted.get, tensors_per_expert=per_expert)__all__ = ["Nvfp4ExpertSourceSpec", "iter_nvfp4_expert_pieces"]diff --git a/python/freetoken/moe/expert_pieces.py b/python/freetoken/moe/expert_pieces.pyindex 917d84bb..74c4a41b 100644--- a/python/freetoken/moe/expert_pieces.py+++ b/python/freetoken/moe/expert_pieces.py@@ -151,7 +151,7 @@ def per_expert_pieces("""Group per-expert tensors into one piece per expert, in whatever order they arrive.``locate(name)`` -> ``(bank_layer, expert, role)`` or None to skip; a role tensor is one- expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals may be scalars.+ expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals and activation scales may be scalars."""pending: dict[tuple[int, int], dict[str, torch.Tensor]] = {}for name, tensor in tensors:@@ -160,7 +160,7 @@ def per_expert_pieces(continuebank_layer, expert, role = wherepiece = pending.setdefault((bank_layer, expert), {})- piece[role] = tensor.reshape(1, -1) if role.endswith("_global") else tensor.unsqueeze(0)+ piece[role] = tensor.reshape(1, -1) if role.endswith(("_global", "_input")) else tensor.unsqueeze(0)if len(piece) == tensors_per_expert:del pending[(bank_layer, expert)]yield bank_layer, expert, expert + 1, piecediff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.pyindex 197ef547..6bedd57b 100644--- a/python/freetoken/moe/fused_nvfp4.py+++ b/python/freetoken/moe/fused_nvfp4.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport torchfrom freetoken.kernel import moe_sum_reduce_triton, nvfp4_moe+from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4from freetoken.layers import gated_act_and_mulfrom freetoken.moe.fused import moe_align_block_size@@ -156,8 +157,60 @@ def fused_experts_nvfp4(return out+def fused_experts_nvfp4_w4a4(+ hidden_states: torch.Tensor,+ gate_up_packed: torch.Tensor,+ gate_up_scale: torch.Tensor,+ gate_up_global: torch.Tensor,+ down_packed: torch.Tensor,+ down_scale: torch.Tensor,+ down_global: torch.Tensor,+ gate_up_input_scale: torch.Tensor,+ down_input_scale: torch.Tensor,+ topk_weights: torch.Tensor,+ topk_ids: torch.Tensor,+ num_experts: int,+ activation: str = "silu",+ act_alpha: float = 1.702,+ act_limit: float = 7.0,+) -> torch.Tensor:+ """Prefill NVFP4 MoE with NVFP4 activations (W4A4) where the GPU and the batch favour it, else :func:`fused_experts_nvfp4`.++ ``*_input_scale`` are the per-expert static activation scales of the ``num_experts`` banks; each GEMM input is quantized under their max, as vLLM does.+ """+ M, H = hidden_states.shape+ top_k = topk_ids.shape[1]+ inter = gate_up_packed.shape[1] // 2+ cfg = nvfp4_moe.w4a4_prefill_config(M, top_k, num_experts, H, inter, activation, hidden_states.dtype, hidden_states.device.index)+ if cfg is None:+ return fused_experts_nvfp4(+ hidden_states, gate_up_packed, gate_up_scale, gate_up_global, down_packed, down_scale, down_global,+ topk_weights, topk_ids, num_experts, activation, False, act_alpha, act_limit,+ )+ dev = hidden_states.device+ g = torch.stack((gate_up_input_scale[:num_experts], down_input_scale[:num_experts])).float().amax(dim=1)+ sorted_ids, expert_ids, ntpp = moe_align_block_size(topk_ids, cfg["BLOCK_SIZE_M"], num_experts)+ tw = topk_weights.reshape(-1).contiguous()+ num_valid = topk_ids.numel()++ xq, xs = quantize_nvfp4(hidden_states, g[0:1])+ q = torch.empty((num_valid, inter // 2), device=dev, dtype=torch.uint8)+ qs = torch.empty((num_valid, inter // 16), device=dev, dtype=torch.float8_e4m3fn)+ nvfp4_moe.w4a4_gate_up_gemm(+ xq, xs, g[0:1], gate_up_packed, gate_up_scale, gate_up_global, q, qs, g[1:2], sorted_ids, expert_ids, ntpp,+ num_valid, top_k, activation, act_alpha, act_limit, cfg["gate_up"],+ )+ out = torch.zeros_like(hidden_states)+ nvfp4_moe.w4a4_down_gemm(+ q, qs, g[1:2], down_packed, down_scale, down_global, out, tw, sorted_ids, expert_ids, ntpp, num_valid, top_k,+ cfg["down"],+ )+ return out++__all__ = ["fused_experts_decode_nvfp4_marlin","fused_experts_decode_nvfp4_serial","fused_experts_nvfp4",+ "fused_experts_nvfp4_w4a4",]diff --git a/python/freetoken/moe/legacy_format.py b/python/freetoken/moe/legacy_format.pyindex 973a3dba..8f347e5c 100644--- a/python/freetoken/moe/legacy_format.py+++ b/python/freetoken/moe/legacy_format.py@@ -30,12 +30,16 @@ LEGACY_FORMAT = {(QuantKind.NONE, "fused"): "bf16",(QuantKind.FP8_BLOCK, "triton"): "fp8_block",(QuantKind.NVFP4, "triton"): "nvfp4",+ # same banks as triton, plus the resident input scales; a "nvfp4" file reads back as triton+ (QuantKind.NVFP4, "w4a4"): "nvfp4",(QuantKind.NVFP4, "marlin"): "nvfp4_marlin",(QuantKind.NVFP4, "b12x"): "nvfp4_b12x",(QuantKind.MXFP4, "triton_gptoss"): "mxfp4_triton",(QuantKind.MXFP4, "triton"): "ds_fp4",}-_KIND_KERNEL = {fmt: kk for kk, fmt in LEGACY_FORMAT.items()}+_KIND_KERNEL: dict[str, tuple[QuantKind, str]] = {}+for _kk, _fmt in LEGACY_FORMAT.items():+ _KIND_KERNEL.setdefault(_fmt, _kk)def legacy_format_for(kind: QuantKind, kernel: str) -> str:diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.pyindex 29c042ba..fd98c2ee 100644--- a/tests/moe/test_nvfp4_backends.py+++ b/tests/moe/test_nvfp4_backends.py@@ -10,13 +10,15 @@ Coverage by hardware:baseline-kernel equality guard. This is the path used on sm_120 + CUDA 12.x.- Marlin (sm_80..sm_99, e.g. H100): prefill + decode + overlap.- b12x (sm_120 + CUDA>=13): pack + fused decode, skipped where the kernel cannot run.- - Gluon (sm_120): the prefill GEMMs it takes over from the Triton kernel, against that kernel.+ - Gluon (sm_120): the prefill GEMMs it takes over from the Triton kernel, against that kernel, and the W4A4 prefill+ against a torch emulation of its NVFP4 activation rounding.Banks are built the way the loader builds them: the bound kernel object packs native pieces."""from __future__ import annotationsimport importlib.util+import mathimport pytestimport torch@@ -413,6 +415,100 @@ def test_triton_prefill_routes_to_gluon_on_sm120(monkeypatch, hidden_size, intertorch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))+def _round_e2m1(v: torch.Tensor) -> torch.Tensor:+ """Round to the nearest e2m1 value, ties to the even code, saturating at 6 (cvt.rn.satfinite)."""+ grid = _E2M1[:8].to(v.device)+ a = v.abs().clamp(max=6.0)+ hi = torch.searchsorted(grid, a.contiguous()).clamp(max=7)+ lo = (hi - 1).clamp(min=0)+ d_lo, d_hi = a - grid[lo], grid[hi] - a+ code = torch.where(d_lo < d_hi, lo, torch.where(d_hi < d_lo, hi, torch.where(lo % 2 == 0, lo, hi)))+ return torch.copysign(grid[code], v)+++def _nvfp4_qdq(x: torch.Tensor, g: float) -> torch.Tensor:+ """x [R, K] through NVFP4 under the static global g, back to fp32."""+ xb = x.float().view(x.shape[0], -1, 16)+ # fp32 scalar math like the kernels: a one-ulp difference in 6 * g can flip an e4m3 block scale+ g = torch.tensor(g, dtype=torch.float32, device=x.device)+ sf = ((xb.abs().amax(-1) / (6.0 * g)).to(torch.float8_e4m3fn).float() * g).unsqueeze(-1)+ inv = torch.where(sf > 0, 1.0 / sf, torch.zeros_like(sf))+ return (_round_e2m1(xb * inv) * sf).view_as(x)+++def _gated_ref(h: torch.Tensor, activation: str, inter: int, alpha: float, limit: float) -> torch.Tensor:+ gate, up = h[:, :inter], h[:, inter:]+ if activation == "gelu_tanh":+ return torch.nn.functional.gelu(gate, approximate="tanh") * up+ if activation in ("swiglu_clamp", "swigluoai"):+ gate = gate.clamp(max=limit)+ return gate * torch.sigmoid(alpha * gate) * (up.clamp(-limit, limit) + (activation == "swigluoai"))+ return torch.nn.functional.silu(gate) * up+++@cuda[email protected](not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 0), reason="needs sm_120")[email protected]("hidden_size,inter,activation,tokens", [+ (256, 128, "silu", 256), (320, 192, "gelu_tanh", 1024), (512, 1024, "swiglu_clamp", 256), (256, 128, "swigluoai", 256),+])+def test_w4a4_prefill_matches_nvfp4_activation_reference(hidden_size, inter, activation, tokens):+ """The w4a4 kernel from checkpoint pieces to the prefill output, against a torch emulation of the same NVFP4 activation rounding.++ Covers 256- / 128-row gated tiles, K % 256 != 0 in both GEMMs, the persistent down GEMM (I < 1024) and the other one, BLOCK_SIZE_M 64 and 128.+ """+ from types import SimpleNamespace++ from freetoken.kernel import nvfp4_moe+ from freetoken.layers.quantization.moe.base import ExpertView, MoEConfig+ from freetoken.layers.quantization.moe.nvfp4 import W4a4Nvfp4MoEKernel++ device = torch.device("cuda")+ if not nvfp4_moe.w4a4_available(device.index or 0):+ pytest.skip("the W4A4 Gluon kernels need Triton's Gluon on sm_120")+ alpha, limit = (1.5, 2.0) if activation in ("swiglu_clamp", "swigluoai") else (1.0, None)+ g = torch.Generator(device=device).manual_seed(0)++ def bank(n, k):+ packed = torch.randint(0, 256, (E, n, k // 2), dtype=torch.uint8, device=device, generator=g)+ scale = (torch.rand(E, n, k // 16, device=device, generator=g) + 0.25).to(torch.float8_e4m3fn)+ return packed, scale, (torch.rand(E, n, device=device, generator=g) * 0.02 + 0.01).to(torch.float16)++ gu, gus, gug = bank(2 * inter, hidden_size)+ dn, dns, dng = bank(hidden_size, inter)+ hidden = torch.randn(tokens, hidden_size, dtype=torch.bfloat16, device=device, generator=g)+ topk_ids = torch.randint(0, E, (tokens, TOPK), dtype=torch.int32, device=device, generator=g)+ topk_weights = torch.rand(tokens, TOPK, device=device, generator=g)++ # static scales near a calibration's amax / (6 * 448), off by an irrational factor: with amax / 2688 itself many values sit exactly on an+ # e2m1 midpoint, where the kernel's approximate fp32 reciprocal and torch's exact one round apart; per expert, below the max for some experts+ g1 = float(hidden.float().abs().max()) / 2688 * math.pi / 3+ xq = _nvfp4_qdq(hidden, g1)+ w_gu = torch.stack([_dequant_ref(gu[e], gus[e], gug[e]) for e in range(E)])+ w_dn = torch.stack([_dequant_ref(dn[e], dns[e], dng[e]) for e in range(E)])+ routes = topk_ids.reshape(-1).long()+ h = torch.bmm(w_gu[routes], xq.repeat_interleave(TOPK, 0).unsqueeze(-1)).squeeze(-1).bfloat16().float()+ act = _gated_ref(h, activation, inter, alpha, limit or 0.0).bfloat16().float()+ g2 = float(act.abs().max()) / 2688 * math.pi / 3+ ref = torch.bmm(w_dn[routes], _nvfp4_qdq(act, g2).unsqueeze(-1)).squeeze(-1)+ ref = (ref * topk_weights.reshape(-1, 1)).view(tokens, TOPK, hidden_size).sum(1)++ def input_scale(top):+ return torch.full((E, 1), top, device=device).index_fill_(0, torch.tensor([1, 2], device=device), top / 2)++ pieces = {"gate_up": gu, "gate_up_scale": gus, "gate_up_global": gug, "down": dn, "down_scale": dns, "down_global": dng,+ "gate_input": input_scale(g1), "up_input": input_scale(g1).roll(1, 0), "down_input": input_scale(g2)}+ cfg = MoEConfig(num_experts=E, hidden=hidden_size, intermediate=inter, top_k=TOPK, activation=activation, alpha=alpha, limit=limit)+ kernel = W4a4Nvfp4MoEKernel()+ layout = kernel.layout(cfg)+ banks = {role: torch.empty((E, *spec.shape), dtype=spec.dtype, device=device) for role, spec in layout.items() if not spec.resident}+ alphas = kernel.pack(pieces, cfg, banks)+ view = ExpertView(banks, n=E, alphas=(alphas["gate_up_alpha"], alphas["down_alpha"]))+ layer = SimpleNamespace(activation=activation, alpha=alpha, limit=limit, apply_router_weight_on_input=False)+ out = kernel.apply(layer, hidden, topk_weights, topk_ids, view, is_prefill=True)+ assert torch.equal(alphas["gate_up_alpha"], torch.maximum(pieces["gate_input"], pieces["up_input"]).squeeze(1))+ torch.testing.assert_close(out.float(), ref, rtol=1e-2, atol=1e-2 * float(ref.abs().max()))++def _triton_cache(device, *, cache_size=S, prefill_overlap=False):"""Native 6-bank NVFP4 cache (no repack), consumed directly by the Triton kernels.The banks are not transformed, so ``sources`` doubles as the dequant reference."""