Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

14.4 KB
Raw Blame
"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels.

One program per token fuses the sublayer-boundary mix: apply the previous
sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre --
the fn GEMV over the flattened streams, flat-RMS normalization, and the three
gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an
n x n held in registers) -- and the pre-mixed layer input.

Semantics are defined by layers/mhc.py's torch reference (bit-comparable in
fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N
(hc_mult) is a constexpr; only N == 4 is exercised.
"""

from __future__ import annotations

import torch
import triton
import triton.language as tl


@triton.jit
def _mhc_stage1_kernel(
    x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr,
    res_out_ptr,
    sq_part_ptr,    # [T, NS] fp32
    mix_part_ptr,   # [T, NS, BLK_MIX] fp32
    H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
    SPLIT: tl.constexpr,     # hidden elems per split (multiple of BLOCK_H)
    BLOCK_H: tl.constexpr,
    NS: tl.constexpr,        # splits per token (the grid's second axis; need not be a power of two)
    HAS_POST: tl.constexpr,
):
    """Split-K stage of the fused mHC: each program owns one hidden slice of one
    token -- applies hc_post there, stores the updated streams, and reduces its
    partial sq-sum + fn-GEMV contribution over NS splits."""
    t = tl.program_id(0).to(tl.int64)
    s = tl.program_id(1)
    offs_mix = tl.arange(0, BLK_MIX)
    mix_mask = offs_mix < MIX
    offs_n = tl.arange(0, N)

    if HAS_POST:
        b_post = tl.load(post_ptr + t * N + offs_n)
        b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :])

    sqsum = 0.0
    acc = tl.zeros([BLK_MIX], dtype=tl.float32)
    for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H):
        offs_h = h0 + tl.arange(0, BLOCK_H)
        h_mask = offs_h < H
        if HAS_POST:
            b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
        for n in tl.static_range(N):
            if HAS_POST:
                r_new = tl.zeros([BLOCK_H], dtype=tl.float32)
                for i in tl.static_range(N):
                    r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
                    c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0))
                    r_new += c_in * r_i
                p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0))
                r_new += p_n * b_x
                # Round through the STORAGE dtype before the sq-sum and fn-GEMV:
                # the torch reference reads back what it stored (dtype-generic --
                # fp16/fp32 residuals must not be silently bf16-rounded).
                r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32)
                tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
            else:
                r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
                tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
            sqsum += tl.sum(r_new * r_new)
            fn_tile = tl.load(
                fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :],
                mask=mix_mask[:, None] & h_mask[None, :], other=0.0,
            )
            acc += tl.sum(fn_tile * r_new[None, :], axis=1)

    tl.store(sq_part_ptr + t * NS + s, sqsum)
    tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc)


@triton.jit
def _mhc_gates(
    t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
    post_out_ptr, comb_out_ptr, pre_out_ptr,
    rms_eps, hc_eps, post_mult,
    SINKHORN: tl.constexpr,
    H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
    NS: tl.constexpr, BLK_NS: tl.constexpr,
):
    """Reduce token ``t``'s split partials and run the tiny gate math (sigmoid gates,
    row-softmax + in-register 4x4 Sinkhorn); writes post / comb / pre for the next sublayer."""
    offs_mix = tl.arange(0, BLK_MIX)
    mix_mask = offs_mix < MIX
    offs_s = tl.arange(0, BLK_NS)
    s_mask = offs_s < NS

    sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s, mask=s_mask, other=0.0))
    acc = tl.sum(
        tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :], mask=s_mask[:, None], other=0.0),
        axis=0,
    )
    inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps)
    mixes = acc * inv_rms
    s0 = tl.load(scale_ptr + 0)
    s1 = tl.load(scale_ptr + 1)
    s2 = tl.load(scale_ptr + 2)
    b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0)

    is_pre = offs_mix < N
    is_post = (offs_mix >= N) & (offs_mix < 2 * N)
    gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2))
    logits = mixes * gate_scale + b_base
    pre = tl.sigmoid(logits) + hc_eps
    post_new = tl.sigmoid(logits) * post_mult

    offs_n2 = tl.arange(0, N)
    comb_logits = tl.zeros([N, N], dtype=tl.float32)
    for r in tl.static_range(N):
        for c in tl.static_range(N):
            lane = 2 * N + r * N + c
            v = tl.sum(tl.where(offs_mix == lane, logits, 0.0))
            comb_logits += tl.where(
                (offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0
            )
    row_max = tl.max(comb_logits, axis=1)
    e = tl.exp(comb_logits - row_max[:, None])
    comb = e / tl.sum(e, axis=1)[:, None] + hc_eps
    comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
    for _ in range(SINKHORN - 1):
        comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps)
        comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)

    post_g = tl.sum(
        tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0),
        axis=1,
    )
    pre_g = tl.sum(
        tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1
    )
    tl.store(post_out_ptr + t * N + offs_n2, post_g)
    tl.store(pre_out_ptr + t * N + offs_n2, pre_g)
    tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb)



@triton.jit
def _mhc_stage2_kernel(
    sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
    post_out_ptr, comb_out_ptr, pre_out_ptr,
    rms_eps, hc_eps, post_mult,
    SINKHORN: tl.constexpr,
    H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
    NS: tl.constexpr, BLK_NS: tl.constexpr,
):
    """Gate math alone (the two-pass variant, whose stage 3 consumes these pre gates)."""
    _mhc_gates(
        tl.program_id(0).to(tl.int64), sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
        rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
    )


@triton.jit
def _mhc_stage3_kernel(
    res_out_ptr, pre_ptr, li_out_ptr,
    H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr,
):
    """layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks."""
    t = tl.program_id(0).to(tl.int64)
    hb = tl.program_id(1)
    offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
    h_mask = offs_h < H
    offs_n = tl.arange(0, N)
    pre = tl.load(pre_ptr + t * N + offs_n)
    li = tl.zeros([BLOCK_H], dtype=tl.float32)
    for n in tl.static_range(N):
        r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
        li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
    tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)


@triton.jit
def _mhc_stage23_kernel(
    res_out_ptr, pre_ptr, li_out_ptr,
    sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
    post_out_ptr, comb_out_ptr, pre_out_ptr,
    rms_eps, hc_eps, post_mult,
    SINKHORN: tl.constexpr,
    H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
    NS: tl.constexpr, BLK_NS: tl.constexpr, BLOCK_H: tl.constexpr,
):
    """Single-pass mHC: the layer input mixes with the PREVIOUS sublayer's pre gates, so the
    gate math for the next sublayer (stage 2) does not gate stage 3 -- one launch does both:
    every program mixes one hidden chunk, program 0 of each token also reduces the split
    partials and writes the next sublayer's gates."""
    t = tl.program_id(0).to(tl.int64)
    hb = tl.program_id(1)
    offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
    h_mask = offs_h < H
    offs_n = tl.arange(0, N)
    pre = tl.load(pre_ptr + t * N + offs_n)
    li = tl.zeros([BLOCK_H], dtype=tl.float32)
    for n in tl.static_range(N):
        r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
        li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
    tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
    if hb == 0:
        _mhc_gates(
            t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
            rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
        )


def _stage1_hidden_tile(h: int) -> int:
    """The stage-1 hidden split. It fixes the fp32 reduction order over the hidden width, so it
    depends on the width alone and results reproduce across processes, machines and chunk sizes.
    128 gives 32 / 40 programs per token at widths 4096 / 5120."""
    return min(128, triton.next_power_of_2(h))


def _mhc_fused_launch(
    x: torch.Tensor,
    residual: torch.Tensor,
    post_mix: torch.Tensor | None,
    comb_mix: torch.Tensor | None,
    fn: torch.Tensor,
    hc_scale: torch.Tensor,
    hc_base: torch.Tensor,
    rms_eps: float,
    hc_eps: float,
    post_mult: float,
    sinkhorn_repeat: int,
    pre_mix_in: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """The three stages behind both variants. ``pre_mix_in`` selects single-pass: the
    layer input is mixed with it instead of with the pre gates stage 2 just computed
    (those are still returned, for the next sublayer). Returns (residual_new, post,
    comb, pre_new [T,N] fp32, layer_input)."""
    t, n, h = residual.shape
    mix = 2 * n + n * n
    assert fn.shape == (mix, n * h) and fn.dtype == torch.float32
    residual = residual.contiguous()
    has_post = post_mix is not None
    dev = residual.device

    res_out = torch.empty_like(residual)
    post_out = torch.empty(t, n, dtype=torch.float32, device=dev)
    comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev)
    li_out = torch.empty(t, h, dtype=residual.dtype, device=dev)

    block_h = split = _stage1_hidden_tile(h)
    ns = triton.cdiv(h, split)
    blk_mix = triton.next_power_of_2(mix)
    sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev)
    mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev)
    pre_out = torch.empty(t, n, dtype=torch.float32, device=dev)

    _mhc_stage1_kernel[(t, ns)](
        x.contiguous() if has_post else residual,  # dummy ptr when unused
        residual,
        post_mix.contiguous().view(t, n) if has_post else post_out,
        comb_mix.contiguous() if has_post else comb_out,
        fn, res_out, sq_part, mix_part,
        H=h, N=n, MIX=mix, BLK_MIX=blk_mix,
        SPLIT=split, BLOCK_H=block_h, NS=ns,
        HAS_POST=has_post,
        num_warps=4, num_stages=2,
    )
    block_h3 = min(1024, triton.next_power_of_2(h))
    if pre_mix_in is None:
        # two-pass: stage 3 mixes with the gates stage 2 just produced
        _mhc_stage2_kernel[(t,)](
            sq_part, mix_part, hc_scale, hc_base,
            post_out, comb_out, pre_out,
            rms_eps, hc_eps, post_mult,
            SINKHORN=sinkhorn_repeat,
            H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns),
            num_warps=1,
        )
        _mhc_stage3_kernel[(t, triton.cdiv(h, block_h3))](
            res_out, pre_out, li_out,
            H=h, N=n, BLOCK_H=block_h3,
            num_warps=4,
        )
    else:
        # single-pass: the mix uses the previous sublayer's gates, so stages 2 and 3 share a launch
        assert pre_mix_in.shape == (t, n), (pre_mix_in.shape, (t, n))
        _mhc_stage23_kernel[(t, triton.cdiv(h, block_h3))](
            res_out, pre_mix_in.to(torch.float32).contiguous(), li_out,
            sq_part, mix_part, hc_scale, hc_base,
            post_out, comb_out, pre_out,
            rms_eps, hc_eps, post_mult,
            SINKHORN=sinkhorn_repeat,
            H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns), BLOCK_H=block_h3,
            num_warps=4,
        )
    return res_out, post_out.view(t, n, 1), comb_out, pre_out, li_out


def mhc_fused_post_pre_triton(
    x: torch.Tensor,
    residual: torch.Tensor,
    post_mix: torch.Tensor | None,
    comb_mix: torch.Tensor | None,
    fn: torch.Tensor,
    hc_scale: torch.Tensor,
    hc_base: torch.Tensor,
    rms_eps: float,
    hc_eps: float,
    post_mult: float,
    sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage
    split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns
    (residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32,
    layer_input [T,H] bf16)."""
    res_out, post_out, comb_out, _pre_out, li_out = _mhc_fused_launch(
        x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
        rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=None,
    )
    return res_out, post_out, comb_out, li_out


def mhc_fused_post_pre_single_pass_triton(
    x: torch.Tensor,
    residual: torch.Tensor,
    post_mix: torch.Tensor | None,
    comb_mix: torch.Tensor | None,
    pre_mix_in: torch.Tensor,
    fn: torch.Tensor,
    hc_scale: torch.Tensor,
    hc_base: torch.Tensor,
    rms_eps: float,
    hc_eps: float,
    post_mult: float,
    sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """Single-pass mHC (DeepSeek-V4.1): same fused stages, but the layer input is mixed
    with the PREVIOUS sublayer's ``pre_mix_in`` and this sublayer's pre gates come back
    as ``pre_new`` for the next one. Returns (residual_new, post [T,N,1], comb [T,N,N],
    pre_new [T,N] fp32, layer_input)."""
    return _mhc_fused_launch(
        x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
        rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=pre_mix_in,
    )


__all__ = ["mhc_fused_post_pre_triton", "mhc_fused_post_pre_single_pass_triton"]