Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
17.2 KB
Raw Blame
// src/prefill/moe_f16_kq.cu - see moe_f16_kq.hpp.  gfx11 only (opt-in STRATA_PF_F16=1 with STRATA_PF_FUSED_KQ=1).
//
// gufo's routed-expert kernels for the K-quants, on Strata's tables, int8 activations and H (the work item, the
// activation staging and the epilogues are moe_f16_common.cuh's):
//   - gate/up (Q4_K, Q5_K): the weight codes stay packed in LDS (one 32-byte nibble group per row and stage) and
//     each wave turns its own 16 rows into F16 right after the LDS read: w = q * (d sc) - dmin m as one packed FMA;
//   - down (Q5_1): a K block's 16 code bytes and its high-bit word in LDS, w = q d + m per wave.
// The output differs from native_w11_kernel's (F16 weights instead of int8 sub-blocks), like STRATA_PF_FUSED_KQ's
// does from the MMQ path.
//
// Adapted from gufo 97b8645f: src/models/qwen38_flash_next/kernels/rocm/kernels.hip.cpp
// (RoutedF16GEMMKernel and its helpers).
// MIT License
// Copyright (c) 2026 gufo contributors
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "moe_f16_kq.hpp"

#include "moe_f16_common.cuh"

#include "ggml.h"

#include <algorithm>

namespace strata::prefill::fused::f16kq {
namespace {

using namespace f16;

constexpr int T_Q4_K = GGML_TYPE_Q4_K, T_Q5_K = GGML_TYPE_Q5_K;
constexpr int GU_NS = 2560 / 64, D_NS = 640 / 64;   // stages of 64 values
constexpr int FEAT = 64;                             // features (gate rows = up rows) per gate/up item

// Byte i of the 16-byte K-quant block header (d, dmin, scales[12]).
__device__ __forceinline__ uint32_t header_byte(const uint4& h, uint32_t i) {
    const uint32_t word = i < 4 ? h.x : i < 8 ? h.y : i < 12 ? h.z : h.w;
    return (word >> (8U * (i & 3U))) & 0xFFU;
}
// Bit s of each of the four bytes of w, packed into bits 0-3.
__device__ __forceinline__ uint32_t gather_bit(uint32_t w, int s) { return (((w >> s) & 0x01010101U) * 0x01020408U) >> 24U; }
// Bits 0-3 of `bits` to bit 4 of each byte.
__device__ __forceinline__ uint32_t spread_high_bits(uint32_t bits) { return (__umul24(bits, 0x00204081U) & 0x01010101U) << 4U; }

constexpr uint32_t HALF_MAGIC = 0x64646464U;   // the high bytes of 1024.0 as F16
// Four code bytes (< 32) to four halves: 1024 + q exactly, minus 1024, then q * scale + bias.
__device__ __forceinline__ void codes_to_halves(uint32_t codes, __half2 magic, __half2 scale2, __half2 bias2, __half2& lo,
                                                __half2& hi) {
#if STRATA_F16_DEVICE
    const uint32_t p0 = __builtin_amdgcn_perm(codes, HALF_MAGIC, 0x01050004U);
    const uint32_t p1 = __builtin_amdgcn_perm(codes, HALF_MAGIC, 0x03070206U);
    lo = __hfma2(__hadd2(__builtin_bit_cast(__half2, p0), magic), scale2, bias2);
    hi = __hfma2(__hadd2(__builtin_bit_cast(__half2, p1), magic), scale2, bias2);
#else
    (void) codes; (void) magic; (void) scale2; (void) bias2; lo = hi = __half2{};
#endif
}

// Gate/up (Q4_K / Q5_K): code rows 0..63 are the item's gate rows, 64..127 its up rows; a thread pair per row.
template <int WT, int BN>
__global__ void __launch_bounds__(THREADS) gate_up_kernel(const Batch b, const NativeGeom geo, const int32_t* __restrict__ ts,
                                                          const int2* __restrict__ tiles, const int32_t* __restrict__ off,
                                                          const uint8_t* __restrict__ act, const int32_t* __restrict__ src,
                                                          uint8_t* __restrict__ out) {
#if STRATA_F16_DEVICE
    constexpr bool Q5K = WT == T_Q5_K;
    constexpr int BLOCK_CHUNKS = Q5K ? 11 : 9;   // 16-byte chunks per 256-value block
    constexpr int CODE_CHUNK = Q5K ? 3 : 1;      // the first chunk of qs
    // LDS (bytes): codes ROWS x 2 chunks (swizzled); Q5_K's high bits and the (scale, bias) half2 per row and K
    // block; the activations.  The epilogue reuses the front.
    constexpr int CODE_BYTES = ROWS * 2 * 16;
    constexpr int HIGH_BYTES = Q5K ? 2 * ROWS * 4 : 0;
    constexpr int SCALE_BYTES = 2 * ROWS * 4;
    constexpr int STAGE_BYTES = CODE_BYTES + HIGH_BYTES + SCALE_BYTES + ActStage<BN>::BYTES;
    static_assert(GU_EPI_BYTES <= STAGE_BYTES, "the epilogue fits the stage");
    __shared__ __attribute__((aligned(16))) uint8_t lds[STAGE_BYTES];
    __shared__ int srow[BN];
    auto* s_codes = reinterpret_cast<uint4*>(lds);
    auto* s_high = reinterpret_cast<uint32_t*>(lds + CODE_BYTES);
    auto* s_scale = reinterpret_cast<uint32_t*>(lds + CODE_BYTES + HIGH_BYTES);
    auto* s_act = reinterpret_cast<uint4*>(lds + CODE_BYTES + HIGH_BYTES + SCALE_BYTES);

    const int tid = threadIdx.x, wave = tid >> 5, sub_lane = tid & 15;
    const int f_c = tid & 1;   // this thread's chunk (and K block) of its weight row
    const auto swizzle = [](int row, int c) { return row * 2 + (c ^ ((row >> 2) & 1)); };
    const __half2 magic = __floats2half2_rn(-1024.0f, -1024.0f);

    int fb, e, row0, nrows;
    if (!work_item<BN, GU_NFB>(b, ts, tiles, off, blockIdx.x, fb, e, row0, nrows)) return;
    const int live_tiles = (nrows + 15) / 16;
    const uint8_t* blob = b.blob[e - b.e0];
    if (tid < BN) srow[tid] = tid < nrows ? src[row0 + tid] : -1;
    __syncthreads();

    const uint8_t* f_ptr = blob + ((tid >> 1) >= FEAT ? geo.up_off : 0) + (size_t) (fb * FEAT + ((tid >> 1) & (FEAT - 1))) * geo.gu_row;
    uint4 f_header = make_uint4(0u, 0u, 0u, 0u), f_codes, f_qh[2] = {};
    uint4 code_cache[4];   // the block's four nibble groups, this thread's chunk of each
    uint32_t f_high = 0;
    int f_sb32 = 0;
    ActStage<BN> as;
    as.init(act, GU_ACT_LD, srow, tid);   // the tokens' activation rows

    const auto fetch_stage = [&](int s) {
        const int kb0 = 2 * s;
        const uint4* blk = reinterpret_cast<const uint4*>(f_ptr) + (kb0 / 8) * BLOCK_CHUNKS;
        if (kb0 % 8 == 0) {   // a new 256-value block every four stages
            f_header = blk[0];
#pragma unroll
            for (int group = 0; group < 4; ++group) code_cache[group] = blk[CODE_CHUNK + group * 2 + f_c];
            if constexpr (Q5K) {
                f_qh[0] = blk[1];
                f_qh[1] = blk[2];
            }
        }
        const int sb32 = (kb0 % 8) + f_c;
        {
            const int group = sb32 / 2;   // uniform across the wave
            const uint32_t low = 0u - (group & 1u), high = 0u - ((group >> 1u) & 1u);
            const auto select = [](uint32_t a, uint32_t bb, uint32_t mask) {
                uint32_t value;
                asm("v_cndmask_b32_e64 %0, %1, %2, %3" : "=v"(value) : "v"(a), "v"(bb), "s"(mask));
                return value;
            };
            const uint4 &c0 = code_cache[0], &c1 = code_cache[1], &c2 = code_cache[2], &c3 = code_cache[3];
            f_codes = make_uint4(select(select(c0.x, c1.x, low), select(c2.x, c3.x, low), high),
                                 select(select(c0.y, c1.y, low), select(c2.y, c3.y, low), high),
                                 select(select(c0.z, c1.z, low), select(c2.z, c3.z, low), high),
                                 select(select(c0.w, c1.w, low), select(c2.w, c3.w, low), high));
        }
        f_sb32 = sb32;
        if constexpr (Q5K) {
            const uint32_t qh[8] = {f_qh[0].x, f_qh[0].y, f_qh[0].z, f_qh[0].w, f_qh[1].x, f_qh[1].y, f_qh[1].z, f_qh[1].w};
            uint32_t hb = 0;
#pragma unroll
            for (int i = 0; i < 8; ++i) hb |= gather_bit(qh[i], sb32) << (4 * i);
            f_high = hb;
        }
        as.fetch(s);
    };

    const auto commit_stage = [&]() {
        const int row = tid >> 1;
        s_codes[swizzle(row, f_c)] = f_codes;
        if constexpr (Q5K) s_high[f_c * ROWS + row] = f_high;
        const int sb32 = f_sb32;
        uint32_t sc, mn;
        if (sb32 < 4) {
            sc = header_byte(f_header, 4 + sb32) & 0x3FU;
            mn = header_byte(f_header, 8 + sb32) & 0x3FU;
        } else {
            sc = (header_byte(f_header, 8 + sb32) & 0x0FU) | ((header_byte(f_header, sb32) >> 6U) << 4U);
            mn = (header_byte(f_header, 8 + sb32) >> 4U) | ((header_byte(f_header, 4 + sb32) >> 6U) << 4U);
        }
        const __half2 dm = __builtin_bit_cast(__half2, f_header.x);
        const float scale = __low2float(dm) * (float) sc, offset = __high2float(dm) * (float) mn;
        s_scale[f_c * ROWS + row] = __builtin_bit_cast(uint32_t, __floats2half2_rn(scale, -offset));
        as.commit(s_act);
    };

    v8f acc[BN / 16];
#pragma unroll
    for (int j = 0; j < BN / 16; ++j) acc[j] = v8f{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};

    const auto compute_stage = [&]() {
        const int row = wave * 16 + sub_lane;   // waves 0-3: gate rows, 4-7: the matching up rows
        const uint4 raw0 = s_codes[swizzle(row, 0)], raw1 = s_codes[swizzle(row, 1)];
#pragma unroll
        for (int kb = 0; kb < 2; ++kb) {
            const __half2 sb = __builtin_bit_cast(__half2, s_scale[kb * ROWS + row]);
            const __half2 scale2 = __low2half2(sb), bias2 = __high2half2(sb);
            // elements 0-15 of K block 2s + kb are the low (kb = 0) or high (kb = 1) nibbles of chunk 0, 16-31 of chunk 1
            const unsigned shift = 4U * (unsigned) kb;
            const uint32_t words[8] = {raw0.x, raw0.y, raw0.z, raw0.w, raw1.x, raw1.y, raw1.z, raw1.w};
            const uint32_t hb = Q5K ? s_high[kb * ROWS + row] : 0U;
            __half2 h[16];
#pragma unroll
            for (int i = 0; i < 8; ++i) {
                uint32_t nib = (words[i] >> shift) & 0x0F0F0F0FU;
                if constexpr (Q5K) nib |= spread_high_bits((hb >> (4 * i)) & 0xFU);
                codes_to_halves(nib, magic, scale2, bias2, h[2 * i], h[2 * i + 1]);
            }
            v16h a_lo, a_hi;
            __builtin_memcpy(&a_lo, &h[0], 32);
            __builtin_memcpy(&a_hi, &h[8], 32);
            mma_tiles<BN>(acc, a_lo, a_hi, s_act, kb, sub_lane, live_tiles);
        }
    };

    fetch_stage(0);
    for (int s = 0; s < GU_NS; ++s) {
        commit_stage();
        __syncthreads();
        if (s + 1 < GU_NS) fetch_stage(s + 1);
        compute_stage();
        __syncthreads();
    }
    gu_epilogue<BN>(acc, lds, tid, live_tiles, nrows, row0, fb, out);
#else
    (void) b; (void) geo; (void) ts; (void) tiles; (void) off; (void) act; (void) src; (void) out;
    __builtin_trap();
#endif
}

// Q5_1 down: 128 of the 2560 output rows against a tile's (or pair's) H rows, K = 640 in 10 stages; the routed rows'
// FP32 outputs in `dm` (native_w11_kernel<T_Q5_1, false>'s).
template <int BN>
__global__ void __launch_bounds__(THREADS) down_kernel(const Batch b, const NativeGeom geo, const int32_t* __restrict__ ts,
                                                       const int2* __restrict__ tiles, const int32_t* __restrict__ off,
                                                       const uint8_t* __restrict__ h, float* __restrict__ dm) {
#if STRATA_F16_DEVICE
    constexpr int CODE_BYTES = ROWS * 2 * 16, HIGH_BYTES = 2 * ROWS * 4, SCALE_BYTES = 2 * ROWS * 4;
    constexpr int STAGE_BYTES = CODE_BYTES + HIGH_BYTES + SCALE_BYTES + ActStage<BN>::BYTES;
    static_assert(D_EPI_BYTES <= STAGE_BYTES, "the epilogue fits the stage");
    __shared__ __attribute__((aligned(16))) uint8_t lds[STAGE_BYTES];
    __shared__ int srow[BN];
    auto* s_codes = reinterpret_cast<uint4*>(lds);
    auto* s_high = reinterpret_cast<uint32_t*>(lds + CODE_BYTES);
    auto* s_scale = reinterpret_cast<uint32_t*>(lds + CODE_BYTES + HIGH_BYTES);
    auto* s_act = reinterpret_cast<uint4*>(lds + CODE_BYTES + HIGH_BYTES + SCALE_BYTES);

    const int tid = threadIdx.x, wave = tid >> 5, sub_lane = tid & 15;
    const int f_c = tid & 1;   // this thread's K block of its weight row
    const auto swizzle = [](int row, int c) { return row * 2 + (c ^ ((row >> 2) & 1)); };
    const __half2 magic = __floats2half2_rn(-1024.0f, -1024.0f);

    int fb, e, row0, nrows;
    if (!work_item<BN, D_NFB>(b, ts, tiles, off, blockIdx.x, fb, e, row0, nrows)) return;
    const int live_tiles = (nrows + 15) / 16;
    if (tid < BN) srow[tid] = tid < nrows ? row0 + tid : -1;   // H is indexed by the routed row
    __syncthreads();

    const uint2* f_ptr = reinterpret_cast<const uint2*>(b.blob[e - b.e0] + geo.down_off + (size_t) (fb * ROWS + (tid >> 1)) * geo.d_row);
    uint4 f_codes;
    uint32_t f_high = 0, f_dm = 0;
    ActStage<BN> as;
    as.init(h, H_LD, srow, tid);
    const auto fetch_stage = [&](int s) {
        const uint2* words = f_ptr + (2 * s + f_c) * 3;   // block_q5_1: {d, m}, qh, qs[16]
        const uint2 w0 = words[0], w1 = words[1], w2 = words[2];
        f_codes = make_uint4(w1.x, w1.y, w2.x, w2.y);
        f_high = w0.y;
        f_dm = w0.x;
        as.fetch(s);
    };
    const auto commit_stage = [&]() {
        const int row = tid >> 1;
        s_codes[swizzle(row, f_c)] = f_codes;
        s_high[f_c * ROWS + row] = f_high;
        s_scale[f_c * ROWS + row] = f_dm;   // the half2 (d, m): w = q d + m
        as.commit(s_act);
    };
    v8f acc[BN / 16];
#pragma unroll
    for (int j = 0; j < BN / 16; ++j) acc[j] = v8f{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
    const auto compute_stage = [&]() {
        const int row = wave * 16 + sub_lane;
        const uint4 raw[2] = {s_codes[swizzle(row, 0)], s_codes[swizzle(row, 1)]};
#pragma unroll
        for (int kb = 0; kb < 2; ++kb) {
            const __half2 sb = __builtin_bit_cast(__half2, s_scale[kb * ROWS + row]);
            const __half2 scale2 = __low2half2(sb), bias2 = __high2half2(sb);
            // elements 0-15 are the low nibbles of the 16 code bytes, 16-31 the high, plus bit j of the high-bit word
            const uint32_t words[4] = {raw[kb].x, raw[kb].y, raw[kb].z, raw[kb].w};
            const uint32_t hb = s_high[kb * ROWS + row];
            __half2 hh[16];
#pragma unroll
            for (int i = 0; i < 4; ++i) {
                const uint32_t lo = (words[i] & 0x0F0F0F0FU) | spread_high_bits((hb >> (4 * i)) & 0xFU);
                const uint32_t hi = ((words[i] >> 4U) & 0x0F0F0F0FU) | spread_high_bits((hb >> (16 + 4 * i)) & 0xFU);
                codes_to_halves(lo, magic, scale2, bias2, hh[2 * i], hh[2 * i + 1]);
                codes_to_halves(hi, magic, scale2, bias2, hh[8 + 2 * i], hh[8 + 2 * i + 1]);
            }
            v16h a_lo, a_hi;
            __builtin_memcpy(&a_lo, &hh[0], 32);
            __builtin_memcpy(&a_hi, &hh[8], 32);
            mma_tiles<BN>(acc, a_lo, a_hi, s_act, kb, sub_lane, live_tiles);
        }
    };
    fetch_stage(0);
    for (int s = 0; s < D_NS; ++s) {
        commit_stage();
        __syncthreads();
        if (s + 1 < D_NS) fetch_stage(s + 1);
        compute_stage();
        __syncthreads();
    }
    down_epilogue<BN>(acc, lds, tid, live_tiles, nrows, row0, fb, dm);
#else
    (void) b; (void) geo; (void) ts; (void) tiles; (void) off; (void) h; (void) dm;
    __builtin_trap();
#endif
}

}  // namespace

bool covers(int gu_type) { return gu_type == T_Q4_K || gu_type == T_Q5_K; }

bool covers_down(int d_type) { return d_type == GGML_TYPE_Q5_1; }

void gate_up(const Batch& b, const NativeGeom& g, const int32_t* ts, const int2* tiles, const int32_t* off, int64_t n,
             int n_expert, int64_t work_bound, const uint8_t* act, const int32_t* src, uint8_t* out, cudaStream_t s) {
    // one block per work item, the grid the batch's bound (gufo's launch: the hardware balances the blocks; the items
    // of a tile are consecutive, so they meet its activations in L2)
    const unsigned grid = (unsigned) std::max<int64_t>(1, work_bound * GU_NFB);
    const bool q5k = g.gu_type == T_Q5_K;
    if (pick_bn(n, n_expert) == 2 * kTileRows) {
        if (q5k) gate_up_kernel<T_Q5_K, 2 * kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, act, src, out);
        else gate_up_kernel<T_Q4_K, 2 * kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, act, src, out);
    } else {
        if (q5k) gate_up_kernel<T_Q5_K, kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, act, src, out);
        else gate_up_kernel<T_Q4_K, kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, act, src, out);
    }
}

void down(const Batch& b, const NativeGeom& g, const int32_t* ts, const int2* tiles, const int32_t* off, int64_t n, int n_expert,
          int64_t work_bound, const uint8_t* h, float* dm, cudaStream_t s) {
    const unsigned grid = (unsigned) std::max<int64_t>(1, work_bound * D_NFB);
    if (pick_bn(n, n_expert) == 2 * kTileRows) down_kernel<2 * kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, h, dm);
    else down_kernel<kTileRows><<<grid, THREADS, 0, s>>>(b, g, ts, tiles, off, h, dm);
}

}  // namespace strata::prefill::fused::f16kq