Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/cuda/s2_expert_grouped.cu - R4's grouped GPU expert. Read the header first.
//
// THE ARITHMETIC IS `src/kernels/cpu/expert.cpp`'s, over the same bytes, one warp per output row.
//
// sum_j (c_j - 1) * d_w * xhat_j * d_x == d_w * d_x * ( sum_j c_j*xhat_j - sum_j xhat_j )
//
// Both sums are INT8 x INT8, so both are `__dp4a` - four multiply-accumulates per instruction, exact in
// integer and with no dequantization inside the loop. The weight scale `d_w` is fp16 (one per 64 elements)
// and the activation scale `d_x` is fp16 (one per 32 elements), so the float work is one multiply-add per
// 32-element chunk rather than one per element.
//
// **THE SUMMATION ORDER IS NOT THE CPU's AND CANNOT BE.** Lane `L` takes chunks `L, L+32, ...` and the
// partials are reduced through shuffle; the CPU walks every chunk in order with its own accumulator shape.
// The two agree to float rounding and not to the bit, which is the same contract `s_gemv_parity` carries for
// the same reason. `bench/micro/moe_hit_parity.cu` is the check.
#include "strata/kernels/s2_expert_grouped.hpp"
#include "strata/core/geometry.hpp"
#include "strata/kernels/dp4a.hpp"
#include "strata/kernels/f16_bits.hpp"
#include "strata/kernels/quantize_act.hpp"
#include "strata/kernels/fuse_decode.hpp"
#include "strata/kernels/verify_kernels.hpp"
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
namespace strata::kernels {
namespace {
// THE BLOB'S OWN GEOMETRY, from `include/strata/kernels/cpu/expert.hpp`. Restated as literals because that
// header is the CPU path's and this file must not silently follow it if the two ever disagree: the sizes below
// are what the CPU kernel's indexing computes, and `moe_hit_parity` compares the two end to end.
constexpr int H = 2560;
constexpr int FF = 640;
static_assert(H == strata::core::kCompiledGeometry.n_embd && FF == strata::core::kCompiledGeometry.n_ff, "s2_expert_grouped is compiled for core/geometry.hpp");
constexpr int QK = 64; // Q2_0's group: one fp16 scale per 64 weights
constexpr int ROW_GU = H / 4; // 640 B of codes per gate/up row (2 bits per element)
constexpr int ROW_D = FF / 4; // 160 B per down row
constexpr int SC_GU = H / QK; // 40 fp16 scales per gate/up row
constexpr int SC_D = FF / QK; // 10 per down row
constexpr size_t O_D_CODES = (size_t) 2 * FF * ROW_GU;
constexpr size_t O_GU_SCALES = O_D_CODES + (size_t) H * ROW_D;
constexpr size_t O_D_SCALES = O_GU_SCALES + (size_t) 2 * FF * SC_GU * 2;
__device__ __forceinline__ float f16_at(const uint8_t* p) {
return __half2float(__ushort_as_half((uint16_t) (p[0] | (p[1] << 8))));
}
/// **ONE S2 ROW AGAINST A Q8_0 ACTIVATION, WARP-WIDE.** Every lane accumulates its own float partial over a
/// strided set of 32-element chunks and the caller reduces; `chunk` indices are absolute so the caller can
/// start the lane at any offset.
///
/// Returns the lane's partial. `codes` is `n_in/4` bytes and `scales` `n_in/QK` fp16, both for THIS row.
///
/// **`x_scales` IS R4.2h AND IT IS OPTIONAL ON PURPOSE.** When it is non-null the activation's multiplier
/// comes from an fp32 array instead of the block's fp16 `d`. The CPU pool multiplies by the fp32
/// `ActQ::scale` (`cpu/expert.cpp:92`), and the two disagreed by **4.761e-04 relative on 80 of 80 chunks**
/// (`bench/micro/act_quant_parity.cu`), which is what made a cache hit compute a different expert from a
/// cache miss. Null keeps the previous fp16 behaviour **exactly**, so `moe_hit_parity` - which passes no
/// scales - still measures the kernel it always measured.
__device__ __forceinline__ float row_dot_s2_q8(const uint8_t* __restrict__ codes,
const uint8_t* __restrict__ scales,
const uint8_t* __restrict__ x_q8_0, int n_chunks, int lane,
const float* __restrict__ x_scales = nullptr) {
float acc = 0.0f;
for (int c = lane; c < n_chunks; c += 32) {
const uint8_t* cb = codes + (size_t) c * 8; // 8 code bytes = 32 elements
const uint8_t* xb = x_q8_0 + (size_t) c * 34; // one block_q8_0
const float dx = x_scales ? x_scales[c] : f16_at(xb);
const int8_t* xq = (const int8_t*) (xb + 2);
// ---- THE CODES, EXPANDED TO ONE BYTE PER ELEMENT, THEN FOUR AT A TIME INTO `dp4a`.
//
// The packed form is LSB-first: element 4j+k is bits [2k, 2k+2) of code byte j. `dp4a` needs both
// operands as packed int8, so each code byte becomes a word whose four bytes are its four 2-bit
// fields - which is exactly `(c & 3) | ((c>>2)&3)<<8 | ((c>>4)&3)<<16 | ((c>>6)&3)<<24`.
int s = 0; // sum of code * x
int hx = 0; // sum of x - the weight-independent term, as ones * x
const int ones = 0x01010101;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const unsigned cbyte = cb[j];
const int cw = (int) ((cbyte & 3u) | (((cbyte >> 2) & 3u) << 8) | (((cbyte >> 4) & 3u) << 16) |
(((cbyte >> 6) & 3u) << 24));
// **`memcpy`, NOT A CAST.** A `block_q8_0` is 34 BYTES - two of header then 32 of int8 - so the
// activation data at offset 2 is never 4-byte aligned, and `*(const int*)(xq + 4*j)` faults with
// "misaligned address". It faulted exactly that way on this kernel's first run. `memcpy` of a
// known 4 bytes compiles to whatever load is legal for the alignment, which is the point of using
// it rather than reasoning about which cast happens to work.
int xw;
memcpy(&xw, xq + 4 * j, 4);
s = STRATA_DP4A(cw, xw, s);
hx = STRATA_DP4A(ones, xw, hx);
}
// One weight scale per 64 elements, so per TWO 32-element chunks.
const float dw = f16_at(scales + (size_t) (c >> 1) * 2);
acc += dw * dx * (float) (s - hx);
}
return acc;
}
__device__ __forceinline__ float warp_sum(float v) {
#pragma unroll
for (int off = 16; off > 0; off >>= 1) v += __shfl_down_sync(0xFFFFFFFFu, v, off);
return v;
}
/// GATE AND UP, ONE WARP PER ROW.
///
/// **THE ROWS ARE INTERLEAVED AND THE FIRST VERSION OF THIS GOT IT WRONG.** In the blob, row-SLOT `i` of the
/// `2*FF` gate/up rows is gate row `i/2` when `i` is even and up row `(i-1)/2` when it is odd - that is what
/// `O_GU_CODES + (2r)*ROW_GU` and `+ (2r+1)*ROW_GU` mean in `expert.cpp`. This kernel decoded the correct
/// code row for slot `i` and then wrote it to output slot `i` of a layout whose first `FF` entries are gate and
/// whose last `FF` are up, which pairs `silu(gate[r]) * up[r]` with the WRONG `up` for every `r`. It produced
/// finite, plausible numbers. `moe_hit_parity` caught it on the first run, at worst relative error 2.2e+03.
///
/// So slot `i` is DECODED from row-slot `i` and WRITTEN to the output slot its parity says it belongs to.
__global__ void gu_kernel(const uint8_t* __restrict__ blob_base, const int32_t* __restrict__ slot_index,
long long blob_bytes, const uint8_t* __restrict__ x_q8_0,
const float* __restrict__ x_scales, float* __restrict__ gate_up, int n_hits,
const int32_t* __restrict__ d_count = nullptr,
const int32_t* __restrict__ dst_index = nullptr, int tok_div = 0) {
const int warps_per_block = (int) (blockDim.x >> 5);
const long long slot = (long long) blockIdx.x * warps_per_block + (threadIdx.x >> 5);
const long long rows_per_hit = 2LL * FF;
const long long total = (long long) n_hits * rows_per_hit;
if (slot >= total) return;
const int h = (int) (slot / rows_per_hit);
if (d_count != nullptr && h >= *d_count) return; // token graph: capacity layout, device count
const int i = (int) (slot % rows_per_hit);
const int lane = threadIdx.x & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
if (tok_div > 0) { // plan v0.3 P6 verify window: each hit reads its own token's activation
const int tok = dst_index[h] / tok_div;
x_q8_0 += (size_t) tok * (size_t) (H / 32) * 34;
if (x_scales != nullptr) x_scales += (size_t) tok * (size_t) (H / 32);
}
const float acc = row_dot_s2_q8(blob + (size_t) i * ROW_GU,
blob + O_GU_SCALES + (size_t) i * SC_GU * 2, x_q8_0, H / 32, lane, x_scales);
const float s = warp_sum(acc);
if (lane != 0) return;
// ---- THE OUTPUT LAYOUT IS GATE-MAJOR, AND THAT IS NOT COSMETIC.
//
// The first version wrote `gate_up[h*2FF + {0..FF-1}] = gate` and `[h*2FF + FF..] = up`, i.e. a per-hit
// [gate | up] pair, and then called the shared `swiglu_kernel` and `quantize_q8_0` over the whole buffer.
// Both of those walk a CONTIGUOUS range, so with more than one hit they read hit 0's UP rows where they
// wanted hit 1's GATE rows - finite numbers, wrong expert. With one hit it would have passed.
//
// Gate-major removes the mismatch instead of adding a stride to two other kernels: every hit's gate rows
// are contiguous from 0, every hit's up rows are contiguous from `n_hits * FF`, and the swiglu's output
// lands in the first `n_hits * FF` floats exactly where `quantize_q8_0` reads it.
const int r = i >> 1;
const size_t base = (i & 1) ? ((size_t) n_hits * FF + (size_t) h * FF) : ((size_t) h * FF);
gate_up[base + (size_t) r] = s;
}
// `silu(gate) * up` lives in `strata/kernels/swiglu.cuh` (the fused kernels call it in their epilogues);
// the standalone pass it replaced is gone. The scratch layout is unchanged: `[0, n_pairs)` ends up holding
// the products, `[n_pairs, 2*n_pairs)` the raw up rows.
/// `silu(gate) * up`, in place, over a GATE-MAJOR buffer: `[0, n_pairs)` is every hit's gate and
/// `[n_pairs, 2*n_pairs)` is every hit's up, so hit `h`'s row `r` meets itself at `h*FF + r`.
///
/// SiLU on the GATE and multiplied by up - the reading `docs/semantics.md` records, and the one that is wrong
/// the other way round in a way that still produces a finite number.
__global__ void swiglu_kernel(float* __restrict__ gate_up, long long n_pairs) {
const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n_pairs) return;
const float g = gate_up[i];
const float u = gate_up[n_pairs + i];
gate_up[i] = (g / (1.0f + __expf(-g))) * u;
}
__global__ void swiglu_quantize_q8_0_scaled_kernel(float* __restrict__ gate_up, long long n_pairs,
uint8_t* __restrict__ blocks, float* __restrict__ scales) {
const long long n_blocks = n_pairs >> 5;
const long long b = ((long long) blockIdx.x * blockDim.x + threadIdx.x) >> 5;
if (b >= n_blocks) return;
const int lane = threadIdx.x & 31;
const long long idx = b * 32 + lane;
const float g = gate_up[idx];
const float u = gate_up[n_pairs + idx];
const float xv = (g / (1.0f + __expf(-g))) * u;
gate_up[idx] = xv;
uint8_t* out = blocks + b * 34;
float amax = fabsf(xv);
#pragma unroll
for (int off = 16; off > 0; off >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, off));
const float s = amax > 0.f ? amax / 127.f : 0.f;
const float inv = s > 0.f ? 1.f / s : 0.f;
if (lane == 0) {
scales[b] = s;
const uint16_t d16bits = f16_from_f32(s);
out[0] = (uint8_t) (d16bits & 0xFF);
out[1] = (uint8_t) (d16bits >> 8);
}
const float t = xv * inv;
const float r = t + (t >= 0.f ? 0.5f : -0.5f);
int v = (int) r;
v = v < -127 ? -127 : (v > 127 ? 127 : v);
out[2 + lane] = (uint8_t) (int8_t) v;
}
/// DOWN, ONE WARP PER ROW, reading the quantized intermediate the caller produced.
///
/// `dst_index[h]` is which row of the shared output buffer hit `h` fills - see the header. It is the router's
/// slot, not `h`, and the two differ on every layer where some experts are resident and some are not.
__global__ void down_kernel(const uint8_t* __restrict__ blob_base, const int32_t* __restrict__ slot_index,
const int32_t* __restrict__ dst_index, long long blob_bytes,
const uint8_t* __restrict__ h_q8_0, const float* __restrict__ h_scales,
float* __restrict__ out, int n_hits, const int32_t* __restrict__ d_count = nullptr) {
const int warps_per_block = (int) (blockDim.x >> 5);
const long long row = (long long) blockIdx.x * warps_per_block + (threadIdx.x >> 5);
const long long total = (long long) n_hits * H;
if (row >= total) return;
const int h = (int) (row / H);
if (d_count != nullptr && h >= *d_count) return;
const int r = (int) (row % H);
const int lane = threadIdx.x & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
const uint8_t* xb = h_q8_0 + (size_t) h * (size_t) (FF / 32) * 34;
const float acc = row_dot_s2_q8(blob + O_D_CODES + (size_t) r * ROW_D,
blob + O_D_SCALES + (size_t) r * SC_D * 2, xb, FF / 32, lane,
h_scales ? h_scales + (size_t) h * (size_t) (FF / 32) : nullptr);
const float s = warp_sum(acc);
if (lane == 0) out[(size_t) dst_index[h] * H + r] = s;
}
// ---- THE SAME INTEGERS FROM FEWER INSTRUCTIONS. `STRATA_OLD_GROUPED=1` keeps the kernels above.
//
// **INSIDE A CHUNK, WHICH CODE MEETS WHICH ACTIVATION IN A `dp4a` WORD IS FREE.** `s = sum_e code_e * x_e` and
// `hx = sum_e x_e` are exact integer sums (|s| <= 32 * 3 * 128), so any grouping of the 32 products into words
// gives the same two integers. The grouping used here costs no transposition of the codes: `(w >> 2f) & 0x03030303`
// holds field `f` of each of a code word's four bytes - elements `4b + f`, plus 16 for the second word - so it is
// the ACTIVATIONS that are regrouped, once per chunk, into
//
// X[4h + f] = { x[16h + f], x[16h + 4 + f], x[16h + 8 + f], x[16h + 12 + f] }
//
// a 4x4 byte transpose of each half of the chunk, eight `__byte_perm`. The kernels above spend seven integer
// operations per code BYTE and a second `dp4a` chain for `hx` on every row; here a code word costs seven for 16
// elements and `hx` is summed once per chunk. `s`, `hx` and the float expression `dw * dx * (float) (s - hx)` are
// unchanged, so every output is bitwise the previous kernels' - `s2_expert_grouped_parity` checks exactly that.
// fp16 at an even address, one 16-bit load (`f16_at` reads it as two bytes); same value.
__device__ __forceinline__ float f16_ld(const uint8_t* p) {
return __half2float(__ushort_as_half(*(const unsigned short*) p));
}
// A chunk's two code words as the eight `dp4a` operands that pair with `X[0..7]`: `m[4h + f]` byte `b` is the
// code of element `16h + 4b + f`.
__device__ __forceinline__ void expand_codes(uint2 cb, int m[8]) {
const unsigned M = 0x03030303u;
m[0] = (int) (cb.x & M);
m[1] = (int) ((cb.x >> 2) & M);
m[2] = (int) ((cb.x >> 4) & M);
m[3] = (int) ((cb.x >> 6) & M);
m[4] = (int) (cb.y & M);
m[5] = (int) ((cb.y >> 2) & M);
m[6] = (int) ((cb.y >> 4) & M);
m[7] = (int) ((cb.y >> 6) & M);
}
// The 32 int8 of the `block_q8_0` at `xb` regrouped into `X[0..7]` (see above), and their sum `hx`.
//
// **NINE ALIGNED WORD LOADS INSTEAD OF 32 BYTE LOADS.** The caller guarantees a 4-byte aligned activation row, so
// chunk `c`'s int8 start at `34c + 2`, i.e. 0 bytes (odd `c`) or 2 bytes (even `c`) into an aligned word; the
// words are read aligned and funnel-shifted into place. The ninth word is read only in the 2-byte case, and then
// it lies inside block `c + 1` of the same row: both row lengths (80 and 20 chunks) are even, so an even chunk is
// never a row's last.
__device__ __forceinline__ int load_x_chunk(const uint8_t* __restrict__ xb, int X[8]) {
const uint8_t* q = xb + 2;
const int off = (int) ((uintptr_t) q & 3);
const unsigned* p = (const unsigned*) (q - off);
const unsigned sh = (unsigned) off * 8;
unsigned v[9];
#pragma unroll
for (int j = 0; j < 8; ++j) v[j] = __ldg(p + j);
v[8] = off != 0 ? __ldg(p + 8) : 0u;
unsigned n[8];
int hx = 0;
#pragma unroll
for (int j = 0; j < 8; ++j) {
// natural word j: x[4j .. 4j+3]; a 64-bit shift (sh is 0, 8, 16 or 24) rather than __funnelshift_r, so the same
// source needs no CUDA-only intrinsic.
n[j] = (unsigned) ((((unsigned long long) v[j + 1] << 32) | v[j]) >> sh);
hx = STRATA_DP4A(0x01010101, (int) n[j], hx);
}
#pragma unroll
for (int h = 0; h < 2; ++h) {
const unsigned t0 = __byte_perm(n[4 * h], n[4 * h + 1], 0x5140); // a0 b0 a1 b1
const unsigned t1 = __byte_perm(n[4 * h], n[4 * h + 1], 0x7362); // a2 b2 a3 b3
const unsigned t2 = __byte_perm(n[4 * h + 2], n[4 * h + 3], 0x5140); // c0 d0 c1 d1
const unsigned t3 = __byte_perm(n[4 * h + 2], n[4 * h + 3], 0x7362); // c2 d2 c3 d3
X[4 * h + 0] = (int) __byte_perm(t0, t2, 0x5410); // a0 b0 c0 d0
X[4 * h + 1] = (int) __byte_perm(t0, t2, 0x7632); // a1 b1 c1 d1
X[4 * h + 2] = (int) __byte_perm(t1, t3, 0x5410); // a2 b2 c2 d2
X[4 * h + 3] = (int) __byte_perm(t1, t3, 0x7632); // a3 b3 c3 d3
}
return hx;
}
// `s` for one chunk: the eight expanded code words against the eight regrouped activation words.
__device__ __forceinline__ int chunk_s(const int m[8], const int X[8]) {
int s = 0;
#pragma unroll
for (int j = 0; j < 8; ++j) s = STRATA_DP4A(m[j], X[j], s);
return s;
}
/// `gu_kernel`, new: ONE WARP PER (gate, up) PAIR - row-slots `2r` and `2r + 1`, adjacent in the blob - so each
/// activation chunk is loaded and regrouped once for both rows. Per row, the lane's chunks, their order, the float
/// expression and the shuffle reduction are `row_dot_s2_q8`'s; the outputs land where `gu_kernel` puts them.
__global__ void gu_pair_kernel(const uint8_t* __restrict__ blob_base, const int32_t* __restrict__ slot_index,
long long blob_bytes, const uint8_t* __restrict__ x_q8_0,
const float* __restrict__ x_scales, float* __restrict__ gate_up, int n_hits,
const int32_t* __restrict__ d_count, const int32_t* __restrict__ dst_index,
int tok_div) {
const int warps_per_block = (int) (blockDim.x >> 5);
const long long pair = (long long) blockIdx.x * warps_per_block + (threadIdx.x >> 5);
const long long total = (long long) n_hits * FF;
if (pair >= total) return;
const int h = (int) (pair / FF);
if (d_count != nullptr && h >= *d_count) return;
const int r = (int) (pair % FF);
const int lane = threadIdx.x & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
if (tok_div > 0) {
const int tok = dst_index[h] / tok_div;
x_q8_0 += (size_t) tok * (size_t) (H / 32) * 34;
if (x_scales != nullptr) x_scales += (size_t) tok * (size_t) (H / 32);
}
const uint8_t* codes = blob + (size_t) (2 * r) * ROW_GU; // gate row; the up row follows it
const uint8_t* scales = blob + O_GU_SCALES + (size_t) (2 * r) * SC_GU * 2;
float acc_g = 0.0f, acc_u = 0.0f;
for (int c = lane; c < H / 32; c += 32) {
const uint8_t* xb = x_q8_0 + (size_t) c * 34;
const float dx = x_scales ? x_scales[c] : f16_ld(xb);
int X[8], m[8];
const int hx = load_x_chunk(xb, X);
expand_codes(*(const uint2*) (codes + (size_t) c * 8), m);
const float dw_g = f16_ld(scales + (size_t) (c >> 1) * 2);
acc_g += dw_g * dx * (float) (chunk_s(m, X) - hx);
expand_codes(*(const uint2*) (codes + ROW_GU + (size_t) c * 8), m);
const float dw_u = f16_ld(scales + SC_GU * 2 + (size_t) (c >> 1) * 2);
acc_u += dw_u * dx * (float) (chunk_s(m, X) - hx);
}
const float sg = warp_sum(acc_g);
const float su = warp_sum(acc_u);
if (lane != 0) return;
gate_up[(size_t) h * FF + (size_t) r] = sg; // gate-major, as in `gu_kernel`
gate_up[(size_t) n_hits * FF + (size_t) h * FF + (size_t) r] = su;
}
/// `down_kernel`, new: ONE WARP PER PAIR OF ROWS `r, r + 1` of one hit, the intermediate's chunk loaded once for both.
__global__ void down_pair_kernel(const uint8_t* __restrict__ blob_base, const int32_t* __restrict__ slot_index,
const int32_t* __restrict__ dst_index, long long blob_bytes,
const uint8_t* __restrict__ h_q8_0, const float* __restrict__ h_scales,
float* __restrict__ out, int n_hits, const int32_t* __restrict__ d_count) {
const int warps_per_block = (int) (blockDim.x >> 5);
const long long pair = (long long) blockIdx.x * warps_per_block + (threadIdx.x >> 5);
const long long total = (long long) n_hits * (H / 2);
if (pair >= total) return;
const int h = (int) (pair / (H / 2));
if (d_count != nullptr && h >= *d_count) return;
const int r = 2 * (int) (pair % (H / 2));
const int lane = threadIdx.x & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
const uint8_t* xrow = h_q8_0 + (size_t) h * (size_t) (FF / 32) * 34;
const float* xs = h_scales ? h_scales + (size_t) h * (size_t) (FF / 32) : nullptr;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
float acc0 = 0.0f, acc1 = 0.0f;
for (int c = lane; c < FF / 32; c += 32) {
const uint8_t* xb = xrow + (size_t) c * 34;
const float dx = xs ? xs[c] : f16_ld(xb);
int X[8], m[8];
const int hx = load_x_chunk(xb, X);
expand_codes(*(const uint2*) (codes + (size_t) c * 8), m);
const float dw0 = f16_ld(scales + (size_t) (c >> 1) * 2);
acc0 += dw0 * dx * (float) (chunk_s(m, X) - hx);
expand_codes(*(const uint2*) (codes + ROW_D + (size_t) c * 8), m);
const float dw1 = f16_ld(scales + SC_D * 2 + (size_t) (c >> 1) * 2);
acc1 += dw1 * dx * (float) (chunk_s(m, X) - hx);
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane != 0) return;
const size_t o = (size_t) dst_index[h] * H + (size_t) r;
out[o] = s0;
out[o + 1] = s1;
}
// The CPU subtracts the weight bias after its eight FMA accumulators have been reduced. Moving the
// subtraction into each integer dot, as the legacy kernel does, changes rounding even with equal scales.
__global__ void activation_correction_kernel(const uint8_t* q8, const float* scales, float* hx, int chunks) {
const int c = blockIdx.x * blockDim.x + threadIdx.x;
if (c >= chunks) return;
const int8_t* q = (const int8_t*) (q8 + (size_t) c * 34 + 2);
int sum = 0;
for (int j = 0; j < 32; ++j) sum += q[j];
hx[c] = __fmul_rn(scales[c], (float) sum);
}
__device__ __forceinline__ int dot4(const uint8_t* codes, const int8_t* q) {
const unsigned c = *codes;
const int cw = (int) ((c & 3u) | (((c >> 2) & 3u) << 8) |
(((c >> 4) & 3u) << 16) | (((c >> 6) & 3u) << 24));
int xw;
memcpy(&xw, q, sizeof xw);
return STRATA_DP4A(cw, xw, 0);
}
__device__ __forceinline__ float row_dot_cpu_order(const uint8_t* codes, const uint8_t* scales,
const uint8_t* xq, const float* xs,
const float* hx, int blocks, int lane) {
float acc = 0.0f;
float corr = 0.0f;
for (int b = 0; b < blocks; ++b) {
const float d = f16_at(scales + 2 * b);
const int lo = dot4(codes + b * 16 + lane,
(const int8_t*) (xq + (size_t) (2 * b) * 34 + 2) + lane * 4);
const int hi = dot4(codes + b * 16 + 8 + lane,
(const int8_t*) (xq + (size_t) (2 * b + 1) * 34 + 2) + lane * 4);
acc = __fmaf_rn(__fmul_rn(d, xs[2 * b]), (float) lo, acc);
acc = __fmaf_rn(__fmul_rn(d, xs[2 * b + 1]), (float) hi, acc);
if (lane == 0)
corr = __fadd_rn(corr, __fmul_rn(d, __fadd_rn(hx[2 * b], hx[2 * b + 1])));
}
// _mm_add_ps(low128, high128), then two _mm_hadd_ps. The pair order is 4, 1, 2;
// a standard shuffle tree in the order 4, 2, 1 is a different floating-point expression.
constexpr unsigned mask = 0xffffffffu;
acc = __fadd_rn(acc, __shfl_down_sync(mask, acc, 4, 8));
acc = __fadd_rn(acc, __shfl_down_sync(mask, acc, 1, 8));
acc = __fadd_rn(acc, __shfl_down_sync(mask, acc, 2, 8));
return __fsub_rn(acc, corr); // Only lane zero is consumed.
}
template <bool DOWN>
__global__ void cpu_order_projection_kernel(const uint8_t* blob_base, const int32_t* slots,
const int32_t* destinations, long long blob_bytes,
const uint8_t* xq, const float* xs, const float* hx,
float* out, int n_hits) {
constexpr int rows_per_hit = DOWN ? H : 2 * FF;
const int row = blockIdx.x * (blockDim.x / 8) + threadIdx.x / 8;
if (row >= n_hits * rows_per_hit) return;
const int h = row / rows_per_hit;
const int r = row % rows_per_hit;
const int lane = threadIdx.x & 7;
const uint8_t* blob = blob_base + (size_t) slots[h] * (size_t) blob_bytes;
const int chunks_offset = DOWN ? h * (FF / 32) : 0;
const uint8_t* codes = DOWN ? blob + O_D_CODES + (size_t) r * ROW_D : blob + (size_t) r * ROW_GU;
const uint8_t* scales = DOWN ? blob + O_D_SCALES + (size_t) r * SC_D * 2
: blob + O_GU_SCALES + (size_t) r * SC_GU * 2;
const float value = row_dot_cpu_order(codes, scales, xq + (size_t) chunks_offset * 34,
xs + chunks_offset, hx + chunks_offset,
DOWN ? SC_D : SC_GU, lane);
if (lane != 0) return;
if (DOWN) out[(size_t) destinations[h] * H + r] = value;
else out[((r & 1) ? (size_t) n_hits * FF : 0) + (size_t) h * FF + (r >> 1)] = value;
}
__global__ void cpu_order_swiglu_kernel(float* gu, int pairs) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= pairs) return;
const float g = gu[i];
// Accurate fp32 exponential; __expf's approximation would introduce an additional source of error.
// CPU/GPU libc last-bit differences are diagnosed separately by the micro, not hidden with FP64 here.
const float eg = expf(-g);
gu[i] = __fmul_rn(__fdiv_rn(g, __fadd_rn(1.0f, eg)), gu[pairs + i]);
}
__global__ void cpu_order_quantize_kernel(const float* x, uint8_t* blocks, float* scales,
float* hx, int chunks) {
const int c = blockIdx.x * blockDim.x + threadIdx.x;
if (c >= chunks) return;
const float* xb = x + c * 32;
uint8_t* out = blocks + (size_t) c * 34;
float amax = 0.0f;
for (int j = 0; j < 32; ++j) amax = fmaxf(amax, fabsf(xb[j]));
const float s = amax > 0.0f ? __fdiv_rn(amax, 127.0f) : 0.0f;
const float inv = s > 0.0f ? __fdiv_rn(1.0f, s) : 0.0f;
scales[c] = s;
const uint16_t bits = __half_as_ushort(__float2half_rn(s));
out[0] = (uint8_t) bits;
out[1] = (uint8_t) (bits >> 8);
int sum = 0;
for (int j = 0; j < 32; ++j) {
const float t = __fmul_rn(xb[j], inv);
int v = (int) __fadd_rn(t, t >= 0.0f ? 0.5f : -0.5f);
v = v < -127 ? -127 : (v > 127 ? 127 : v);
out[2 + j] = (uint8_t) (int8_t) v;
sum += v;
}
hx[c] = __fmul_rn(s, (float) sum);
}
constexpr int THREADS = 256;
void check(const char* who, void* stream) {
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
std::fprintf(stderr, "%s launch: %s\n", who, cudaGetErrorString(e));
std::exit(1);
}
// Deliberately NOT synchronising for a non-null stream: this is called once per layer from a captured
// graph's worth of work, and `finish()`'s null-stream sync in `s_gemv.cu` is the pattern that made a whole
// round of measurements the driver's cost instead of the kernel's (Memory/ERRORS.md RC-7).
(void) stream;
}
// The previous kernels stay selectable for A/B - `STRATA_OLD_GROUPED=1` in the environment, or
// `moe_grouped_select_old` (the parity test runs both in one process). The environment is read once, on first use;
// the choice is made at each launch, so a captured graph keeps the kernels it was captured with.
std::atomic<int> g_select_old{-1};
// Which kernels the last launch of an entry point used: 1 = new, 0 = previous, -1 = none since the last query.
std::atomic<int> g_last_path{-1};
bool old_kernels() {
const int s = g_select_old.load(std::memory_order_relaxed);
if (s >= 0) return s != 0;
static const bool env = [] {
const char* e = std::getenv("STRATA_OLD_GROUPED");
return e != nullptr && e[0] == '1';
}();
return env;
}
// `STRATA_GROUPED_PAIR_MIN_HITS=N`: the per-hit path keeps the previous one-warp-per-row kernels below N hits of
// capacity. The new per-hit kernels launch half the warps (two rows each), so one hit's gate/up is 80 blocks - fewer
// than an RTX 5090's SMs - and whether that costs time at one to three hits is what `--bench` measures. Bitwise
// the same either way; default 0 (always the new kernels). Ignored while `moe_grouped_select_old` forces a choice.
long long pair_min_hits() {
if (g_select_old.load(std::memory_order_relaxed) >= 0) return 0;
static const long long n = [] {
const char* e = std::getenv("STRATA_GROUPED_PAIR_MIN_HITS");
return e != nullptr ? std::atoll(e) : 0LL;
}();
return n;
}
// The new kernels read the activations as aligned words (`load_x_chunk`), so they need 4-byte aligned activation
// rows - the input's and the intermediate's in `scratch`; the per-hit ones also read a blob's codes as uint2, which
// needs an 8-byte aligned arena and slot size (the grouped kernels above already did). Anything else keeps the
// previous kernels rather than issuing a misaligned load.
bool new_grouped(const void* x_q8_0, const void* scratch) {
const bool fast = !old_kernels() && ((uintptr_t) x_q8_0 & 3) == 0 && ((uintptr_t) scratch & 3) == 0;
g_last_path.store(fast ? 1 : 0, std::memory_order_relaxed);
return fast;
}
bool new_hit(const void* blob_base, long long blob_bytes, const void* x_q8_0, const void* scratch, long long cap) {
const bool fast = new_grouped(x_q8_0, scratch) && ((uintptr_t) blob_base & 7) == 0 && (blob_bytes & 7) == 0 &&
cap >= pair_min_hits();
g_last_path.store(fast ? 1 : 0, std::memory_order_relaxed);
return fast;
}
// The per-hit path's two projections, previous or new kernels (one warp per row, or per pair of rows).
void launch_hit_gu(bool fast, const uint8_t* blob_base, const int32_t* slot_index, long long blob_bytes,
const uint8_t* x_q8_0, const float* x_scales, float* gate_up, long long cap,
const int32_t* d_count, const int32_t* dst_index, int tok_div, cudaStream_t cs) {
const int warps = THREADS / 32;
if (fast) {
const long long pairs = cap * (long long) FF;
gu_pair_kernel<<<(unsigned) ((pairs + warps - 1) / warps), THREADS, 0, cs>>>(
blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, (int) cap, d_count, dst_index, tok_div);
} else {
const long long rows = cap * 2LL * FF;
gu_kernel<<<(unsigned) ((rows + warps - 1) / warps), THREADS, 0, cs>>>(
blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, (int) cap, d_count, dst_index, tok_div);
}
}
void launch_hit_down(bool fast, const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
long long blob_bytes, const uint8_t* h_q8_0, const float* h_scales, float* out, long long cap,
const int32_t* d_count, cudaStream_t cs) {
const int warps = THREADS / 32;
if (fast) {
const long long pairs = cap * (long long) (H / 2);
down_pair_kernel<<<(unsigned) ((pairs + warps - 1) / warps), THREADS, 0, cs>>>(
blob_base, slot_index, dst_index, blob_bytes, h_q8_0, h_scales, out, (int) cap, d_count);
} else {
const long long rows = cap * (long long) H;
down_kernel<<<(unsigned) ((rows + warps - 1) / warps), THREADS, 0, cs>>>(
blob_base, slot_index, dst_index, blob_bytes, h_q8_0, h_scales, out, (int) cap, d_count);
}
}
} // namespace
void moe_grouped_select_old(int old) { g_select_old.store(old < 0 ? -1 : (old != 0 ? 1 : 0)); }
int moe_grouped_last_path() { return g_last_path.exchange(-1, std::memory_order_relaxed); }
uint64_t moe_hit_grouped_scratch_bytes(int64_t n_hits, int64_t n_embd, int64_t n_ff) {
if (n_hits <= 0) return 0;
const uint64_t gu = (uint64_t) n_hits * (uint64_t) (2 * n_ff) * 4;
const uint64_t q8 = (uint64_t) n_hits * (uint64_t) (n_ff / 32) * 34;
// R4.2h: the fp32 scales for the INTERMEDIATE's own quantization, one per 32-element chunk per hit.
const uint64_t hs = (uint64_t) n_hits * (uint64_t) (n_ff / 32) * 4;
const uint64_t xh = (uint64_t) (n_embd / 32) * 4;
return ((gu + 15) & ~15ull) + ((q8 + 15) & ~15ull) + 2 * ((hs + 15) & ~15ull) +
((xh + 15) & ~15ull);
}
void moe_hit_grouped_s2(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
int64_t n_hits, int64_t blob_bytes, const uint8_t* x_q8_0, void* scratch, float* out,
void* stream, const float* x_scales) {
if (n_hits <= 0) return;
cudaStream_t cs = (cudaStream_t) stream;
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, n_hits);
const uint64_t gu_bytes = ((uint64_t) n_hits * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) n_hits * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
// 1. gate + up, one launch for every row of every hit.
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, n_hits, nullptr, nullptr,
0, cs);
check("moe_hit_grouped_s2/gu", stream);
}
// 2+3. silu(gate) * up, then the intermediate's own contract - ONE kernel where two ran. The products
// land in the gate half exactly as `swiglu_kernel` left them (the parity test compares that scratch),
// and the q8_0 blocks are the same bytes `quantize_q8_0` produced. `swilu_kind` 1 is this file's
// `swiglu_kernel` expression (see `strata/kernels/swiglu.cuh`).
// The down weight is Q2_0, whose `vec_dot_type` is Q8_0 - `ggml_mul_mat`'s rule, not a choice. With
// `x_scales` the intermediate takes the CPU's contract (`quantize_q8_0_scaled`), R4.2h.
{
const long long pairs = n_hits * (long long) FF;
#if defined(STRATA_EXPERIMENTAL_SM60)
if (strata::kernels::fuse_decode()) {
if (x_scales != nullptr)
swilu_quantize_q8_0_scaled(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, h_scales, stream);
else
swilu_quantize_q8_0(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, stream);
} else
#endif
{
const unsigned blocks = (unsigned) ((pairs + THREADS - 1) / THREADS);
swiglu_kernel<<<blocks, THREADS, 0, cs>>>(gate_up, pairs);
check("moe_hit_grouped_s2/swiglu", stream);
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, n_hits * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, n_hits * (int64_t) FF, stream);
}
}
// 4. down.
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, n_hits, nullptr, cs);
check("moe_hit_grouped_s2/down", stream);
}
}
namespace {
// Plan v0.3 P4 token graph: which of this layer's routed experts are resident, decided ON THE DEVICE from the
// static residency row, so no host step sits between the ring and the hit kernels. One warp; k <= 32.
__global__ void hit_select_kernel(const int32_t* __restrict__ ids, const int32_t* __restrict__ res_row, int k,
int n_expert, int32_t* __restrict__ slot, int32_t* __restrict__ dst,
int32_t* __restrict__ count) {
const int lane = threadIdx.x;
int s = -1;
if (lane < k) {
const int e = ids[lane];
if (e >= 0 && e < n_expert) s = res_row[e];
}
const unsigned hit = __ballot_sync(0xffffffffu, s >= 0);
if (s >= 0) {
const int at = __popc(hit & ((1u << lane) - 1u));
slot[at] = s;
dst[at] = lane;
}
if (lane == 0) *count = __popc(hit);
}
// Plan v0.3 P6: the same for up to 128 routed entries (a verify window of T tokens x k): four warps, ballots
// compacted in entry order.
__global__ void hit_select_multi_kernel(const int32_t* __restrict__ ids, const int32_t* __restrict__ res_row, int n,
int n_expert, int32_t* __restrict__ slot, int32_t* __restrict__ dst,
int32_t* __restrict__ count) {
__shared__ int warp_count[4];
const int i = threadIdx.x, lane = i & 31, warp = i >> 5;
int s = -1;
if (i < n) {
const int e = ids[i];
if (e >= 0 && e < n_expert) s = res_row[e];
}
const unsigned hit = __ballot_sync(0xffffffffu, s >= 0);
if (lane == 0) warp_count[warp] = __popc(hit);
__syncthreads();
int before = 0;
for (int w = 0; w < warp; ++w) before += warp_count[w];
if (s >= 0) {
const int at = before + __popc(hit & ((1u << lane) - 1u));
slot[at] = s;
dst[at] = i;
}
if (i == 0) *count = warp_count[0] + warp_count[1] + warp_count[2] + warp_count[3];
}
__global__ void add_hits_kernel(float* __restrict__ parts, const float* __restrict__ hit_out,
const int32_t* __restrict__ dst, const int32_t* __restrict__ count, int n_embd) {
const int h = blockIdx.y;
if (h >= *count) return;
const size_t row = (size_t) dst[h] * (size_t) n_embd;
for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n_embd; i += gridDim.x * blockDim.x)
parts[row + i] += hit_out[row + i];
}
} // namespace
void moe_hit_select(const int32_t* ids, const int32_t* res_row, int k, int n_expert, int32_t* slot, int32_t* dst,
int32_t* count, void* stream) {
if (k < 1 || k > 32) { std::fprintf(stderr, "moe_hit_select: k must be 1..32\n"); std::exit(1); }
hit_select_kernel<<<1, 32, 0, (cudaStream_t) stream>>>(ids, res_row, k, n_expert, slot, dst, count);
check("moe_hit_select", stream);
}
void moe_hit_grouped_s2_dev(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
const int32_t* d_count, int64_t cap, int64_t blob_bytes, const uint8_t* x_q8_0,
void* scratch, float* out, void* stream, const float* x_scales) {
if (cap <= 0) return;
cudaStream_t cs = (cudaStream_t) stream;
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, cap);
const uint64_t gu_bytes = ((uint64_t) cap * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, cap, d_count, nullptr, 0,
cs);
check("moe_hit_grouped_s2_dev/gu", stream);
}
// silu + intermediate quantization in one kernel (same full-capacity range as the two kernels it replaces,
// sentinel rows included - the parity test compares that scratch).
{
const long long pairs = cap * (long long) FF;
#if defined(STRATA_EXPERIMENTAL_SM60)
if (strata::kernels::fuse_decode()) {
if (x_scales != nullptr)
swilu_quantize_q8_0_scaled(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, h_scales, stream);
else
swilu_quantize_q8_0(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, stream);
} else
#endif
{
swiglu_kernel<<<(unsigned) ((pairs + THREADS - 1) / THREADS), THREADS, 0, cs>>>(gate_up, pairs);
check("moe_hit_grouped_s2_dev/swiglu", stream);
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap * (int64_t) FF, stream);
}
}
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, cap, d_count, cs);
check("moe_hit_grouped_s2_dev/down", stream);
}
}
void moe_hit_select_multi(const int32_t* ids, const int32_t* res_row, int n, int n_expert, int32_t* slot, int32_t* dst,
int32_t* count, void* stream) {
if (n < 1 || n > 128) { std::fprintf(stderr, "moe_hit_select_multi: n must be 1..128\n"); std::exit(1); }
hit_select_multi_kernel<<<1, 128, 0, (cudaStream_t) stream>>>(ids, res_row, n, n_expert, slot, dst, count);
check("moe_hit_select_multi", stream);
}
void moe_hit_grouped_s2_multi(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
const int32_t* d_count, int64_t cap, int64_t blob_bytes, const uint8_t* x_q8_0,
const float* x_scales, int k_per_token, void* scratch, float* out, void* stream) {
if (cap <= 0) return;
cudaStream_t cs = (cudaStream_t) stream;
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, cap);
const uint64_t gu_bytes = ((uint64_t) cap * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, cap, d_count, dst_index,
k_per_token, cs);
check("moe_hit_grouped_s2_multi/gu", stream);
}
{
const long long pairs = cap * (long long) FF;
#if defined(STRATA_EXPERIMENTAL_SM60)
if (strata::kernels::fuse_decode()) {
if (x_scales != nullptr)
swilu_quantize_q8_0_scaled(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, h_scales, stream);
else
swilu_quantize_q8_0(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, stream);
} else
#endif
{
swiglu_kernel<<<(unsigned) ((pairs + THREADS - 1) / THREADS), THREADS, 0, cs>>>(gate_up, pairs);
check("moe_hit_grouped_s2_multi/swiglu", stream);
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap * (int64_t) FF, stream);
}
}
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, cap, d_count, cs);
check("moe_hit_grouped_s2_multi/down", stream);
}
}
namespace {
// Plan v0.3 P6: experts GROUPED - one blob pointer per group (a VRAM slot or a mapped host blob read over PCIe),
// every entry of the group (a token routed to that expert) computed from ONE read of each row. Per entry the
// arithmetic is `row_dot_s2_q8`'s, chunk by chunk in the same lane order, so every entry is bitwise the per-entry
// hit kernel's.
constexpr int GU_CHUNKS = (H / 32 + 31) / 32; // 3: chunks of a gate/up row per lane (80 chunks / 32 lanes)
constexpr int GMAX = 8; // entries per group (tokens routed to one expert in a window)
// a group holds one entry per token of the window routed to its expert, and a kernel keeps at most GMAX of them
// (`min(..., GMAX)`): a longer group would drop entries without a word. A window of 9..16 rows can route more tokens
// than that to one expert, so `moe_grouped_s2` takes the window's largest group and runs the kernel again with
// SKIP = GMAX over the entries from the GMAX-th on (the staging is GMAX entries wide in shared memory: 51 KB at 16
// would not compile on CUDA, whose static limit is 48 KB). Every entry's output is its own dot products, so the
// passes are bitwise the one wide pass; SKIP = 0 is the kernel as it was.
constexpr int kGroupPasses = (kVerifyMaxT + GMAX - 1) / GMAX;
static_assert(kGroupPasses * GMAX >= kVerifyMaxT, "a verify window's group can exceed the passes' entries");
constexpr int GU_ROWS = 32; // gate/up rows per block: 4 per warp
constexpr int D_ROWS = 64; // down rows per block: 8 per warp
// One activation chunk's contribution, `row_dot_s2_q8`'s inner body with the 32 int8 of the chunk already in
// aligned words: same dp4a sequence, same float expression, so the result is bitwise the per-entry kernel's.
__device__ __forceinline__ float chunk_dot(uint2 cb, const int* xw, float dw, float dx) {
const uint8_t* cbytes = (const uint8_t*) &cb;
int s = 0, hx = 0;
const int ones = 0x01010101;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const unsigned cbyte = cbytes[j];
const int cw = (int) ((cbyte & 3u) | (((cbyte >> 2) & 3u) << 8) | (((cbyte >> 4) & 3u) << 16) |
(((cbyte >> 6) & 3u) << 24));
s = STRATA_DP4A(cw, xw[j], s);
hx = STRATA_DP4A(ones, xw[j], hx);
}
return dw * dx * (float) (s - hx);
}
// Gate/up: a block = GU_ROWS rows of ONE group. The group's activations (each entry's token row of x_q8_0 and
// its fp32 scales) are staged once into shared memory as aligned words; each warp then walks its rows, loading
// each lane's code chunks once and dotting them with every entry.
template <int SKIP>
__global__ void __launch_bounds__(256) gu_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
const int32_t* __restrict__ ent_tok,
const uint8_t* __restrict__ x_q8_0,
const float* __restrict__ x_scales,
float* __restrict__ gate_up, int cap_entries) {
__shared__ int xs_q[GMAX][H / 4]; // the entries' int8 activations as words (2560 B each)
__shared__ float xs_d[GMAX][H / 32];
const int g = blockIdx.y;
if (g >= *n_groups) return;
const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (H / 32); i += blockDim.x) {
const int k = i / (H / 32), c = i - k * (H / 32);
const uint8_t* xb = x_q8_0 + (size_t) ent_tok[e0 + k] * (size_t) (H / 32) * 34 + (size_t) c * 34;
xs_d[k][c] = x_scales ? x_scales[(size_t) ent_tok[e0 + k] * (H / 32) + c] : f16_at(xb);
const int8_t* q = (const int8_t*) (xb + 2);
#pragma unroll
for (int w = 0; w < 8; ++w) {
int v;
memcpy(&v, q + 4 * w, 4);
xs_q[k][c * 8 + w] = v;
}
}
__syncthreads();
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = blockIdx.x * GU_ROWS;
for (int rr = warp; rr < GU_ROWS; rr += 8) {
const int i = row0 + rr;
const uint8_t* codes = blob + (size_t) i * ROW_GU;
const uint8_t* scales = blob + O_GU_SCALES + (size_t) i * SC_GU * 2;
uint2 cb[GU_CHUNKS];
float dw[GU_CHUNKS];
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c < H / 32) {
cb[q] = *(const uint2*) (codes + (size_t) c * 8);
dw[q] = f16_at(scales + (size_t) (c >> 1) * 2);
}
}
for (int k = 0; k < ne; ++k) {
float acc = 0.0f;
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c >= H / 32) break;
acc += chunk_dot(cb[q], &xs_q[k][c * 8], dw[q], xs_d[k][c]);
}
const float sum = warp_sum(acc);
if (lane == 0) {
const int e = e0 + k, r = i >> 1;
const size_t base = (i & 1) ? ((size_t) cap_entries * FF + (size_t) e * FF) : ((size_t) e * FF);
gate_up[base + (size_t) r] = sum;
}
}
}
}
// Down: a block = D_ROWS rows of ONE group; the entries' quantized intermediates staged once. A down row is 20
// chunks, so lanes 0..19 each hold one chunk, as in the per-entry kernel.
template <int SKIP>
__global__ void __launch_bounds__(256) down_grouped_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
const int32_t* __restrict__ ent_dst,
const uint8_t* __restrict__ h_q8_0,
const float* __restrict__ h_scales,
float* __restrict__ out) {
__shared__ int hs_q[GMAX][FF / 4];
__shared__ float hs_d[GMAX][FF / 32];
const int g = blockIdx.y;
if (g >= *n_groups) return;
const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (FF / 32); i += blockDim.x) {
const int k = i / (FF / 32), c = i - k * (FF / 32);
const uint8_t* xb = h_q8_0 + (size_t) (e0 + k) * (size_t) (FF / 32) * 34 + (size_t) c * 34;
hs_d[k][c] = h_scales ? h_scales[(size_t) (e0 + k) * (FF / 32) + c] : f16_at(xb);
const int8_t* q = (const int8_t*) (xb + 2);
#pragma unroll
for (int w = 0; w < 8; ++w) {
int v;
memcpy(&v, q + 4 * w, 4);
hs_q[k][c * 8 + w] = v;
}
}
__syncthreads();
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = blockIdx.x * D_ROWS;
for (int rr = warp; rr < D_ROWS; rr += 8) {
const int r = row0 + rr;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
const int c = lane;
uint2 cb = make_uint2(0, 0);
float dw = 0.0f;
if (c < FF / 32) {
cb = *(const uint2*) (codes + (size_t) c * 8);
dw = f16_at(scales + (size_t) (c >> 1) * 2);
}
for (int k = 0; k < ne; ++k) {
float acc = 0.0f;
if (c < FF / 32) acc += chunk_dot(cb, &hs_q[k][c * 8], dw, hs_d[k][c]);
const float sum = warp_sum(acc);
if (lane == 0) out[(size_t) ent_dst[e0 + k] * H + r] = sum;
}
}
}
// ---- The grouped kernels with the staged activations laid out for the reads.
//
// **THE LAYOUT ABOVE IS AN 8-WAY BANK CONFLICT.** `xs_q[k][c * 8 + j]` puts lane `L`'s word `j` at `8L + j`, so
// lanes `L, L + 4, L + 8, ...` share a bank on every one of the chunk's eight reads; and `chunk_dot` recomputes
// `hx`, which depends on the entry alone, for every row. Here the staging writes, once per (entry, chunk):
//
// xs_w[j][k * NC + c] word j of the regrouped chunk (`load_x_chunk`), so lanes read consecutive words
// xs_dh[k * NC + c] (dx, hx) - one 64-bit read per chunk instead of a float and eight `dp4a`
//
// Word-major, the staging writes are consecutive as well (the flat index `i` IS `k * NC + c`), so neither side
// conflicts and no padding is needed. Each warp takes its rows TWO at a time (adjacent rows: gate `r` and up `r`,
// or down rows `r, r + 1`), so every shared read serves both; the codes of both are expanded once per row, not
// once per entry. Per (row, entry) the lane's chunks, their order, the float expression and the shuffle
// reduction are the kernels' above, so every output is bitwise theirs.
// Stages chunk `i = k * NC + c` of a group: its regrouped words at `xs_w[j * STRIDE + i]` and (dx, hx) at `xs_dh[i]`.
template <int STRIDE>
__device__ __forceinline__ void stage_chunk(const uint8_t* __restrict__ xb, float dx, int* xs_w, int2* xs_dh, int i) {
int X[8];
const int hx = load_x_chunk(xb, X);
#pragma unroll
for (int j = 0; j < 8; ++j) xs_w[j * STRIDE + i] = X[j];
xs_dh[i] = make_int2(__float_as_int(dx), hx);
}
template <int SKIP>
__global__ void __launch_bounds__(256) gu_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
const int32_t* __restrict__ ent_tok,
const uint8_t* __restrict__ x_q8_0,
const float* __restrict__ x_scales,
float* __restrict__ gate_up, int cap_entries) {
constexpr int NC = H / 32;
__shared__ int xs_w[8 * GMAX * NC]; // 20 KB: word j of entry k's chunk c at [j][k * NC + c]
__shared__ int2 xs_dh[GMAX * NC]; // 5 KB: (dx as bits, hx)
const int g = blockIdx.y;
if (g >= *n_groups) return;
const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += blockDim.x) {
const int k = i / NC, c = i - k * NC;
const int tok = ent_tok[e0 + k];
const uint8_t* xb = x_q8_0 + (size_t) tok * (size_t) NC * 34 + (size_t) c * 34;
const float dx = x_scales ? x_scales[(size_t) tok * NC + c] : f16_ld(xb);
stage_chunk<GMAX * NC>(xb, dx, xs_w, xs_dh, i);
}
__syncthreads();
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = blockIdx.x * GU_ROWS; // even: a pair is always (gate r, up r)
for (int pp = warp; pp < GU_ROWS / 2; pp += 8) {
const int i = row0 + 2 * pp;
const uint8_t* codes = blob + (size_t) i * ROW_GU;
const uint8_t* scales = blob + O_GU_SCALES + (size_t) i * SC_GU * 2;
int m0[GU_CHUNKS][8], m1[GU_CHUNKS][8];
float dw0[GU_CHUNKS], dw1[GU_CHUNKS];
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c < NC) {
expand_codes(*(const uint2*) (codes + (size_t) c * 8), m0[q]);
expand_codes(*(const uint2*) (codes + ROW_GU + (size_t) c * 8), m1[q]);
dw0[q] = f16_ld(scales + (size_t) (c >> 1) * 2);
dw1[q] = f16_ld(scales + SC_GU * 2 + (size_t) (c >> 1) * 2);
}
}
for (int k = 0; k < ne; ++k) {
float acc0 = 0.0f, acc1 = 0.0f;
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c >= NC) break;
const int at = k * NC + c;
int X[8];
#pragma unroll
for (int j = 0; j < 8; ++j) X[j] = xs_w[j * (GMAX * NC) + at];
const int2 dh = xs_dh[at];
const float dx = __int_as_float(dh.x);
acc0 += dw0[q] * dx * (float) (chunk_s(m0[q], X) - dh.y);
acc1 += dw1[q] * dx * (float) (chunk_s(m1[q], X) - dh.y);
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane == 0) {
const int e = e0 + k, r = i >> 1;
gate_up[(size_t) e * FF + (size_t) r] = s0;
gate_up[(size_t) cap_entries * FF + (size_t) e * FF + (size_t) r] = s1;
}
}
}
}
template <int SKIP>
__global__ void __launch_bounds__(256) down_grouped_t_kernel(const unsigned long long* __restrict__ grp_ptr,
const int32_t* __restrict__ grp_start,
const int32_t* __restrict__ n_groups,
const int32_t* __restrict__ ent_dst,
const uint8_t* __restrict__ h_q8_0,
const float* __restrict__ h_scales,
float* __restrict__ out) {
constexpr int NC = FF / 32;
__shared__ int hs_w[8 * GMAX * NC];
__shared__ int2 hs_dh[GMAX * NC];
const int g = blockIdx.y;
if (g >= *n_groups) return;
const int e0 = grp_start[g] + SKIP, ne = min(grp_start[g + 1] - e0, GMAX);
if (SKIP > 0 && ne <= 0) return;
const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += blockDim.x) {
const int k = i / NC, c = i - k * NC;
const uint8_t* xb = h_q8_0 + (size_t) (e0 + k) * (size_t) NC * 34 + (size_t) c * 34;
const float dx = h_scales ? h_scales[(size_t) (e0 + k) * NC + c] : f16_ld(xb);
stage_chunk<GMAX * NC>(xb, dx, hs_w, hs_dh, i);
}
__syncthreads();
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = blockIdx.x * D_ROWS;
for (int pp = warp; pp < D_ROWS / 2; pp += 8) {
const int r = row0 + 2 * pp;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
const int c = lane; // lanes 0..19 hold one chunk each, as above
int m0[8], m1[8];
float dw0 = 0.0f, dw1 = 0.0f;
if (c < NC) {
expand_codes(*(const uint2*) (codes + (size_t) c * 8), m0);
expand_codes(*(const uint2*) (codes + ROW_D + (size_t) c * 8), m1);
dw0 = f16_ld(scales + (size_t) (c >> 1) * 2);
dw1 = f16_ld(scales + SC_D * 2 + (size_t) (c >> 1) * 2);
}
for (int k = 0; k < ne; ++k) {
float acc0 = 0.0f, acc1 = 0.0f;
if (c < NC) {
const int at = k * NC + c;
int X[8];
#pragma unroll
for (int j = 0; j < 8; ++j) X[j] = hs_w[j * (GMAX * NC) + at];
const int2 dh = hs_dh[at];
const float dx = __int_as_float(dh.x);
acc0 += dw0 * dx * (float) (chunk_s(m0, X) - dh.y);
acc1 += dw1 * dx * (float) (chunk_s(m1, X) - dh.y);
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane == 0) {
const size_t o = (size_t) ent_dst[e0 + k] * H + (size_t) r;
out[o] = s0;
out[o + 1] = s1;
}
}
}
}
} // namespace
namespace {
// Plan v0.3 P6: groups built on the device when every expert is resident at `base + id * blob` (the MTP layer):
// one block of 128 threads, groups in first-appearance order, entries of a group in routing order.
__global__ void group_resident_kernel(const int32_t* __restrict__ ids, int n, int k_per_tok, const uint8_t* base,
long long blob, unsigned long long* __restrict__ grp_ptr,
int32_t* __restrict__ grp_start, int32_t* __restrict__ counts,
int32_t* __restrict__ ent_dst, int32_t* __restrict__ ent_tok) {
__shared__ int e_s[128], first_s[128], size_s[128], gidx_s[128], gstart_s[129];
const int i = threadIdx.x;
const int e = i < n ? ids[i] : -1;
e_s[i] = e;
__syncthreads();
int first = i, rank = 0, size = 0;
if (i < n) {
for (int j = 0; j < i; ++j)
if (e_s[j] == e) { if (first == i) first = j; ++rank; }
if (first == i)
for (int j = i; j < n; ++j) size += e_s[j] == e;
}
first_s[i] = first;
size_s[i] = (i < n && first == i) ? size : 0;
__syncthreads();
if (i == 0) {
int gi = 0, acc = 0;
for (int j = 0; j < n; ++j)
if (first_s[j] == j) {
gidx_s[j] = gi;
gstart_s[gi] = acc;
grp_ptr[gi] = (unsigned long long) (base + (size_t) e_s[j] * (size_t) blob);
grp_start[gi] = acc;
acc += size_s[j];
++gi;
}
grp_start[gi] = acc;
counts[0] = gi;
counts[1] = acc;
}
__syncthreads();
if (i < n) {
const int at = gstart_s[gidx_s[first]] + rank;
ent_dst[at] = i;
ent_tok[at] = i / k_per_tok;
}
}
} // namespace
void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t* base, int64_t blob,
unsigned long long* grp_ptr, int32_t* grp_start, int32_t* counts, int32_t* ent_dst,
int32_t* ent_tok, void* stream) {
if (n < 1 || n > 128) { std::fprintf(stderr, "moe_group_resident: n must be 1..128\n"); std::exit(1); }
group_resident_kernel<<<1, 128, 0, (cudaStream_t) stream>>>(ids, n, k_per_tok, base, (long long) blob, grp_ptr,
grp_start, counts, ent_dst, ent_tok);
check("moe_group_resident", stream);
}
static void grouped_s2_impl(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
int max_group) {
if (cap_groups <= 0 || cap_entries <= 0) return;
if (max_group < 1 || max_group > kGroupPasses * GMAX) {
std::fprintf(stderr, "moe_grouped_s2: a group of %d entries, at most %d\n", max_group, kGroupPasses * GMAX);
std::exit(1);
}
// a group of more than GMAX entries (a window of 9..16 rows, wide builds only): the entries from the GMAX-th on, too
constexpr bool kWide = kGroupPasses > 1;
const bool second = kWide && max_group > GMAX;
cudaStream_t cs = (cudaStream_t) stream;
const uint64_t gu_bytes = ((uint64_t) cap_entries * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap_entries * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
const bool fast = new_grouped(x_q8_0, scratch);
{
const dim3 grid((unsigned) (2 * FF / GU_ROWS), (unsigned) cap_groups);
if (fast) {
gu_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
gate_up, (int) cap_entries);
if constexpr (kWide)
if (second)
gu_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
gate_up, (int) cap_entries);
} else {
gu_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales, gate_up,
(int) cap_entries);
if constexpr (kWide)
if (second)
gu_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_tok, x_q8_0, x_scales,
gate_up, (int) cap_entries);
}
check("moe_grouped_s2/gu", stream);
}
// #783 PR-k (stuchapin909): SwiGLU and the scaled Q8_0 quantize of its result in one kernel (gate_up still gets the
// activation written back, so the parity checks read the same buffer); STRATA_S2_SWIGLU_Q8=0 keeps the two launches
static const bool swiglu_q8 = [] {
const char* e = std::getenv("STRATA_S2_SWIGLU_Q8");
return e == nullptr || e[0] != '0';
}();
static_assert(FF % 32 == 0, "the fused SwiGLU + Q8_0 kernel works in whole 32-value blocks");
if (fast && x_scales != nullptr && swiglu_q8) {
const long long pairs = cap_entries * (long long) FF;
const long long n_blocks = pairs >> 5;
const int warps = THREADS / 32;
swiglu_quantize_q8_0_scaled_kernel<<<(unsigned) ((n_blocks + warps - 1) / warps), THREADS, 0, cs>>>(
gate_up, pairs, h_q8_0, h_scales);
check("moe_grouped_s2/swiglu_q8", stream);
} else {
const long long pairs = cap_entries * (long long) FF;
#if defined(STRATA_EXPERIMENTAL_SM60)
if (strata::kernels::fuse_decode()) {
if (x_scales != nullptr)
swilu_quantize_q8_0_scaled(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, h_scales, stream);
else
swilu_quantize_q8_0(gate_up, gate_up + pairs, gate_up, pairs, 1, h_q8_0, stream);
} else
#endif
{
swiglu_kernel<<<(unsigned) ((pairs + THREADS - 1) / THREADS), THREADS, 0, cs>>>(gate_up, pairs);
check("moe_grouped_s2/swiglu", stream);
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap_entries * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap_entries * (int64_t) FF, stream);
}
}
{
const dim3 grid((unsigned) (H / D_ROWS), (unsigned) cap_groups);
const float* hs = x_scales != nullptr ? h_scales : nullptr;
if (fast) {
down_grouped_t_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
if constexpr (kWide)
if (second) down_grouped_t_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
} else {
down_grouped_kernel<0><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
if constexpr (kWide)
if (second) down_grouped_kernel<GMAX><<<grid, 256, 0, cs>>>(grp_ptr, grp_start, n_groups, ent_dst, h_q8_0, hs, out);
}
check("moe_grouped_s2/down", stream);
}
}
void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {
grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,
GMAX);
}
#if STRATA_VERIFY_MAX_T > 8
void moe_grouped_s2_wide(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream,
int max_group) {
grouped_s2_impl(grp_ptr, grp_start, n_groups, ent_dst, ent_tok, cap_groups, cap_entries, x_q8_0, x_scales, scratch, out, stream,
max_group);
}
#endif
void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const int32_t* count, int64_t cap,
int64_t n_embd, void* stream) {
if (cap <= 0) return;
const dim3 grid((unsigned) ((n_embd + 255) / 256 < 8 ? (n_embd + 255) / 256 : 8), (unsigned) cap);
add_hits_kernel<<<grid, 256, 0, (cudaStream_t) stream>>>(parts, hit_out, dst, count, (int) n_embd);
check("moe_hit_add", stream);
}
void moe_hit_grouped_s2_cpu_order(const uint8_t* blob_base, const int32_t* slot_index,
const int32_t* dst_index, int64_t n_hits, int64_t blob_bytes,
const uint8_t* x_q8_0, void* scratch, float* out, void* stream,
const float* x_scales, float* gate_up_trace) {
if (n_hits <= 0) return;
if (x_scales == nullptr) {
std::fprintf(stderr, "moe_hit_grouped_s2_cpu_order requires fp32 activation scales\n");
std::exit(1);
}
cudaStream_t cs = (cudaStream_t) stream;
const uint64_t gu_bytes = ((uint64_t) n_hits * 2 * FF * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) n_hits * (FF / 32) * 34 + 15) & ~15ull;
const uint64_t scale_bytes = ((uint64_t) n_hits * (FF / 32) * 4 + 15) & ~15ull;
float* gu = (float*) scratch;
uint8_t* hq = (uint8_t*) scratch + gu_bytes;
float* hs = (float*) (hq + q8_bytes);
float* hh = (float*) ((uint8_t*) hs + scale_bytes);
float* xh = (float*) ((uint8_t*) hh + scale_bytes);
activation_correction_kernel<<<(H / 32 + THREADS - 1) / THREADS, THREADS, 0, cs>>>(
x_q8_0, x_scales, xh, H / 32);
check("cpu_order/input_correction", stream);
const int rows_per_block = THREADS / 8;
cpu_order_projection_kernel<false><<<(unsigned) ((n_hits * 2 * FF + rows_per_block - 1) / rows_per_block),
THREADS, 0, cs>>>(
blob_base, slot_index, dst_index, blob_bytes, x_q8_0, x_scales, xh, gu, (int) n_hits);
check("cpu_order/gate_up", stream);
if (gate_up_trace != nullptr &&
cudaMemcpyAsync(gate_up_trace, gu, (size_t) n_hits * 2 * FF * sizeof(float),
cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
std::fprintf(stderr, "cpu_order/gate_up_trace copy failed\n");
std::exit(1);
}
cpu_order_swiglu_kernel<<<(unsigned) ((n_hits * FF + THREADS - 1) / THREADS), THREADS, 0, cs>>>(
gu, (int) n_hits * FF);
check("cpu_order/swiglu", stream);
cpu_order_quantize_kernel<<<(unsigned) ((n_hits * (FF / 32) + THREADS - 1) / THREADS), THREADS, 0, cs>>>(
gu, hq, hs, hh, (int) n_hits * (FF / 32));
check("cpu_order/intermediate_quantize", stream);
cpu_order_projection_kernel<true><<<(unsigned) ((n_hits * H + rows_per_block - 1) / rows_per_block),
THREADS, 0, cs>>>(
blob_base, slot_index, dst_index, blob_bytes, hq, hs, hh, out, (int) n_hits);
check("cpu_order/down", stream);
}
} // namespace strata::kernels