Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/cuda/s2_expert_grouped.cu - R4's grouped GPU expert. Read the header first.
//
// THE ARITHMETIC IS `src/kernels/cpu/expert.cpp`'s, over the same bytes, one warp per output row.
//
// sum_j (c_j - 1) * d_w * xhat_j * d_x == d_w * d_x * ( sum_j c_j*xhat_j - sum_j xhat_j )
//
// Both sums are INT8 x INT8, so both are `__dp4a` - four multiply-accumulates per instruction, exact in
// integer and with no dequantization inside the loop. The weight scale `d_w` is fp16 (one per 64 elements)
// and the activation scale `d_x` is fp16 (one per 32 elements), so the float work is one multiply-add per
// 32-element chunk rather than one per element.
//
// **THE SUMMATION ORDER IS NOT THE CPU's AND CANNOT BE.** Lane `L` takes chunks `L, L+32, ...` and the
// partials are reduced through shuffle; the CPU walks every chunk in order with its own accumulator shape.
// The two agree to float rounding and not to the bit, which is the same contract `s_gemv_parity` carries for
// the same reason. `bench/micro/moe_hit_parity.cu` is the check.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include "strata/core/geometry.hpp"
#include <dpct/dpct.hpp>
#include "strata/sycl_math.hpp"
#include "strata/sycl_queue.hpp"
#include "strata/kernels/s2_expert_grouped.hpp"
#include "strata/kernels/dp4a.hpp"
#include "strata/kernels/f16_bits.hpp"
#include "strata/kernels/quantize_act.hpp"
#include "strata/kernels/verify_kernels.hpp"
#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
namespace strata::kernels {
namespace {
// THE BLOB'S OWN GEOMETRY, from `include/strata/kernels/cpu/expert.hpp`. Restated as literals because that
// header is the CPU path's and this file must not silently follow it if the two ever disagree: the sizes below
// are what the CPU kernel's indexing computes, and `moe_hit_parity` compares the two end to end.
constexpr int H = 2560;
constexpr int FF = 640;
static_assert(H == strata::core::kCompiledGeometry.n_embd && FF == strata::core::kCompiledGeometry.n_ff, "s2_expert_grouped is compiled for core/geometry.hpp");
constexpr int QK = 64; // Q2_0's group: one fp16 scale per 64 weights
constexpr int ROW_GU = H / 4; // 640 B of codes per gate/up row (2 bits per element)
constexpr int ROW_D = FF / 4; // 160 B per down row
constexpr int SC_GU = H / QK; // 40 fp16 scales per gate/up row
constexpr int SC_D = FF / QK; // 10 per down row
constexpr size_t O_D_CODES = (size_t) 2 * FF * ROW_GU;
constexpr size_t O_GU_SCALES = O_D_CODES + (size_t) H * ROW_D;
constexpr size_t O_D_SCALES = O_GU_SCALES + (size_t) 2 * FF * SC_GU * 2;
__dpct_inline__ float f16_at(const uint8_t *p) {
return sycl::vec<sycl::half, 1>(sycl::bit_cast<sycl::half, unsigned short>(
(uint16_t)(p[0] | (p[1] << 8))))
.convert<float, sycl::rounding_mode::automatic>()[0];
}
/// **ONE S2 ROW AGAINST A Q8_0 ACTIVATION, WARP-WIDE.** Every lane accumulates its own float partial over a
/// strided set of 32-element chunks and the caller reduces; `chunk` indices are absolute so the caller can
/// start the lane at any offset.
///
/// Returns the lane's partial. `codes` is `n_in/4` bytes and `scales` `n_in/QK` fp16, both for THIS row.
///
/// **`x_scales` IS R4.2h AND IT IS OPTIONAL ON PURPOSE.** When it is non-null the activation's multiplier
/// comes from an fp32 array instead of the block's fp16 `d`. The CPU pool multiplies by the fp32
/// `ActQ::scale` (`cpu/expert.cpp:92`), and the two disagreed by **4.761e-04 relative on 80 of 80 chunks**
/// (`bench/micro/act_quant_parity.cu`), which is what made a cache hit compute a different expert from a
/// cache miss. Null keeps the previous fp16 behaviour **exactly**, so `moe_hit_parity` - which passes no
/// scales - still measures the kernel it always measured.
__dpct_inline__ float
row_dot_s2_q8(const uint8_t *__restrict__ codes,
const uint8_t *__restrict__ scales,
const uint8_t *__restrict__ x_q8_0, int n_chunks, int lane,
const float *__restrict__ x_scales = nullptr) {
float acc = 0.0f;
for (int c = lane; c < n_chunks; c += 32) {
const uint8_t* cb = codes + (size_t) c * 8; // 8 code bytes = 32 elements
const uint8_t* xb = x_q8_0 + (size_t) c * 34; // one block_q8_0
const float dx = x_scales ? x_scales[c] : f16_at(xb);
const int8_t* xq = (const int8_t*) (xb + 2);
// ---- THE CODES, EXPANDED TO ONE BYTE PER ELEMENT, THEN FOUR AT A TIME INTO `dp4a`.
//
// The packed form is LSB-first: element 4j+k is bits [2k, 2k+2) of code byte j. `dp4a` needs both
// operands as packed int8, so each code byte becomes a word whose four bytes are its four 2-bit
// fields - which is exactly `(c & 3) | ((c>>2)&3)<<8 | ((c>>4)&3)<<16 | ((c>>6)&3)<<24`.
int s = 0; // sum of code * x
int hx = 0; // sum of x - the weight-independent term, as ones * x
const int ones = 0x01010101;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const unsigned cbyte = cb[j];
const int cw = (int) ((cbyte & 3u) | (((cbyte >> 2) & 3u) << 8) | (((cbyte >> 4) & 3u) << 16) |
(((cbyte >> 6) & 3u) << 24));
// **`memcpy`, NOT A CAST.** A `block_q8_0` is 34 BYTES - two of header then 32 of int8 - so the
// activation data at offset 2 is never 4-byte aligned, and `*(const int*)(xq + 4*j)` faults with
// "misaligned address". It faulted exactly that way on this kernel's first run. `memcpy` of a
// known 4 bytes compiles to whatever load is legal for the alignment, which is the point of using
// it rather than reasoning about which cast happens to work.
int xw;
memcpy(&xw, xq + 4 * j, 4);
s = strata::dp4a(cw, xw, s);
hx = strata::dp4a(ones, xw, hx);
}
// One weight scale per 64 elements, so per TWO 32-element chunks.
const float dw = f16_at(scales + (size_t) (c >> 1) * 2);
acc += dw * dx * (float) (s - hx);
}
return acc;
}
__dpct_inline__ float warp_sum(float v) {
#pragma unroll
/*
DPCT1108: '__shfl_down_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
/*
DPCT1121: Make sure that the "v" which is used in the SYCL group
function/algorithm is initialized.
*/
for (int off = 16; off > 0; off >>= 1) v +=
dpct::experimental::shift_sub_group_left(
0xFFFFFFFFu, sycl::ext::oneapi::this_work_item::get_sub_group(), v,
off);
return v;
}
/// GATE AND UP, ONE WARP PER ROW.
///
/// **THE ROWS ARE INTERLEAVED AND THE FIRST VERSION OF THIS GOT IT WRONG.** In the blob, row-SLOT `i` of the
/// `2*FF` gate/up rows is gate row `i/2` when `i` is even and up row `(i-1)/2` when it is odd - that is what
/// `O_GU_CODES + (2r)*ROW_GU` and `+ (2r+1)*ROW_GU` mean in `expert.cpp`. This kernel decoded the correct
/// code row for slot `i` and then wrote it to output slot `i` of a layout whose first `FF` entries are gate and
/// whose last `FF` are up, which pairs `silu(gate[r]) * up[r]` with the WRONG `up` for every `r`. It produced
/// finite, plausible numbers. `moe_hit_parity` caught it on the first run, at worst relative error 2.2e+03.
///
/// So slot `i` is DECODED from row-slot `i` and WRITTEN to the output slot its parity says it belongs to.
__dpct_inline__ void
gu_kernel(const uint8_t *__restrict__ blob_base,
const int32_t *__restrict__ slot_index, long long blob_bytes,
const uint8_t *__restrict__ x_q8_0,
const float *__restrict__ x_scales, float *__restrict__ gate_up,
int n_hits, const int32_t *__restrict__ d_count = nullptr,
const int32_t *__restrict__ dst_index = nullptr, int tok_div = 0) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int warps_per_block = (int)(item_ct1.get_local_range(2) >> 5);
const long long slot = (long long)item_ct1.get_group(2) * warps_per_block +
(item_ct1.get_local_id(2) >> 5);
const long long rows_per_hit = 2LL * FF;
const long long total = (long long) n_hits * rows_per_hit;
if (slot >= total) return;
const int h = (int) (slot / rows_per_hit);
if (d_count != nullptr && h >= *d_count) return; // token graph: capacity layout, device count
const int i = (int) (slot % rows_per_hit);
const int lane = item_ct1.get_local_id(2) & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
if (tok_div > 0) { // plan v0.3 P6 verify window: each hit reads its own token's activation
const int tok = dst_index[h] / tok_div;
x_q8_0 += (size_t) tok * (size_t) (H / 32) * 34;
if (x_scales != nullptr) x_scales += (size_t) tok * (size_t) (H / 32);
}
const float acc = row_dot_s2_q8(blob + (size_t) i * ROW_GU,
blob + O_GU_SCALES + (size_t) i * SC_GU * 2, x_q8_0, H / 32, lane, x_scales);
const float s = warp_sum(acc);
if (lane != 0) return;
// ---- THE OUTPUT LAYOUT IS GATE-MAJOR, AND THAT IS NOT COSMETIC.
//
// The first version wrote `gate_up[h*2FF + {0..FF-1}] = gate` and `[h*2FF + FF..] = up`, i.e. a per-hit
// [gate | up] pair, and then called the shared `swiglu_kernel` and `quantize_q8_0` over the whole buffer.
// Both of those walk a CONTIGUOUS range, so with more than one hit they read hit 0's UP rows where they
// wanted hit 1's GATE rows - finite numbers, wrong expert. With one hit it would have passed.
//
// Gate-major removes the mismatch instead of adding a stride to two other kernels: every hit's gate rows
// are contiguous from 0, every hit's up rows are contiguous from `n_hits * FF`, and the swiglu's output
// lands in the first `n_hits * FF` floats exactly where `quantize_q8_0` reads it.
const int r = i >> 1;
const size_t base = (i & 1) ? ((size_t) n_hits * FF + (size_t) h * FF) : ((size_t) h * FF);
gate_up[base + (size_t) r] = s;
}
/// `silu(gate) * up`, in place, over a GATE-MAJOR buffer: `[0, n_pairs)` is every hit's gate and
/// `[n_pairs, 2*n_pairs)` is every hit's up, so hit `h`'s row `r` meets itself at `h*FF + r`.
///
/// SiLU on the GATE and multiplied by up - the reading `docs/semantics.md` records, and the one that is wrong
/// the other way round in a way that still produces a finite number.
__dpct_inline__ void swiglu_kernel(float *__restrict__ gate_up,
long long n_pairs) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const long long i =
(long long)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (i >= n_pairs) return;
const float g = gate_up[i];
const float u = gate_up[n_pairs + i];
gate_up[i] = (g / (1.0f + sycl::native::exp(-g))) * u;
}
__dpct_inline__ void swiglu_quantize_q8_0_scaled_kernel(
float *__restrict__ gate_up, long long n_pairs,
uint8_t *__restrict__ blocks, float *__restrict__ scales) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const long long n_blocks = n_pairs >> 5;
const long long b =
((long long)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2)) >>
5;
if (b >= n_blocks) return;
const int lane = item_ct1.get_local_id(2) & 31;
const long long idx = b * 32 + lane;
const float g = gate_up[idx];
const float u = gate_up[n_pairs + idx];
const float xv = (g / (1.0f + sycl::native::exp(-g))) * u;
gate_up[idx] = xv;
uint8_t* out = blocks + b * 34;
float amax = sycl::fabs(xv);
#pragma unroll
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
for (int off = 16; off > 0; off >>= 1) amax =
sycl::fmax(amax, dpct::experimental::permute_sub_group_by_xor(
0xffffffffu,
sycl::ext::oneapi::this_work_item::get_sub_group(),
amax, off));
const float s = amax > 0.f ? amax / 127.f : 0.f;
const float inv = s > 0.f ? 1.f / s : 0.f;
if (lane == 0) {
scales[b] = s;
const uint16_t d16bits = f16_from_f32(s);
out[0] = (uint8_t) (d16bits & 0xFF);
out[1] = (uint8_t) (d16bits >> 8);
}
const float t = xv * inv;
const float r = t + (t >= 0.f ? 0.5f : -0.5f);
int v = (int) r;
v = v < -127 ? -127 : (v > 127 ? 127 : v);
out[2 + lane] = (uint8_t) (int8_t) v;
}
/// DOWN, ONE WARP PER ROW, reading the quantized intermediate the caller produced.
///
/// `dst_index[h]` is which row of the shared output buffer hit `h` fills - see the header. It is the router's
/// slot, not `h`, and the two differ on every layer where some experts are resident and some are not.
__dpct_inline__ void
down_kernel(const uint8_t *__restrict__ blob_base,
const int32_t *__restrict__ slot_index,
const int32_t *__restrict__ dst_index, long long blob_bytes,
const uint8_t *__restrict__ h_q8_0,
const float *__restrict__ h_scales, float *__restrict__ out,
int n_hits, const int32_t *__restrict__ d_count = nullptr) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int warps_per_block = (int)(item_ct1.get_local_range(2) >> 5);
const long long row = (long long)item_ct1.get_group(2) * warps_per_block +
(item_ct1.get_local_id(2) >> 5);
const long long total = (long long) n_hits * H;
if (row >= total) return;
const int h = (int) (row / H);
if (d_count != nullptr && h >= *d_count) return;
const int r = (int) (row % H);
const int lane = item_ct1.get_local_id(2) & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
const uint8_t* xb = h_q8_0 + (size_t) h * (size_t) (FF / 32) * 34;
const float acc = row_dot_s2_q8(blob + O_D_CODES + (size_t) r * ROW_D,
blob + O_D_SCALES + (size_t) r * SC_D * 2, xb, FF / 32, lane,
h_scales ? h_scales + (size_t) h * (size_t) (FF / 32) : nullptr);
const float s = warp_sum(acc);
if (lane == 0) out[(size_t) dst_index[h] * H + r] = s;
}
// ---- THE SAME INTEGERS FROM FEWER INSTRUCTIONS. `STRATA_OLD_GROUPED=1` keeps the kernels above.
//
// **INSIDE A CHUNK, WHICH CODE MEETS WHICH ACTIVATION IN A `dp4a` WORD IS FREE.** `s = sum_e code_e * x_e` and
// `hx = sum_e x_e` are exact integer sums (|s| <= 32 * 3 * 128), so any grouping of the 32 products into words
// gives the same two integers. The grouping used here costs no transposition of the codes: `(w >> 2f) & 0x03030303`
// holds field `f` of each of a code word's four bytes - elements `4b + f`, plus 16 for the second word - so it is
// the ACTIVATIONS that are regrouped, once per chunk, into
//
// X[4h + f] = { x[16h + f], x[16h + 4 + f], x[16h + 8 + f], x[16h + 12 + f] }
//
// a 4x4 byte transpose of each half of the chunk, eight `__byte_perm`. The kernels above spend seven integer
// operations per code BYTE and a second `dp4a` chain for `hx` on every row; here a code word costs seven for 16
// elements and `hx` is summed once per chunk. `s`, `hx` and the float expression `dw * dx * (float) (s - hx)` are
// unchanged, so every output is bitwise the previous kernels' - `s2_expert_grouped_parity` checks exactly that.
// fp16 at an even address, one 16-bit load (`f16_at` reads it as two bytes); same value.
__dpct_inline__ float f16_ld(const uint8_t *p) {
return sycl::vec<sycl::half, 1>(sycl::bit_cast<sycl::half, unsigned short>(
*(const unsigned short *)p))
.convert<float, sycl::rounding_mode::automatic>()[0];
}
// A chunk's two code words as the eight `dp4a` operands that pair with `X[0..7]`: `m[4h + f]` byte `b` is the
// code of element `16h + 4b + f`.
__dpct_inline__ void expand_codes(sycl::uint2 cb, int m[8]) {
const unsigned M = 0x03030303u;
m[0] = (int)(cb.x() & M);
m[1] = (int)((cb.x() >> 2) & M);
m[2] = (int)((cb.x() >> 4) & M);
m[3] = (int)((cb.x() >> 6) & M);
m[4] = (int)(cb.y() & M);
m[5] = (int)((cb.y() >> 2) & M);
m[6] = (int)((cb.y() >> 4) & M);
m[7] = (int)((cb.y() >> 6) & M);
}
// The 32 int8 of the `block_q8_0` at `xb` regrouped into `X[0..7]` (see above), and their sum `hx`.
//
// **NINE ALIGNED WORD LOADS INSTEAD OF 32 BYTE LOADS.** The caller guarantees a 4-byte aligned activation row, so
// chunk `c`'s int8 start at `34c + 2`, i.e. 0 bytes (odd `c`) or 2 bytes (even `c`) into an aligned word; the
// words are read aligned and funnel-shifted into place. The ninth word is read only in the 2-byte case, and then
// it lies inside block `c + 1` of the same row: both row lengths (80 and 20 chunks) are even, so an even chunk is
// never a row's last.
__dpct_inline__ int load_x_chunk(const uint8_t *__restrict__ xb, int X[8]) {
const uint8_t* q = xb + 2;
const int off = (int) ((uintptr_t) q & 3);
const unsigned* p = (const unsigned*) (q - off);
const unsigned sh = (unsigned) off * 8;
unsigned v[9];
#pragma unroll
/*
DPCT1098: The '*' expression is used instead of the __ldg call. These
two expressions do not provide the exact same functionality. Check the
generated code for potential precision and/or performance issues.
*/
for (int j = 0; j < 8; ++j) v[j] = *(p + j);
/*
DPCT1098: The '*' expression is used instead of the __ldg call. These
two expressions do not provide the exact same functionality. Check the
generated code for potential precision and/or performance issues.
*/
v[8] = off != 0 ? *(p + 8) : 0u;
unsigned n[8];
int hx = 0;
#pragma unroll
for (int j = 0; j < 8; ++j) {
// natural word j: x[4j .. 4j+3]; a 64-bit shift (sh is 0, 8, 16 or 24) rather than __funnelshift_r, so the same
// source needs no CUDA-only intrinsic.
n[j] = (unsigned) ((((unsigned long long) v[j + 1] << 32) | v[j]) >> sh);
hx = strata::dp4a(0x01010101, (int) n[j], hx);
}
#pragma unroll
for (int h = 0; h < 2; ++h) {
const unsigned t0 = dpct::byte_level_permute(n[4 * h], n[4 * h + 1],
0x5140); // a0 b0 a1 b1
const unsigned t1 = dpct::byte_level_permute(n[4 * h], n[4 * h + 1],
0x7362); // a2 b2 a3 b3
const unsigned t2 = dpct::byte_level_permute(n[4 * h + 2], n[4 * h + 3],
0x5140); // c0 d0 c1 d1
const unsigned t3 = dpct::byte_level_permute(n[4 * h + 2], n[4 * h + 3],
0x7362); // c2 d2 c3 d3
X[4 * h + 0] =
(int)dpct::byte_level_permute(t0, t2, 0x5410); // a0 b0 c0 d0
X[4 * h + 1] =
(int)dpct::byte_level_permute(t0, t2, 0x7632); // a1 b1 c1 d1
X[4 * h + 2] =
(int)dpct::byte_level_permute(t1, t3, 0x5410); // a2 b2 c2 d2
X[4 * h + 3] =
(int)dpct::byte_level_permute(t1, t3, 0x7632); // a3 b3 c3 d3
}
return hx;
}
// `s` for one chunk: the eight expanded code words against the eight regrouped activation words.
__dpct_inline__ int chunk_s(const int m[8], const int X[8]) {
int s = 0;
#pragma unroll
for (int j = 0; j < 8; ++j) s = strata::dp4a(m[j], X[j], s);
return s;
}
/// `gu_kernel`, new: ONE WARP PER (gate, up) PAIR - row-slots `2r` and `2r + 1`, adjacent in the blob - so each
/// activation chunk is loaded and regrouped once for both rows. Per row, the lane's chunks, their order, the float
/// expression and the shuffle reduction are `row_dot_s2_q8`'s; the outputs land where `gu_kernel` puts them.
/*
DPCT1110: The total declared local variable size in device function
gu_pair_kernel exceeds 128 bytes and may cause high register pressure. Consult
with your hardware vendor to find the total register size available and adjust
the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void
gu_pair_kernel(const uint8_t *__restrict__ blob_base,
const int32_t *__restrict__ slot_index, long long blob_bytes,
const uint8_t *__restrict__ x_q8_0,
const float *__restrict__ x_scales, float *__restrict__ gate_up,
int n_hits, const int32_t *__restrict__ d_count,
const int32_t *__restrict__ dst_index, int tok_div) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int warps_per_block = (int)(item_ct1.get_local_range(2) >> 5);
const long long pair = (long long)item_ct1.get_group(2) * warps_per_block +
(item_ct1.get_local_id(2) >> 5);
const long long total = (long long) n_hits * FF;
if (pair >= total) return;
const int h = (int) (pair / FF);
if (d_count != nullptr && h >= *d_count) return;
const int r = (int) (pair % FF);
const int lane = item_ct1.get_local_id(2) & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
if (tok_div > 0) {
const int tok = dst_index[h] / tok_div;
x_q8_0 += (size_t) tok * (size_t) (H / 32) * 34;
if (x_scales != nullptr) x_scales += (size_t) tok * (size_t) (H / 32);
}
const uint8_t* codes = blob + (size_t) (2 * r) * ROW_GU; // gate row; the up row follows it
const uint8_t* scales = blob + O_GU_SCALES + (size_t) (2 * r) * SC_GU * 2;
float acc_g = 0.0f, acc_u = 0.0f;
for (int c = lane; c < H / 32; c += 32) {
const uint8_t* xb = x_q8_0 + (size_t) c * 34;
const float dx = x_scales ? x_scales[c] : f16_ld(xb);
int X[8], m[8];
const int hx = load_x_chunk(xb, X);
expand_codes(*(const sycl::uint2 *)(codes + (size_t)c * 8), m);
const float dw_g = f16_ld(scales + (size_t) (c >> 1) * 2);
acc_g += dw_g * dx * (float) (chunk_s(m, X) - hx);
expand_codes(*(const sycl::uint2 *)(codes + ROW_GU + (size_t)c * 8), m);
const float dw_u = f16_ld(scales + SC_GU * 2 + (size_t) (c >> 1) * 2);
acc_u += dw_u * dx * (float) (chunk_s(m, X) - hx);
}
const float sg = warp_sum(acc_g);
const float su = warp_sum(acc_u);
if (lane != 0) return;
gate_up[(size_t) h * FF + (size_t) r] = sg; // gate-major, as in `gu_kernel`
gate_up[(size_t) n_hits * FF + (size_t) h * FF + (size_t) r] = su;
}
/// `down_kernel`, new: ONE WARP PER PAIR OF ROWS `r, r + 1` of one hit, the intermediate's chunk loaded once for both.
/*
DPCT1110: The total declared local variable size in device function
down_pair_kernel exceeds 128 bytes and may cause high register pressure. Consult
with your hardware vendor to find the total register size available and adjust
the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void down_pair_kernel(const uint8_t *__restrict__ blob_base,
const int32_t *__restrict__ slot_index,
const int32_t *__restrict__ dst_index,
long long blob_bytes,
const uint8_t *__restrict__ h_q8_0,
const float *__restrict__ h_scales,
float *__restrict__ out, int n_hits,
const int32_t *__restrict__ d_count) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int warps_per_block = (int)(item_ct1.get_local_range(2) >> 5);
const long long pair = (long long)item_ct1.get_group(2) * warps_per_block +
(item_ct1.get_local_id(2) >> 5);
const long long total = (long long) n_hits * (H / 2);
if (pair >= total) return;
const int h = (int) (pair / (H / 2));
if (d_count != nullptr && h >= *d_count) return;
const int r = 2 * (int) (pair % (H / 2));
const int lane = item_ct1.get_local_id(2) & 31;
const uint8_t* blob = blob_base + (size_t) slot_index[h] * (size_t) blob_bytes;
const uint8_t* xrow = h_q8_0 + (size_t) h * (size_t) (FF / 32) * 34;
const float* xs = h_scales ? h_scales + (size_t) h * (size_t) (FF / 32) : nullptr;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
float acc0 = 0.0f, acc1 = 0.0f;
for (int c = lane; c < FF / 32; c += 32) {
const uint8_t* xb = xrow + (size_t) c * 34;
const float dx = xs ? xs[c] : f16_ld(xb);
int X[8], m[8];
const int hx = load_x_chunk(xb, X);
expand_codes(*(const sycl::uint2 *)(codes + (size_t)c * 8), m);
const float dw0 = f16_ld(scales + (size_t) (c >> 1) * 2);
acc0 += dw0 * dx * (float) (chunk_s(m, X) - hx);
expand_codes(*(const sycl::uint2 *)(codes + ROW_D + (size_t)c * 8), m);
const float dw1 = f16_ld(scales + SC_D * 2 + (size_t) (c >> 1) * 2);
acc1 += dw1 * dx * (float) (chunk_s(m, X) - hx);
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane != 0) return;
const size_t o = (size_t) dst_index[h] * H + (size_t) r;
out[o] = s0;
out[o + 1] = s1;
}
// The CPU subtracts the weight bias after its eight FMA accumulators have been reduced. Moving the
// subtraction into each integer dot, as the legacy kernel does, changes rounding even with equal scales.
__dpct_inline__ void activation_correction_kernel(const uint8_t *q8,
const float *scales,
float *hx, int chunks) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int c = item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (c >= chunks) return;
const int8_t* q = (const int8_t*) (q8 + (size_t) c * 34 + 2);
int sum = 0;
#pragma unroll
for (int j = 0; j < 32; ++j) sum += q[j];
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
hx[c] = scales[c] * (float)sum;
}
__dpct_inline__ int dot4(const uint8_t *codes, const int8_t *q) {
const unsigned c = *codes;
const int cw = (int) ((c & 3u) | (((c >> 2) & 3u) << 8) |
(((c >> 4) & 3u) << 16) | (((c >> 6) & 3u) << 24));
int xw;
memcpy(&xw, q, sizeof xw);
return strata::dp4a(cw, xw, 0);
}
__dpct_inline__ float row_dot_cpu_order(const uint8_t *codes,
const uint8_t *scales,
const uint8_t *xq, const float *xs,
const float *hx, int blocks, int lane) {
float acc = 0.0f;
float corr = 0.0f;
for (int b = 0; b < blocks; ++b) {
const float d = f16_at(scales + 2 * b);
const int lo = dot4(codes + b * 16 + lane,
(const int8_t*) (xq + (size_t) (2 * b) * 34 + 2) + lane * 4);
const int hi = dot4(codes + b * 16 + 8 + lane,
(const int8_t*) (xq + (size_t) (2 * b + 1) * 34 + 2) + lane * 4);
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
acc = sycl::fma(d * xs[2 * b], (float)lo, acc);
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
acc = sycl::fma(d * xs[2 * b + 1], (float)hi, acc);
if (lane == 0)
/*
DPCT1013: The rounding mode could not be specified and the
generated code may have different accuracy than the original code.
Verify the correctness. SYCL math built-in function rounding mode is
aligned with OpenCL C 1.2 standard.
*/
corr = corr + d * hx[2 * b] + hx[2 * b + 1];
}
// _mm_add_ps(low128, high128), then two _mm_hadd_ps. The pair order is 4, 1, 2;
// a standard shuffle tree in the order 4, 2, 1 is a different floating-point expression.
constexpr unsigned mask = 0xffffffffu;
/*
DPCT1108: '__shfl_down_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
acc = acc + dpct::experimental::shift_sub_group_left(
mask, sycl::ext::oneapi::this_work_item::get_sub_group(),
acc, 4, 8);
/*
DPCT1108: '__shfl_down_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
acc = acc + dpct::experimental::shift_sub_group_left(
mask, sycl::ext::oneapi::this_work_item::get_sub_group(),
acc, 1, 8);
/*
DPCT1108: '__shfl_down_sync' was migrated with the experimental feature
masked sub_group function which may not be supported by all compilers or
runtimes. You may need to adjust the code.
*/
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
acc = acc + dpct::experimental::shift_sub_group_left(
mask, sycl::ext::oneapi::this_work_item::get_sub_group(),
acc, 2, 8);
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
return acc - corr; // Only lane zero is consumed.
}
template <bool DOWN>
__dpct_inline__ void
cpu_order_projection_kernel(const uint8_t *blob_base, const int32_t *slots,
const int32_t *destinations, long long blob_bytes,
const uint8_t *xq, const float *xs, const float *hx,
float *out, int n_hits) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
constexpr int rows_per_hit = DOWN ? H : 2 * FF;
const int row = item_ct1.get_group(2) * (item_ct1.get_local_range(2) / 8) +
item_ct1.get_local_id(2) / 8;
if (row >= n_hits * rows_per_hit) return;
const int h = row / rows_per_hit;
const int r = row % rows_per_hit;
const int lane = item_ct1.get_local_id(2) & 7;
const uint8_t* blob = blob_base + (size_t) slots[h] * (size_t) blob_bytes;
const int chunks_offset = DOWN ? h * (FF / 32) : 0;
const uint8_t* codes = DOWN ? blob + O_D_CODES + (size_t) r * ROW_D : blob + (size_t) r * ROW_GU;
const uint8_t* scales = DOWN ? blob + O_D_SCALES + (size_t) r * SC_D * 2
: blob + O_GU_SCALES + (size_t) r * SC_GU * 2;
const float value = row_dot_cpu_order(codes, scales, xq + (size_t) chunks_offset * 34,
xs + chunks_offset, hx + chunks_offset,
DOWN ? SC_D : SC_GU, lane);
if (lane != 0) return;
if (DOWN) out[(size_t) destinations[h] * H + r] = value;
else out[((r & 1) ? (size_t) n_hits * FF : 0) + (size_t) h * FF + (r >> 1)] = value;
}
__dpct_inline__ void cpu_order_swiglu_kernel(float *gu, int pairs) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int i = item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (i >= pairs) return;
const float g = gu[i];
// Accurate fp32 exponential; __expf's approximation would introduce an additional source of error.
// CPU/GPU libc last-bit differences are diagnosed separately by the micro, not hidden with FP64 here.
const float eg = sycl::native::exp(-g);
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
gu[i] = g / (1.0f + eg) * gu[pairs + i];
}
__dpct_inline__ void cpu_order_quantize_kernel(const float *x, uint8_t *blocks,
float *scales, float *hx,
int chunks) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int c = item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (c >= chunks) return;
const float* xb = x + c * 32;
uint8_t* out = blocks + (size_t) c * 34;
float amax = 0.0f;
#pragma unroll
for (int j = 0; j < 32; ++j) amax = sycl::fmax(amax, sycl::fabs(xb[j]));
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
const float s = amax > 0.0f ? amax / 127.0f : 0.0f;
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
const float inv = s > 0.0f ? 1.0f / s : 0.0f;
scales[c] = s;
const uint16_t bits = sycl::bit_cast<unsigned short, sycl::half>(
sycl::vec<float, 1>(s)
.convert<sycl::half, sycl::rounding_mode::rte>()[0]);
out[0] = (uint8_t) bits;
out[1] = (uint8_t) (bits >> 8);
int sum = 0;
for (int j = 0; j < 32; ++j) {
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
const float t = xb[j] * inv;
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
int v = (int) (t + (t >= 0.0f ? 0.5f : -0.5f)); // SYCL port: dpct dropped the parentheses (it rounded every value to 0)
v = v < -127 ? -127 : (v > 127 ? 127 : v);
out[2 + j] = (uint8_t) (int8_t) v;
sum += v;
}
/*
DPCT1013: The rounding mode could not be specified and the generated
code may have different accuracy than the original code. Verify the
correctness. SYCL math built-in function rounding mode is aligned with
OpenCL C 1.2 standard.
*/
hx[c] = s * (float)sum;
}
constexpr int THREADS = 256;
void check(const char* who, void* stream) {
/*
DPCT1010: SYCL uses exceptions to report errors and does not use the
error codes. The cudaGetLastError function call was replaced with 0. You
need to rewrite this code.
*/
const dpct::err0 e = 0;
// Deliberately NOT synchronising for a non-null stream: this is called once per layer from a captured
// graph's worth of work, and `finish()`'s null-stream sync in `s_gemv.cu` is the pattern that made a whole
// round of measurements the driver's cost instead of the kernel's (Memory/ERRORS.md RC-7).
(void) stream;
}
// The previous kernels stay selectable for A/B - `STRATA_OLD_GROUPED=1` in the environment, or
// `moe_grouped_select_old` (the parity test runs both in one process). The environment is read once, on first use;
// the choice is made at each launch, so a captured graph keeps the kernels it was captured with.
std::atomic<int> g_select_old{-1};
// Which kernels the last launch of an entry point used: 1 = new, 0 = previous, -1 = none since the last query.
std::atomic<int> g_last_path{-1};
bool old_kernels() {
const int s = g_select_old.load(std::memory_order_relaxed);
if (s >= 0) return s != 0;
static const bool env = [] {
const char* e = std::getenv("STRATA_OLD_GROUPED");
return e != nullptr && e[0] == '1';
}();
return env;
}
// `STRATA_GROUPED_PAIR_MIN_HITS=N`: the per-hit path keeps the previous one-warp-per-row kernels below N hits of
// capacity. The new per-hit kernels launch half the warps (two rows each), so one hit's gate/up is 80 blocks - fewer
// than an RTX 5090's SMs - and whether that costs time at one to three hits is what `--bench` measures. Bitwise
// the same either way; default 0 (always the new kernels). Ignored while `moe_grouped_select_old` forces a choice.
long long pair_min_hits() {
if (g_select_old.load(std::memory_order_relaxed) >= 0) return 0;
static const long long n = [] {
const char* e = std::getenv("STRATA_GROUPED_PAIR_MIN_HITS");
return e != nullptr ? std::atoll(e) : 0LL;
}();
return n;
}
// The new kernels read the activations as aligned words (`load_x_chunk`), so they need 4-byte aligned activation
// rows - the input's and the intermediate's in `scratch`; the per-hit ones also read a blob's codes as uint2, which
// needs an 8-byte aligned arena and slot size (the grouped kernels above already did). Anything else keeps the
// previous kernels rather than issuing a misaligned load.
bool new_grouped(const void* x_q8_0, const void* scratch) {
const bool fast = !old_kernels() && ((uintptr_t) x_q8_0 & 3) == 0 && ((uintptr_t) scratch & 3) == 0;
g_last_path.store(fast ? 1 : 0, std::memory_order_relaxed);
return fast;
}
bool new_hit(const void* blob_base, long long blob_bytes, const void* x_q8_0, const void* scratch, long long cap) {
const bool fast = new_grouped(x_q8_0, scratch) && ((uintptr_t) blob_base & 7) == 0 && (blob_bytes & 7) == 0 &&
cap >= pair_min_hits();
g_last_path.store(fast ? 1 : 0, std::memory_order_relaxed);
return fast;
}
// The per-hit path's two projections, previous or new kernels (one warp per row, or per pair of rows).
void launch_hit_gu(bool fast, const uint8_t *blob_base,
const int32_t *slot_index, long long blob_bytes,
const uint8_t *x_q8_0, const float *x_scales, float *gate_up,
long long cap, const int32_t *d_count,
const int32_t *dst_index, int tok_div, dpct::queue_ptr cs) {
const int warps = THREADS / 32;
if (fast) {
const long long pairs = cap * (long long) FF;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class gu_pair_kernel_787004>>(
sycl::nd_range<3>(
sycl::range(1, 1, (unsigned)((pairs + warps - 1) / warps)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
gu_pair_kernel(blob_base, slot_index, blob_bytes,
x_q8_0, x_scales, gate_up, (int)cap,
d_count, dst_index, tok_div);
});
}
} else {
const long long rows = cap * 2LL * FF;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class gu_kernel_55bc52>>(
sycl::nd_range<3>(
sycl::range(1, 1, (unsigned)((rows + warps - 1) / warps)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
gu_kernel(blob_base, slot_index, blob_bytes, x_q8_0,
x_scales, gate_up, (int)cap, d_count,
dst_index, tok_div);
});
}
}
}
void launch_hit_down(bool fast, const uint8_t *blob_base,
const int32_t *slot_index, const int32_t *dst_index,
long long blob_bytes, const uint8_t *h_q8_0,
const float *h_scales, float *out, long long cap,
const int32_t *d_count, dpct::queue_ptr cs) {
const int warps = THREADS / 32;
if (fast) {
const long long pairs = cap * (long long) (H / 2);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class down_pair_kernel_7d15d9>>(
sycl::nd_range<3>(
sycl::range(1, 1, (unsigned)((pairs + warps - 1) / warps)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
down_pair_kernel(blob_base, slot_index, dst_index,
blob_bytes, h_q8_0, h_scales, out,
(int)cap, d_count);
});
}
} else {
const long long rows = cap * (long long) H;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class down_kernel_b3587f>>(
sycl::nd_range<3>(
sycl::range(1, 1, (unsigned)((rows + warps - 1) / warps)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
down_kernel(blob_base, slot_index, dst_index,
blob_bytes, h_q8_0, h_scales, out, (int)cap,
d_count);
});
}
}
}
} // namespace
void moe_grouped_select_old(int old) { g_select_old.store(old < 0 ? -1 : (old != 0 ? 1 : 0)); }
int moe_grouped_last_path() { return g_last_path.exchange(-1, std::memory_order_relaxed); }
uint64_t moe_hit_grouped_scratch_bytes(int64_t n_hits, int64_t n_embd, int64_t n_ff) {
if (n_hits <= 0) return 0;
const uint64_t gu = (uint64_t) n_hits * (uint64_t) (2 * n_ff) * 4;
const uint64_t q8 = (uint64_t) n_hits * (uint64_t) (n_ff / 32) * 34;
// R4.2h: the fp32 scales for the INTERMEDIATE's own quantization, one per 32-element chunk per hit.
const uint64_t hs = (uint64_t) n_hits * (uint64_t) (n_ff / 32) * 4;
const uint64_t xh = (uint64_t) (n_embd / 32) * 4;
return ((gu + 15) & ~15ull) + ((q8 + 15) & ~15ull) + 2 * ((hs + 15) & ~15ull) +
((xh + 15) & ~15ull);
}
void moe_hit_grouped_s2(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
int64_t n_hits, int64_t blob_bytes, const uint8_t* x_q8_0, void* scratch, float* out,
void* stream, const float* x_scales) {
if (n_hits <= 0) return;
dpct::queue_ptr cs = strata::q_of(stream);
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, n_hits);
const uint64_t gu_bytes = ((uint64_t) n_hits * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) n_hits * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
// 1. gate + up, one launch for every row of every hit.
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, n_hits, nullptr, nullptr,
0, cs);
check("moe_hit_grouped_s2/gu", stream);
}
// 2. silu(gate) * up.
{
const long long pairs = n_hits * (long long) FF;
const unsigned blocks = (unsigned) ((pairs + THREADS - 1) / THREADS);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class swiglu_kernel_792534>>(
sycl::nd_range<3>(sycl::range(1, 1, blocks) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
swiglu_kernel(gate_up, pairs);
});
}
check("moe_hit_grouped_s2/swiglu", stream);
}
// 3. the intermediate's own contract, which is `ggml_mul_mat`'s rule and NOT a choice: the down weight is
// Q2_0, whose `vec_dot_type` is Q8_0. `gate_up` holds the pairs; the product went into the first half.
// **R4.2h: the CPU quantizes this intermediate into `a2` with fp32 scales too** (`expert.cpp:232`), so
// when the caller supplies `x_scales` the intermediate gets the CPU's contract as well - otherwise the
// down projection would keep the very disagreement the gate/up projection just had removed.
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, n_hits * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, n_hits * (int64_t) FF, stream);
// 4. down.
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, n_hits, nullptr, cs);
check("moe_hit_grouped_s2/down", stream);
}
}
namespace {
// Plan v0.3 P4 token graph: which of this layer's routed experts are resident, decided ON THE DEVICE from the
// static residency row, so no host step sits between the ring and the hit kernels. One warp; k <= 32.
__dpct_inline__ void hit_select_kernel(const int32_t *__restrict__ ids,
const int32_t *__restrict__ res_row,
int k, int n_expert,
int32_t *__restrict__ slot,
int32_t *__restrict__ dst,
int32_t *__restrict__ count) {
const int lane =
sycl::ext::oneapi::this_work_item::get_nd_item<3>().get_local_id(2);
int s = -1;
if (lane < k) {
const int e = ids[lane];
if (e >= 0 && e < n_expert) s = res_row[e];
}
const unsigned hit = sycl::reduce_over_group(
sycl::ext::oneapi::this_work_item::get_sub_group(),
(0xffffffffu &
(0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()
.get_local_linear_id())) &&
s >= 0
? (0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()
.get_local_linear_id())
: 0,
sycl::ext::oneapi::plus<>());
if (s >= 0) {
const int at = sycl::popcount(hit & ((1u << lane) - 1u));
slot[at] = s;
dst[at] = lane;
}
if (lane == 0) *count = sycl::popcount(hit);
}
// Plan v0.3 P6: the same for up to 128 routed entries (a verify window of T tokens x k): four warps, ballots
// compacted in entry order.
__dpct_inline__ void hit_select_multi_kernel(
const int32_t *__restrict__ ids, const int32_t *__restrict__ res_row, int n,
int n_expert, int32_t *__restrict__ slot, int32_t *__restrict__ dst,
int32_t *__restrict__ count) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
auto &warp_count = *sycl::ext::oneapi::group_local_memory_for_overwrite<int[4]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int i = item_ct1.get_local_id(2), lane = i & 31, warp = i >> 5;
int s = -1;
if (i < n) {
const int e = ids[i];
if (e >= 0 && e < n_expert) s = res_row[e];
}
const unsigned hit = sycl::reduce_over_group(
sycl::ext::oneapi::this_work_item::get_sub_group(),
(0xffffffffu &
(0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()
.get_local_linear_id())) &&
s >= 0
? (0x1 << sycl::ext::oneapi::this_work_item::get_sub_group()
.get_local_linear_id())
: 0,
sycl::ext::oneapi::plus<>());
if (lane == 0) warp_count[warp] = sycl::popcount(hit);
item_ct1.barrier(sycl::access::fence_space::local_space);
int before = 0;
#pragma unroll
for (int w = 0; w < warp; ++w) before += warp_count[w];
if (s >= 0) {
const int at = before + sycl::popcount(hit & ((1u << lane) - 1u));
slot[at] = s;
dst[at] = i;
}
if (i == 0) *count = warp_count[0] + warp_count[1] + warp_count[2] + warp_count[3];
}
__dpct_inline__ void add_hits_kernel(float *__restrict__ parts,
const float *__restrict__ hit_out,
const int32_t *__restrict__ dst,
const int32_t *__restrict__ count,
int n_embd) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int h = item_ct1.get_group(1);
if (h >= *count) return;
const size_t row = (size_t) dst[h] * (size_t) n_embd;
#pragma unroll
for (int i = item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
i < n_embd;
i += item_ct1.get_group_range(2) * item_ct1.get_local_range(2))
parts[row + i] += hit_out[row + i];
}
} // namespace
void moe_hit_select(const int32_t* ids, const int32_t* res_row, int k, int n_expert, int32_t* slot, int32_t* dst,
int32_t* count, void* stream) {
if (k < 1 || k > 32) { std::fprintf(stderr, "moe_hit_select: k must be 1..32\n"); std::exit(1); }
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
strata::q_of(stream)
->parallel_for<dpct_kernel_name<class hit_select_kernel_f87f42>>(
sycl::nd_range<3>(sycl::range(1, 1, 32), sycl::range(1, 1, 32)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
hit_select_kernel(ids, res_row, k, n_expert, slot, dst,
count);
});
}
check("moe_hit_select", stream);
}
void moe_hit_grouped_s2_dev(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
const int32_t* d_count, int64_t cap, int64_t blob_bytes, const uint8_t* x_q8_0,
void* scratch, float* out, void* stream, const float* x_scales) {
if (cap <= 0) return;
dpct::queue_ptr cs = strata::q_of(stream);
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, cap);
const uint64_t gu_bytes = ((uint64_t) cap * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, cap, d_count, nullptr, 0,
cs);
check("moe_hit_grouped_s2_dev/gu", stream);
}
{
const long long pairs = cap * (long long) FF;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class swiglu_kernel_5aa9a9>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((pairs + THREADS - 1) / THREADS)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
swiglu_kernel(gate_up, pairs);
});
}
check("moe_hit_grouped_s2_dev/swiglu", stream);
}
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap * (int64_t) FF, stream);
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, cap, d_count, cs);
check("moe_hit_grouped_s2_dev/down", stream);
}
}
void moe_hit_select_multi(const int32_t* ids, const int32_t* res_row, int n, int n_expert, int32_t* slot, int32_t* dst,
int32_t* count, void* stream) {
if (n < 1 || n > 128) { std::fprintf(stderr, "moe_hit_select_multi: n must be 1..128\n"); std::exit(1); }
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
strata::q_of(stream)
->parallel_for<
dpct_kernel_name<class hit_select_multi_kernel_4749ea>>(
sycl::nd_range<3>(sycl::range(1, 1, 128),
sycl::range(1, 1, 128)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
hit_select_multi_kernel(ids, res_row, n, n_expert, slot,
dst, count);
});
}
check("moe_hit_select_multi", stream);
}
void moe_hit_grouped_s2_multi(const uint8_t* blob_base, const int32_t* slot_index, const int32_t* dst_index,
const int32_t* d_count, int64_t cap, int64_t blob_bytes, const uint8_t* x_q8_0,
const float* x_scales, int k_per_token, void* scratch, float* out, void* stream) {
if (cap <= 0) return;
dpct::queue_ptr cs = strata::q_of(stream);
const bool fast = new_hit(blob_base, blob_bytes, x_q8_0, scratch, cap);
const uint64_t gu_bytes = ((uint64_t) cap * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
{
launch_hit_gu(fast, blob_base, slot_index, blob_bytes, x_q8_0, x_scales, gate_up, cap, d_count, dst_index,
k_per_token, cs);
check("moe_hit_grouped_s2_multi/gu", stream);
}
{
const long long pairs = cap * (long long) FF;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class swiglu_kernel_a0c31a>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((pairs + THREADS - 1) / THREADS)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
swiglu_kernel(gate_up, pairs);
});
}
check("moe_hit_grouped_s2_multi/swiglu", stream);
}
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap * (int64_t) FF, stream);
{
launch_hit_down(fast, blob_base, slot_index, dst_index, blob_bytes, h_q8_0,
x_scales != nullptr ? h_scales : nullptr, out, cap, d_count, cs);
check("moe_hit_grouped_s2_multi/down", stream);
}
}
namespace {
// Plan v0.3 P6: experts GROUPED - one blob pointer per group (a VRAM slot or a mapped host blob read over PCIe),
// every entry of the group (a token routed to that expert) computed from ONE read of each row. Per entry the
// arithmetic is `row_dot_s2_q8`'s, chunk by chunk in the same lane order, so every entry is bitwise the per-entry
// hit kernel's.
constexpr int GU_CHUNKS = (H / 32 + 31) / 32; // 3: chunks of a gate/up row per lane (80 chunks / 32 lanes)
constexpr int GMAX = 8; // entries per group (tokens routed to one expert in a window)
// a group holds one entry per token of the window routed to its expert, and the kernels below keep at
// most GMAX of them (`min(..., GMAX)`): a longer window would drop entries without a word.
static_assert(GMAX >= kVerifyMaxT, "a verify window's group can exceed GMAX entries");
constexpr int GU_ROWS = 32; // gate/up rows per block: 4 per warp
constexpr int D_ROWS = 64; // down rows per block: 8 per warp
// One activation chunk's contribution, `row_dot_s2_q8`'s inner body with the 32 int8 of the chunk already in
// aligned words: same dp4a sequence, same float expression, so the result is bitwise the per-entry kernel's.
__dpct_inline__ float chunk_dot(sycl::uint2 cb, const int *xw, float dw,
float dx) {
const uint8_t* cbytes = (const uint8_t*) &cb;
int s = 0, hx = 0;
const int ones = 0x01010101;
#pragma unroll
for (int j = 0; j < 8; ++j) {
const unsigned cbyte = cbytes[j];
const int cw = (int) ((cbyte & 3u) | (((cbyte >> 2) & 3u) << 8) | (((cbyte >> 4) & 3u) << 16) |
(((cbyte >> 6) & 3u) << 24));
s = strata::dp4a(cw, xw[j], s);
hx = strata::dp4a(ones, xw[j], hx);
}
return dw * dx * (float) (s - hx);
}
// Gate/up: a block = GU_ROWS rows of ONE group. The group's activations (each entry's token row of x_q8_0 and
// its fp32 scales) are staged once into shared memory as aligned words; each warp then walks its rows, loading
// each lane's code chunks once and dotting them with every entry.
/*
DPCT1110: The total declared local variable size in device function
gu_grouped_kernel exceeds 128 bytes and may cause high register pressure.
Consult with your hardware vendor to find the total register size available and
adjust the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void gu_grouped_kernel(
const unsigned long long *__restrict__ grp_ptr,
const int32_t *__restrict__ grp_start, const int32_t *__restrict__ n_groups,
const int32_t *__restrict__ ent_tok, const uint8_t *__restrict__ x_q8_0,
const float *__restrict__ x_scales, float *__restrict__ gate_up,
int cap_entries) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
auto &xs_q =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[GMAX][H / 4]>(
sycl::ext::oneapi::this_work_item::get_work_group<
3>()); // the entries' int8 activations as words (2560 B each)
auto &xs_d = *sycl::ext::oneapi::group_local_memory_for_overwrite<
float[GMAX][H / 32]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int g = item_ct1.get_group(1);
if (g >= *n_groups) return;
const int e0 = grp_start[g], ne = sycl::min(grp_start[g + 1] - e0, GMAX);
const int t = item_ct1.get_local_id(2), lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (H / 32); i += item_ct1.get_local_range(2)) {
const int k = i / (H / 32), c = i - k * (H / 32);
const uint8_t* xb = x_q8_0 + (size_t) ent_tok[e0 + k] * (size_t) (H / 32) * 34 + (size_t) c * 34;
xs_d[k][c] = x_scales ? x_scales[(size_t) ent_tok[e0 + k] * (H / 32) + c] : f16_at(xb);
const int8_t* q = (const int8_t*) (xb + 2);
#pragma unroll
for (int w = 0; w < 8; ++w) {
int v;
memcpy(&v, q + 4 * w, 4);
xs_q[k][c * 8 + w] = v;
}
}
item_ct1.barrier(sycl::access::fence_space::local_space);
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = item_ct1.get_group(2) * GU_ROWS;
for (int rr = warp; rr < GU_ROWS; rr += 8) {
const int i = row0 + rr;
const uint8_t* codes = blob + (size_t) i * ROW_GU;
const uint8_t* scales = blob + O_GU_SCALES + (size_t) i * SC_GU * 2;
sycl::uint2 cb[GU_CHUNKS];
float dw[GU_CHUNKS];
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c < H / 32) {
cb[q] = *(const sycl::uint2 *)(codes + (size_t)c * 8);
dw[q] = f16_at(scales + (size_t) (c >> 1) * 2);
}
}
for (int k = 0; k < ne; ++k) {
float acc = 0.0f;
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c >= H / 32) break;
acc += chunk_dot(cb[q], &xs_q[k][c * 8], dw[q], xs_d[k][c]);
}
const float sum = warp_sum(acc);
if (lane == 0) {
const int e = e0 + k, r = i >> 1;
const size_t base = (i & 1) ? ((size_t) cap_entries * FF + (size_t) e * FF) : ((size_t) e * FF);
gate_up[base + (size_t) r] = sum;
}
}
}
}
// Down: a block = D_ROWS rows of ONE group; the entries' quantized intermediates staged once. A down row is 20
// chunks, so lanes 0..19 each hold one chunk, as in the per-entry kernel.
__dpct_inline__ void down_grouped_kernel(
const unsigned long long *__restrict__ grp_ptr,
const int32_t *__restrict__ grp_start, const int32_t *__restrict__ n_groups,
const int32_t *__restrict__ ent_dst, const uint8_t *__restrict__ h_q8_0,
const float *__restrict__ h_scales, float *__restrict__ out) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
auto &hs_q =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[GMAX][FF / 4]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &hs_d = *sycl::ext::oneapi::group_local_memory_for_overwrite<
float[GMAX][FF / 32]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int g = item_ct1.get_group(1);
if (g >= *n_groups) return;
const int e0 = grp_start[g], ne = sycl::min(grp_start[g + 1] - e0, GMAX);
const int t = item_ct1.get_local_id(2), lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * (FF / 32); i += item_ct1.get_local_range(2)) {
const int k = i / (FF / 32), c = i - k * (FF / 32);
const uint8_t* xb = h_q8_0 + (size_t) (e0 + k) * (size_t) (FF / 32) * 34 + (size_t) c * 34;
hs_d[k][c] = h_scales ? h_scales[(size_t) (e0 + k) * (FF / 32) + c] : f16_at(xb);
const int8_t* q = (const int8_t*) (xb + 2);
#pragma unroll
for (int w = 0; w < 8; ++w) {
int v;
memcpy(&v, q + 4 * w, 4);
hs_q[k][c * 8 + w] = v;
}
}
item_ct1.barrier(sycl::access::fence_space::local_space);
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = item_ct1.get_group(2) * D_ROWS;
for (int rr = warp; rr < D_ROWS; rr += 8) {
const int r = row0 + rr;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
const int c = lane;
sycl::uint2 cb = sycl::uint2(0, 0);
float dw = 0.0f;
if (c < FF / 32) {
cb = *(const sycl::uint2 *)(codes + (size_t)c * 8);
dw = f16_at(scales + (size_t) (c >> 1) * 2);
}
for (int k = 0; k < ne; ++k) {
float acc = 0.0f;
if (c < FF / 32) acc += chunk_dot(cb, &hs_q[k][c * 8], dw, hs_d[k][c]);
const float sum = warp_sum(acc);
if (lane == 0) out[(size_t) ent_dst[e0 + k] * H + r] = sum;
}
}
}
// ---- The grouped kernels with the staged activations laid out for the reads.
//
// **THE LAYOUT ABOVE IS AN 8-WAY BANK CONFLICT.** `xs_q[k][c * 8 + j]` puts lane `L`'s word `j` at `8L + j`, so
// lanes `L, L + 4, L + 8, ...` share a bank on every one of the chunk's eight reads; and `chunk_dot` recomputes
// `hx`, which depends on the entry alone, for every row. Here the staging writes, once per (entry, chunk):
//
// xs_w[j][k * NC + c] word j of the regrouped chunk (`load_x_chunk`), so lanes read consecutive words
// xs_dh[k * NC + c] (dx, hx) - one 64-bit read per chunk instead of a float and eight `dp4a`
//
// Word-major, the staging writes are consecutive as well (the flat index `i` IS `k * NC + c`), so neither side
// conflicts and no padding is needed. Each warp takes its rows TWO at a time (adjacent rows: gate `r` and up `r`,
// or down rows `r, r + 1`), so every shared read serves both; the codes of both are expanded once per row, not
// once per entry. Per (row, entry) the lane's chunks, their order, the float expression and the shuffle
// reduction are the kernels' above, so every output is bitwise theirs.
// Stages chunk `i = k * NC + c` of a group: its regrouped words at `xs_w[j * STRIDE + i]` and (dx, hx) at `xs_dh[i]`.
template <int STRIDE>
__dpct_inline__ void stage_chunk(const uint8_t *__restrict__ xb, float dx,
int *xs_w, sycl::int2 *xs_dh, int i) {
int X[8];
const int hx = load_x_chunk(xb, X);
#pragma unroll
for (int j = 0; j < 8; ++j) xs_w[j * STRIDE + i] = X[j];
xs_dh[i] = sycl::int2(sycl::bit_cast<int>(dx), hx);
}
/*
DPCT1110: The total declared local variable size in device function
gu_grouped_t_kernel exceeds 128 bytes and may cause high register pressure.
Consult with your hardware vendor to find the total register size available and
adjust the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void gu_grouped_t_kernel(
const unsigned long long *__restrict__ grp_ptr,
const int32_t *__restrict__ grp_start, const int32_t *__restrict__ n_groups,
const int32_t *__restrict__ ent_tok, const uint8_t *__restrict__ x_q8_0,
const float *__restrict__ x_scales, float *__restrict__ gate_up,
int cap_entries) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
constexpr int NC = H / 32;
auto &xs_w = *sycl::ext::oneapi::group_local_memory_for_overwrite<
int[8 * GMAX * NC]>(
sycl::ext::oneapi::this_work_item::get_work_group<
3>()); // 20 KB: word j of entry k's chunk c at [j][k * NC + c]
auto &xs_dh = *sycl::ext::oneapi::group_local_memory_for_overwrite<
sycl::int2[GMAX * NC]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>()); // 5 KB: (dx as
// bits, hx)
const int g = item_ct1.get_group(1);
if (g >= *n_groups) return;
const int e0 = grp_start[g], ne = sycl::min(grp_start[g + 1] - e0, GMAX);
const int t = item_ct1.get_local_id(2), lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += item_ct1.get_local_range(2)) {
const int k = i / NC, c = i - k * NC;
const int tok = ent_tok[e0 + k];
const uint8_t* xb = x_q8_0 + (size_t) tok * (size_t) NC * 34 + (size_t) c * 34;
const float dx = x_scales ? x_scales[(size_t) tok * NC + c] : f16_ld(xb);
stage_chunk<GMAX * NC>(xb, dx, xs_w, xs_dh, i);
}
item_ct1.barrier(sycl::access::fence_space::local_space);
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = item_ct1.get_group(2) *
GU_ROWS; // even: a pair is always (gate r, up r)
for (int pp = warp; pp < GU_ROWS / 2; pp += 8) {
const int i = row0 + 2 * pp;
const uint8_t* codes = blob + (size_t) i * ROW_GU;
const uint8_t* scales = blob + O_GU_SCALES + (size_t) i * SC_GU * 2;
int m0[GU_CHUNKS][8], m1[GU_CHUNKS][8];
float dw0[GU_CHUNKS], dw1[GU_CHUNKS];
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c < NC) {
expand_codes(*(const sycl::uint2 *)(codes + (size_t)c * 8),
m0[q]);
expand_codes(
*(const sycl::uint2 *)(codes + ROW_GU + (size_t)c * 8),
m1[q]);
dw0[q] = f16_ld(scales + (size_t) (c >> 1) * 2);
dw1[q] = f16_ld(scales + SC_GU * 2 + (size_t) (c >> 1) * 2);
}
}
for (int k = 0; k < ne; ++k) {
float acc0 = 0.0f, acc1 = 0.0f;
#pragma unroll
for (int q = 0; q < GU_CHUNKS; ++q) {
const int c = lane + 32 * q;
if (c >= NC) break;
const int at = k * NC + c;
int X[8];
#pragma unroll
for (int j = 0; j < 8; ++j) X[j] = xs_w[j * (GMAX * NC) + at];
const sycl::int2 dh = xs_dh[at];
const float dx = sycl::bit_cast<float>(dh.x());
acc0 += dw0[q] * dx * (float)(chunk_s(m0[q], X) - dh.y());
acc1 += dw1[q] * dx * (float)(chunk_s(m1[q], X) - dh.y());
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane == 0) {
const int e = e0 + k, r = i >> 1;
gate_up[(size_t) e * FF + (size_t) r] = s0;
gate_up[(size_t) cap_entries * FF + (size_t) e * FF + (size_t) r] = s1;
}
}
}
}
/*
DPCT1110: The total declared local variable size in device function
down_grouped_t_kernel exceeds 128 bytes and may cause high register pressure.
Consult with your hardware vendor to find the total register size available and
adjust the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void down_grouped_t_kernel(
const unsigned long long *__restrict__ grp_ptr,
const int32_t *__restrict__ grp_start, const int32_t *__restrict__ n_groups,
const int32_t *__restrict__ ent_dst, const uint8_t *__restrict__ h_q8_0,
const float *__restrict__ h_scales, float *__restrict__ out) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
constexpr int NC = FF / 32;
auto &hs_w = *sycl::ext::oneapi::group_local_memory_for_overwrite<
int[8 * GMAX * NC]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &hs_dh = *sycl::ext::oneapi::group_local_memory_for_overwrite<
sycl::int2[GMAX * NC]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int g = item_ct1.get_group(1);
if (g >= *n_groups) return;
const int e0 = grp_start[g], ne = sycl::min(grp_start[g + 1] - e0, GMAX);
const int t = item_ct1.get_local_id(2), lane = t & 31, warp = t >> 5;
for (int i = t; i < ne * NC; i += item_ct1.get_local_range(2)) {
const int k = i / NC, c = i - k * NC;
const uint8_t* xb = h_q8_0 + (size_t) (e0 + k) * (size_t) NC * 34 + (size_t) c * 34;
const float dx = h_scales ? h_scales[(size_t) (e0 + k) * NC + c] : f16_ld(xb);
stage_chunk<GMAX * NC>(xb, dx, hs_w, hs_dh, i);
}
item_ct1.barrier(sycl::access::fence_space::local_space);
const uint8_t* blob = (const uint8_t*) grp_ptr[g];
const int row0 = item_ct1.get_group(2) * D_ROWS;
for (int pp = warp; pp < D_ROWS / 2; pp += 8) {
const int r = row0 + 2 * pp;
const uint8_t* codes = blob + O_D_CODES + (size_t) r * ROW_D;
const uint8_t* scales = blob + O_D_SCALES + (size_t) r * SC_D * 2;
const int c = lane; // lanes 0..19 hold one chunk each, as above
int m0[8], m1[8];
float dw0 = 0.0f, dw1 = 0.0f;
if (c < NC) {
expand_codes(*(const sycl::uint2 *)(codes + (size_t)c * 8), m0);
expand_codes(*(const sycl::uint2 *)(codes + ROW_D + (size_t)c * 8),
m1);
dw0 = f16_ld(scales + (size_t) (c >> 1) * 2);
dw1 = f16_ld(scales + SC_D * 2 + (size_t) (c >> 1) * 2);
}
for (int k = 0; k < ne; ++k) {
float acc0 = 0.0f, acc1 = 0.0f;
if (c < NC) {
const int at = k * NC + c;
int X[8];
#pragma unroll
for (int j = 0; j < 8; ++j) X[j] = hs_w[j * (GMAX * NC) + at];
const sycl::int2 dh = hs_dh[at];
const float dx = sycl::bit_cast<float>(dh.x());
acc0 += dw0 * dx * (float)(chunk_s(m0, X) - dh.y());
acc1 += dw1 * dx * (float)(chunk_s(m1, X) - dh.y());
}
const float s0 = warp_sum(acc0);
const float s1 = warp_sum(acc1);
if (lane == 0) {
const size_t o = (size_t) ent_dst[e0 + k] * H + (size_t) r;
out[o] = s0;
out[o + 1] = s1;
}
}
}
}
} // namespace
namespace {
// Plan v0.3 P6: groups built on the device when every expert is resident at `base + id * blob` (the MTP layer):
// one block of 128 threads, groups in first-appearance order, entries of a group in routing order.
__dpct_inline__ void group_resident_kernel(
const int32_t *__restrict__ ids, int n, int k_per_tok, const uint8_t *base,
long long blob, unsigned long long *__restrict__ grp_ptr,
int32_t *__restrict__ grp_start, int32_t *__restrict__ counts,
int32_t *__restrict__ ent_dst, int32_t *__restrict__ ent_tok) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
auto &e_s = *sycl::ext::oneapi::group_local_memory_for_overwrite<int[128]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &first_s =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[128]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &size_s =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[128]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &gidx_s =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[128]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
auto &gstart_s =
*sycl::ext::oneapi::group_local_memory_for_overwrite<int[129]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int i = item_ct1.get_local_id(2);
const int e = i < n ? ids[i] : -1;
e_s[i] = e;
item_ct1.barrier(sycl::access::fence_space::local_space);
int first = i, rank = 0, size = 0;
if (i < n) {
#pragma unroll
for (int j = 0; j < i; ++j)
if (e_s[j] == e) { if (first == i) first = j; ++rank; }
if (first == i)
#pragma unroll
for (int j = i; j < n; ++j) size += e_s[j] == e;
}
first_s[i] = first;
size_s[i] = (i < n && first == i) ? size : 0;
item_ct1.barrier(sycl::access::fence_space::local_space);
if (i == 0) {
int gi = 0, acc = 0;
#pragma unroll
for (int j = 0; j < n; ++j)
if (first_s[j] == j) {
gidx_s[j] = gi;
gstart_s[gi] = acc;
grp_ptr[gi] = (unsigned long long) (base + (size_t) e_s[j] * (size_t) blob);
grp_start[gi] = acc;
acc += size_s[j];
++gi;
}
grp_start[gi] = acc;
counts[0] = gi;
counts[1] = acc;
}
item_ct1.barrier(sycl::access::fence_space::local_space);
if (i < n) {
const int at = gstart_s[gidx_s[first]] + rank;
ent_dst[at] = i;
ent_tok[at] = i / k_per_tok;
}
}
} // namespace
void moe_group_resident(const int32_t* ids, int n, int k_per_tok, const uint8_t* base, int64_t blob,
unsigned long long* grp_ptr, int32_t* grp_start, int32_t* counts, int32_t* ent_dst,
int32_t* ent_tok, void* stream) {
if (n < 1 || n > 128) { std::fprintf(stderr, "moe_group_resident: n must be 1..128\n"); std::exit(1); }
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
strata::q_of(stream)
->parallel_for<
dpct_kernel_name<class group_resident_kernel_5c38b8>>(
sycl::nd_range<3>(sycl::range(1, 1, 128),
sycl::range(1, 1, 128)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
group_resident_kernel(ids, n, k_per_tok, base,
(long long)blob, grp_ptr, grp_start,
counts, ent_dst, ent_tok);
});
}
check("moe_group_resident", stream);
}
void moe_grouped_s2(const unsigned long long* grp_ptr, const int32_t* grp_start, const int32_t* n_groups,
const int32_t* ent_dst, const int32_t* ent_tok, int64_t cap_groups, int64_t cap_entries,
const uint8_t* x_q8_0, const float* x_scales, void* scratch, float* out, void* stream) {
if (cap_groups <= 0 || cap_entries <= 0) return;
dpct::queue_ptr cs = strata::q_of(stream);
const uint64_t gu_bytes = ((uint64_t) cap_entries * (uint64_t) (2 * FF) * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) cap_entries * (uint64_t) (FF / 32) * 34 + 15) & ~15ull;
float* gate_up = (float*) scratch;
uint8_t* h_q8_0 = (uint8_t*) scratch + gu_bytes;
float* h_scales = (float*) ((uint8_t*) scratch + gu_bytes + q8_bytes);
const bool fast = new_grouped(x_q8_0, scratch);
{
const dpct::dim3 grid((unsigned)(2 * FF / GU_ROWS),
(unsigned)cap_groups);
if (fast)
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<
dpct_kernel_name<class gu_grouped_t_kernel_998f94>>(
sycl::nd_range<3>(grid * sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
gu_grouped_t_kernel(grp_ptr, grp_start, n_groups,
ent_tok, x_q8_0, x_scales, gate_up,
(int)cap_entries);
});
} else {
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class gu_grouped_kernel_c9eb4d>>(
sycl::nd_range<3>(grid * sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
gu_grouped_kernel(grp_ptr, grp_start, n_groups, ent_tok,
x_q8_0, x_scales, gate_up,
(int)cap_entries);
});
}
check("moe_grouped_s2/gu", stream);
}
// #783 PR-k (stuchapin909): SwiGLU and the scaled Q8_0 quantize of its result in one kernel (gate_up still gets the
// activation written back, so the parity checks read the same buffer); STRATA_S2_SWIGLU_Q8=0 keeps the two launches
static const bool swiglu_q8 = [] {
const char* e = std::getenv("STRATA_S2_SWIGLU_Q8");
return e == nullptr || e[0] != '0';
}();
static_assert(FF % 32 == 0, "the fused SwiGLU + Q8_0 kernel works in whole 32-value blocks");
if (fast && x_scales != nullptr && swiglu_q8) {
const long long pairs = cap_entries * (long long) FF;
const long long n_blocks = pairs >> 5;
const int warps = THREADS / 32;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<
class swiglu_quantize_q8_0_scaled_kernel_1271ea>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((n_blocks + warps - 1) / warps)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
swiglu_quantize_q8_0_scaled_kernel(gate_up, pairs,
h_q8_0, h_scales);
});
}
check("moe_grouped_s2/swiglu_q8", stream);
} else {
const long long pairs = cap_entries * (long long) FF;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<dpct_kernel_name<class swiglu_kernel_90849f>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((pairs + THREADS - 1) / THREADS)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
swiglu_kernel(gate_up, pairs);
});
}
check("moe_grouped_s2/swiglu", stream);
if (x_scales != nullptr) quantize_q8_0_scaled(gate_up, h_q8_0, h_scales, cap_entries * (int64_t) FF, stream);
else quantize_q8_0(gate_up, h_q8_0, cap_entries * (int64_t) FF, stream);
}
{
const dpct::dim3 grid((unsigned)(H / D_ROWS), (unsigned)cap_groups);
const float* hs = x_scales != nullptr ? h_scales : nullptr;
if (fast)
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<
dpct_kernel_name<class down_grouped_t_kernel_12b16d>>(
sycl::nd_range<3>(grid * sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
down_grouped_t_kernel(grp_ptr, grp_start, n_groups,
ent_dst, h_q8_0, hs, out);
});
} else {
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<
dpct_kernel_name<class down_grouped_kernel_ea4417>>(
sycl::nd_range<3>(grid * sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
down_grouped_kernel(grp_ptr, grp_start, n_groups,
ent_dst, h_q8_0, hs, out);
});
}
check("moe_grouped_s2/down", stream);
}
}
void moe_hit_add(float* parts, const float* hit_out, const int32_t* dst, const int32_t* count, int64_t cap,
int64_t n_embd, void* stream) {
if (cap <= 0) return;
const dpct::dim3 grid(
(unsigned)((n_embd + 255) / 256 < 8 ? (n_embd + 255) / 256 : 8),
(unsigned)cap);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
strata::q_of(stream)
->parallel_for<dpct_kernel_name<class add_hits_kernel_50cbcb>>(
sycl::nd_range<3>(grid * sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
add_hits_kernel(parts, hit_out, dst, count, (int)n_embd);
});
}
check("moe_hit_add", stream);
}
void moe_hit_grouped_s2_cpu_order(const uint8_t *blob_base,
const int32_t *slot_index,
const int32_t *dst_index, int64_t n_hits,
int64_t blob_bytes, const uint8_t *x_q8_0,
void *scratch, float *out, void *stream,
const float *x_scales,
float *gate_up_trace) try {
if (n_hits <= 0) return;
if (x_scales == nullptr) {
std::fprintf(stderr, "moe_hit_grouped_s2_cpu_order requires fp32 activation scales\n");
std::exit(1);
}
dpct::queue_ptr cs = strata::q_of(stream);
const uint64_t gu_bytes = ((uint64_t) n_hits * 2 * FF * 4 + 15) & ~15ull;
const uint64_t q8_bytes = ((uint64_t) n_hits * (FF / 32) * 34 + 15) & ~15ull;
const uint64_t scale_bytes = ((uint64_t) n_hits * (FF / 32) * 4 + 15) & ~15ull;
float* gu = (float*) scratch;
uint8_t* hq = (uint8_t*) scratch + gu_bytes;
float* hs = (float*) (hq + q8_bytes);
float* hh = (float*) ((uint8_t*) hs + scale_bytes);
float* xh = (float*) ((uint8_t*) hh + scale_bytes);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->submit([&](sycl::handler &cgh) {
auto H_ct3 = H / 32;
cgh.parallel_for<
dpct_kernel_name<class activation_correction_kernel_ca15a9>>(
sycl::nd_range<3>(
sycl::range(1, 1, (H / 32 + THREADS - 1) / THREADS) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
activation_correction_kernel(x_q8_0, x_scales, xh, H_ct3);
});
});
}
check("cpu_order/input_correction", stream);
const int rows_per_block = THREADS / 8;
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<
dpct_kernel_name<class cpu_order_projection_kernel_2fbdd9,
dpct_kernel_scalar<false>>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((n_hits * 2 * FF + rows_per_block - 1) /
rows_per_block)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
cpu_order_projection_kernel<false>(
blob_base, slot_index, dst_index, blob_bytes, x_q8_0,
x_scales, xh, gu, (int)n_hits);
});
}
check("cpu_order/gate_up", stream);
if (gate_up_trace != nullptr &&
/*
DPCT1124: cudaMemcpyAsync is migrated to asynchronous memcpy API.
While the origin API might be synchronous, it depends on the type of
operand memory, so you may need to call wait() on event return by memcpy
API to ensure synchronization behavior.
*/
DPCT_CHECK_ERROR(cs->memcpy(
gate_up_trace, gu, (size_t)n_hits * 2 * FF * sizeof(float))) != 0) {
std::fprintf(stderr, "cpu_order/gate_up_trace copy failed\n");
std::exit(1);
}
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->submit([&](sycl::handler &cgh) {
auto int_n_hits_FF_ct1 = (int)n_hits * FF;
cgh.parallel_for<
dpct_kernel_name<class cpu_order_swiglu_kernel_4d84ce>>(
sycl::nd_range<3>(
sycl::range(
1, 1,
(unsigned)((n_hits * FF + THREADS - 1) / THREADS)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
cpu_order_swiglu_kernel(gu, int_n_hits_FF_ct1);
});
});
}
check("cpu_order/swiglu", stream);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->submit([&](sycl::handler &cgh) {
auto int_n_hits_FF_ct4 = (int)n_hits * (FF / 32);
cgh.parallel_for<
dpct_kernel_name<class cpu_order_quantize_kernel_7d1393>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((n_hits * (FF / 32) + THREADS - 1) /
THREADS)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
cpu_order_quantize_kernel(gu, hq, hs, hh,
int_n_hits_FF_ct4);
});
});
}
check("cpu_order/intermediate_quantize", stream);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
cs->parallel_for<
dpct_kernel_name<class cpu_order_projection_kernel_2e007a,
dpct_kernel_scalar<true>>>(
sycl::nd_range<3>(
sycl::range(1, 1,
(unsigned)((n_hits * H + rows_per_block - 1) /
rows_per_block)) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(32)]] {
cpu_order_projection_kernel<true>(blob_base, slot_index,
dst_index, blob_bytes, hq, hs,
hh, out, (int)n_hits);
});
}
check("cpu_order/down", stream);
}
catch (sycl::exception const &exc) {
std::cerr << exc.what() << "Exception caught at file:" << __FILE__
<< ", line:" << __LINE__ << std::endl;
std::exit(1);
}
} // namespace strata::kernels