Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/prefill/moe_fused.cu - see include/strata/prefill/moe_fused.hpp (#136: the Q2_0 pack's prompt experts on
// Strata's own int8 tensor-core kernels, with the grouping on the GPU).
//
// The arithmetic. A Q2_0 block is 64 weights w = d_w * (q - 1), q in 0..3, four codes per byte, weight j of a block
// in byte j / 4 at bits 2 * (j % 4). An activation block of 32 values is x = d_x * a with a = round(x / d_x) in
// -127..127. Over one 32-value half-block
// sum w x = d_w * d_x * (sum q a - sum a),
// so the int8 product takes the codes as they are (0..3 is a valid s8) and the offset is the activations' own sum,
// stored with the block. The dot product (|sum q a| <= 32 * 3 * 127 < 2^22) becomes a float exactly with one
// integer add: as_float(0x4B400000 + dot) = 1.5 * 2^23 + dot, and the block stores c = -(1.5 * 2^23 + sum a), so
// as_float(0x4B400000 + dot) + c = dot - sum a with no rounding. Per output and 64-block that is
// acc += d_w * (d_x0 * t0 + d_x1 * t1) (t = dot - sum a of each half)
// - two integer adds, two float adds and three multiply-adds.
//
// The K order. mma.sync m16n8k32 takes four consecutive k of a row (A) or column (B) per register. Four consecutive
// weights are one code byte; four codes of one register instead come from the same 2-bit field of four consecutive
// bytes - (word >> 2t) & 0x03030303 holds weights t, t+4, t+8, t+12 of a 16-weight word. So the kernels use that order
// for the k of a half-block: mma k index 16h + 4t + j stands for weight (and activation) 16h + t + 4j. The
// activations are stored in it (perm32), and a lane's two B registers (h = 0 and 1) are adjacent: one 8-byte load.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/sycl_queue.hpp"
#include "strata/prefill/moe_fused.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <mutex>
#include <cmath>
namespace strata::prefill::fused {
namespace {
void ck(dpct::err0 e, const char *what) {
}
constexpr int AB = 80; // activation bytes per 64 values: 64 codes, then {d0, c0, d1, c1}
constexpr int MAGIC = 0x4B400000; // the bits of 1.5 * 2^23
constexpr float MAGICF = 12582912.0f;
// The Strata Q2_0 blob (moe_mmq.cu strata_q2_kernel, kernels.cu blob_dequant_kernel): gate/up codes [1280][640 B]
// (rows interleaved: gate of feature f at row 2f, its up at 2f + 1), down codes [2560][160 B], gate/up scales
// [1280][40] fp16, down scales [2560][10] fp16.
constexpr size_t O_GU_CODES = 0, O_D_CODES = (size_t) 1280 * 640, O_GU_SC = O_D_CODES + (size_t) 2560 * 160,
O_D_SC = O_GU_SC + (size_t) 1280 * 40 * 2;
constexpr int THREADS = 512; // 16 warps: 4 along the weight rows (64 each) x 4 along the tile rows (16 each)
constexpr int WROWS = 256; // weight rows per work item: 128 features of gate/up, or 256 outputs of down
constexpr int STAGES = 4; // cp.async pipeline depth, one 64-weight block of K per stage
constexpr int STAGE_BYTES = WROWS * 16 + kTileRows * AB;
constexpr size_t smem_bytes(bool gu) {
return (size_t) WROWS * (gu ? 40 : 10) * 2 + STAGES * STAGE_BYTES + kTileRows * 4;
}
// the stored position of value k of a 32-value half-block (see "The K order")
__dpct_inline__ int perm32(int k) {
return 8 * (k & 3) + 4 * (k >> 4) + ((k & 15) >> 2);
}
// where value k of a 32-value half-block is stored: perm32 for the CUDA kernel; on HIP the gfx11 WMMA kernel's order
// (value 4b + t of a 16-value word at 4t + b: see expert_w11_kernel)
__dpct_inline__ int act_pos(int k) {
#if defined(__HIPCC__)
return (k & 16) | ((k & 3) << 2) | ((k >> 2) & 3);
#else
return perm32(k);
#endif
}
struct Tables {
int32_t *cnt, *off, *fill, *ts; // per expert: rows, first row (E + 1), placement cursor, first tile (E + 1)
sycl::int2 *tiles; // {expert, first row}, in expert order
};
size_t tiles_at(int n_expert) { return ((size_t) (4 * n_expert + 2) * 4 + 15) / 16 * 16; }
Tables tables(void* scratch, int n_expert) {
int32_t* p = (int32_t*) scratch;
return {p, p + n_expert, p + 2 * n_expert + 1, p + 3 * n_expert + 1,
(sycl::int2 *)((uint8_t *)scratch + tiles_at(n_expert))};
}
// ---- activations: one warp per 64 values
__dpct_inline__ void quant_act_kernel(const float *__restrict__ x, int64_t nblk,
uint8_t *__restrict__ xa) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int64_t w =
(int64_t)item_ct1.get_group(2) * (item_ct1.get_local_range(2) / 32) +
item_ct1.get_local_id(2) / 32;
const int lane = item_ct1.get_local_id(2) & 31;
if (w >= nblk) return;
const float v0 = x[w * 64 + lane], v1 = x[w * 64 + 32 + lane];
float a0 = sycl::fabs(v0), a1 = sycl::fabs(v1);
#pragma unroll
for (int o = 16; o > 0; o >>= 1) {
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
a0 = sycl::fmax(
a0, dpct::experimental::permute_sub_group_by_xor(
0xffffffffu,
sycl::ext::oneapi::this_work_item::get_sub_group(), a0, o));
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
a1 = sycl::fmax(
a1, dpct::experimental::permute_sub_group_by_xor(
0xffffffffu,
sycl::ext::oneapi::this_work_item::get_sub_group(), a1, o));
}
const int q0 = a0 > 0.0f ? sycl::vec<float, 1>{(v0 * (127.0f / a0))}
.convert<int, sycl::rounding_mode::rte>()[0]
: 0;
const int q1 = a1 > 0.0f ? sycl::vec<float, 1>{(v1 * (127.0f / a1))}
.convert<int, sycl::rounding_mode::rte>()[0]
: 0;
int s0 = q0, s1 = q1;
#pragma unroll
for (int o = 16; o > 0; o >>= 1) {
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
s0 += dpct::experimental::permute_sub_group_by_xor(
0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(), s0,
o);
/*
DPCT1108: '__shfl_xor_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
s1 += dpct::experimental::permute_sub_group_by_xor(
0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(), s1,
o);
}
uint8_t* out = xa + w * AB;
out[act_pos(lane)] = (uint8_t) (int8_t) q0;
out[32 + act_pos(lane)] = (uint8_t) (int8_t) q1;
if (lane == 0)
*(sycl::float4 *)(out + 64) =
sycl::float4(a0 / 127.0f, -(MAGICF + (float)s0), a1 / 127.0f,
-(MAGICF + (float)s1));
}
// ---- grouping: counts, offsets and tiles, placement
__dpct_inline__ void count_kernel(const int32_t *__restrict__ ids, int64_t n,
int E, int32_t *__restrict__ cnt,
uint8_t *dpct_local) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
auto hist = (int32_t *)dpct_local;
#pragma unroll
for (int e = item_ct1.get_local_id(2); e < E;
e += item_ct1.get_local_range(2)) hist[e] = 0;
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
performance if there is no access to global memory.
*/
item_ct1.barrier();
for (int64_t i =
(int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
i < n; i += (int64_t)item_ct1.get_group_range(2) *
item_ct1.get_local_range(2)) {
const int e = ids[i];
if ((unsigned)e < (unsigned)E)
dpct::atomic_fetch_add<sycl::access::address_space::generic_space>(
&hist[e], 1);
}
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
performance if there is no access to global memory.
*/
item_ct1.barrier();
#pragma unroll
for (int e = item_ct1.get_local_id(2); e < E;
e += item_ct1.get_local_range(2))
if (hist[e])
dpct::atomic_fetch_add<sycl::access::address_space::generic_space>(
&cnt[e], hist[e]);
}
// exclusive prefix sum over the block (blockDim a multiple of 32); *total = the block's sum
inline int block_excl_sum(int v, int *total, int *sh) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int lane = item_ct1.get_local_id(2) & 31,
w = item_ct1.get_local_id(2) >> 5,
nw = item_ct1.get_local_range(2) >> 5;
int x = v;
#pragma unroll
for (int o = 1; o < 32; o <<= 1) {
/*
DPCT1108: '__shfl_up_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
const int y = dpct::experimental::shift_sub_group_right(
0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(), x,
o);
if (lane >= o) x += y;
}
if (lane == 31) sh[w] = x;
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
performance if there is no access to global memory.
*/
item_ct1.barrier();
if (w == 0) {
int s = lane < nw ? sh[lane] : 0;
#pragma unroll
for (int o = 1; o < 32; o <<= 1) {
/*
DPCT1108: '__shfl_up_sync' was migrated with the experimental
feature masked sub_group function which may not be supported by all
compilers or runtimes. You may need to adjust the code.
*/
const int y = dpct::experimental::shift_sub_group_right(
0xffffffffu, sycl::ext::oneapi::this_work_item::get_sub_group(),
s, o);
if (lane >= o) s += y;
}
sh[lane] = s;
}
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
performance if there is no access to global memory.
*/
item_ct1.barrier();
const int r = x - v + (w > 0 ? sh[w - 1] : 0);
*total = sh[nw - 1];
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
performance if there is no access to global memory.
*/
item_ct1.barrier(); // sh is reused by the next call
return r;
}
// one block, a thread per expert (E <= 1024)
__dpct_inline__ void scan_kernel(Tables tb, int E) {
auto &sh = *sycl::ext::oneapi::group_local_memory_for_overwrite<int[32]>(
sycl::ext::oneapi::this_work_item::get_work_group<3>());
const int e =
sycl::ext::oneapi::this_work_item::get_nd_item<3>().get_local_id(2);
const int c = e < E ? tb.cnt[e] : 0, nt = (c + kTileRows - 1) / kTileRows;
int rows = 0, tiles = 0;
const int o = block_excl_sum(c, &rows, sh), to = block_excl_sum(nt, &tiles, sh);
if (e < E) {
tb.off[e] = o;
tb.fill[e] = o;
tb.ts[e] = to;
#pragma unroll
for (int i = 0; i < nt; ++i)
tb.tiles[to + i] = sycl::int2(e, o + i * kTileRows);
}
if (e == 0) {
tb.off[E] = rows;
tb.ts[E] = tiles;
}
}
__dpct_inline__ void place_kernel(const int32_t *__restrict__ ids, int64_t n,
int k, int E, Tables tb,
int32_t *__restrict__ slot,
int32_t *__restrict__ src) {
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
const int64_t i =
(int64_t)item_ct1.get_group(2) * item_ct1.get_local_range(2) +
item_ct1.get_local_id(2);
if (i >= n) return;
const int e = ids[i];
if ((unsigned) e >= (unsigned) E) { slot[i] = 0; return; } // (the router never writes one)
const int p =
dpct::atomic_fetch_add<sycl::access::address_space::generic_space>(
&tb.fill[e], 1);
slot[i] = p;
src[p] = (int32_t) (i / k);
}
// ---- the expert products
#if defined(DPCT_COMPATIBILITY_TEMP) && DPCT_COMPATIBILITY_TEMP >= 800
__dpct_inline__ void cp16(void *dst, const void *src) {
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"(
(unsigned)__cvta_generic_to_shared(dst)),
"l"(src));
#else
*(((uint32_t *)(uintptr_t)(unsigned)__cvta_generic_to_shared(dst))) =
*(((uint32_t *)(uintptr_t)src));
if (16 > 4)
*(((uint32_t *)(uintptr_t)(unsigned)__cvta_generic_to_shared(dst)) +
1) = *(((uint32_t *)(uintptr_t)src) + 1);
if (16 > 8)
*(((uint32_t *)(uintptr_t)(unsigned)__cvta_generic_to_shared(dst)) +
2) = *(((uint32_t *)(uintptr_t)src) + 2);
if (16 > 12)
*(((uint32_t *)(uintptr_t)(unsigned)__cvta_generic_to_shared(dst)) +
3) = *(((uint32_t *)(uintptr_t)src) + 3);
#endif
}
__dpct_inline__ void cp4(void *dst, const void *src) {
/*
DPCT1053: Migration of device assembly code is not supported.
*/
asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"(
(unsigned)__cvta_generic_to_shared(dst)),
"l"(src));
}
/*
DPCT1026: The call to "cp.async.commit_group;
" was removed because current "cp.async" is migrated to synchronous copy
operation. You may need to adjust the code to tune the performance.
*/
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
__device__ __forceinline__ void cp_commit() {
asm volatile("cp.async.commit_group;\n" ::);
#else
__device__ __forceinline__ void cp_commit() {
#endif }
/*
DPCT1026: The call to "cp.async.wait_group %0;
" was removed because current "cp.async" is migrated to synchronous copy
operation. You may need to adjust the code to tune the performance.
*/
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
template <int N> __device__ __forceinline__ void cp_wait() {
asm volatile("cp.async.wait_group %0;\n" ::"n"(N));
#else
template <int N> __device__ __forceinline__ void cp_wait() {
#endif }
// d = A (16 x 32 s8, row) * B (32 x 8 s8, col), int32, from zero
__dpct_inline__ void mma_s8(int (&d)[4], const uint32_t (&a)[4], uint32_t b0,
uint32_t b1) {
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 "
"{%0,%1,%2,%3}, "
"{%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};\n"
: "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1),
"r"(0));
#else
{
volatile void *d_mat_frag_ct1[4] = {&d[0], &d[1], &d[2], &d[3]};
sycl::vec<uint32_t, 4> a_mat_frag_ct1(a[0], a[1], a[2], a[3]);
sycl::vec<uint32_t, 2> b_mat_frag_ct1(b0, b1);
sycl::vec<int32_t, 4> c_mat_frag_ct1(0, 0, 0, 0);
dpct::experimental::matrix::mma<16, 8, 32, int8_t, int32_t>(
reinterpret_cast<volatile void **>(d_mat_frag_ct1),
&a_mat_frag_ct1, &b_mat_frag_ct1, &c_mat_frag_ct1);
}
#endif
}
#endif
// One work item = (tile of up to 64 routed rows of expert e, a block of 256 weight rows); a persistent grid walks
// the batch's items (their count is on the device). GU: gate/up rows of 128 features against the rows' tokens'
// activations (`act` per token, 2560 values), SwiGLU, H to int8 per 32 features into `out` (per row, 640 values).
// !GU: down rows (256 outputs) against H (`act` per row), FP32 into `dm` per row.
// Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, its gate rows as mma rows 0-7 and
// its up rows as 8-15, so a lane holds gate and up of one feature: SwiGLU in registers), tile rows 16 wt.. as two n8.
template <bool GU>
/*
DPCT1110: The total declared local variable size in device function
expert_kernel exceeds 128 bytes and may cause high register pressure. Consult
with your hardware vendor to find the total register size available and adjust
the code, or use smaller sub-group size to avoid high register pressure.
*/
__dpct_inline__ void expert_kernel(
const Batch b, const Tables tb, const uint8_t *__restrict__ act,
const int32_t *__restrict__ src, uint8_t *__restrict__ out,
float *__restrict__ dm, uint8_t *dpct_local) {
#if defined(DPCT_COMPATIBILITY_TEMP) && DPCT_COMPATIBILITY_TEMP >= 800
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
constexpr int NKB = GU ? 40 : 10; // 64-weight blocks along K
constexpr int NFB = GU ? 5 : 10; // weight-row blocks per tile
constexpr int CODE_LD = GU ? 640 : 160; // code bytes per weight row
constexpr size_t O_CODES = GU ? O_GU_CODES : O_D_CODES, O_SC = GU ? O_GU_SC : O_D_SC;
constexpr int ACT_LD = NKB * AB; // bytes per activation row: 3200 (a token), 800 (a row's H)
constexpr unsigned M2 = 0x03030303u;
auto smem = (uint8_t *)dpct_local;
const sycl::half *wsc =
(const sycl::half *)smem; // [256][NKB] the work item's weight scales
uint8_t* stages = smem + WROWS * NKB * 2; // [STAGES] x {codes [256][16], act [64][AB]}
int* srow = (int*) (stages + STAGES * STAGE_BYTES); // [64] the activation row of each tile row
const int tid = item_ct1.get_local_id(2), lane = tid & 31, warp = tid >> 5,
g = lane >> 2, tig = lane & 3;
const int wf = warp & 3, nb0 = 16 * (warp >> 2);
int ra[4], rb[4]; // the local weight rows of mma rows g and g + 8
#pragma unroll
for (int i = 0; i < 4; ++i) {
if (GU) { ra[i] = 2 * (32 * wf + 8 * i + g); rb[i] = ra[i] + 1; }
else { ra[i] = 64 * wf + 16 * i + g; rb[i] = ra[i] + 8; }
}
const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
for (int w = item_ct1.get_group(2); w < nwork;
w += item_ct1.get_group_range(2)) {
const sycl::int2 tl = tb.tiles[t0 + w / NFB];
const int fb = w % NFB, e = tl.x(), row0 = tl.y(),
nrows = sycl::min(kTileRows, tb.off[e + 1] - row0);
const uint8_t* blob = b.blob[e - b.e0];
const int rbase = fb * WROWS;
/*
DPCT1118: SYCL group functions and algorithms must be encountered in
converged control flow. You may need to adjust the code.
*/
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
better performance if there is no access to global memory.
*/
item_ct1.barrier(); // the previous item is done with the buffers
if (tid < kTileRows) {
const int r =
row0 +
sycl::min(
tid,
nrows - 1); // rows past the tile's end repeat its last one
srow[tid] = GU ? src[r] : r;
}
for (int c = tid; c < WROWS * 5; c += THREADS) { // the scales: 5 x 16 B (GU) or 5 x 4 B a row
const int r = c / 5, q = c % 5;
if (GU) cp16(smem + r * 80 + q * 16, blob + O_SC + (size_t) (rbase + r) * 80 + q * 16);
else cp4(smem + r * 20 + q * 4, blob + O_SC + (size_t) (rbase + r) * 20 + q * 4);
}
/*
DPCT1118: SYCL group functions and algorithms must be encountered in
converged control flow. You may need to adjust the code.
*/
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
better performance if there is no access to global memory.
*/
item_ct1.barrier(); // srow
auto load = [&](int kb) {
uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
if (tid < WROWS) cp16(st + tid * 16, blob + O_CODES + (size_t) (rbase + tid) * CODE_LD + kb * 16);
const int a = tid - (THREADS - kTileRows * 5); // the last 320 threads: 64 rows x 5 x 16 B
if (a >= 0) {
const int r = a / 5, q = a % 5;
cp16(st + WROWS * 16 + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + kb * AB + q * 16);
}
};
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < NKB) load(s);
cp_commit();
}
// a warp whose rows are all past the tile's end only takes part in the loads
const bool on0 = nb0 < nrows, on1 = nb0 + 8 < nrows;
float acc[4][2][4];
#pragma unroll
for (int i = 0; i < 4; ++i)
#pragma unroll
for (int n = 0; n < 2; ++n)
#pragma unroll
for (int q = 0; q < 4; ++q) acc[i][n][q] = 0.0f;
for (int kb = 0; kb < NKB; ++kb) {
cp_wait<STAGES - 2>();
/*
DPCT1118: SYCL group functions and algorithms must be
encountered in converged control flow. You may need to adjust the
code.
*/
/*
DPCT1065: Consider replacing sycl::nd_item::barrier() with
sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
better performance if there is no access to global memory.
*/
item_ct1.barrier();
if (kb + STAGES - 1 < NKB) load(kb + STAGES - 1);
cp_commit();
if (!on0) continue;
const uint8_t* st = stages + (kb % STAGES) * STAGE_BYTES;
const uint8_t* sa = st + WROWS * 16;
sycl::uint2 bq[2][2]; // [n8 tile][half]: the B registers
sycl::float4 sc[2]
[2]; // [n8 tile][column 2 tig + c]: {d0, c0, d1, c1}
#pragma unroll
for (int n = 0; n < 2; ++n) {
if (n == 1 && !on1) break;
const uint8_t* r = sa + (nb0 + 8 * n + g) * AB + 8 * tig;
bq[n][0] = *(const sycl::uint2 *)r;
bq[n][1] = *(const sycl::uint2 *)(r + 32);
sc[n][0] =
*(const sycl::float4 *)(sa + (nb0 + 8 * n + 2 * tig) * AB +
64);
sc[n][1] =
*(const sycl::float4 *)(sa +
(nb0 + 8 * n + 2 * tig + 1) * AB +
64);
}
#pragma unroll
for (int i = 0; i < 4; ++i) {
const sycl::uint4 ca = *(const sycl::uint4 *)(st + ra[i] * 16),
cb = *(const sycl::uint4 *)(st + rb[i] * 16);
const float
da = sycl::vec<sycl::half, 1>(wsc[ra[i] * NKB + kb])
.convert<float,
sycl::rounding_mode::automatic>()[0],
db = sycl::vec<sycl::half, 1>(wsc[rb[i] * NKB + kb])
.convert<float,
sycl::rounding_mode::automatic>()[0];
const int s = 2 * tig;
const uint32_t a0[4] = {(ca.x() >> s) & M2, (cb.x() >> s) & M2,
(ca.y() >> s) & M2, (cb.y() >> s) & M2};
const uint32_t a1[4] = {(ca.z() >> s) & M2, (cb.z() >> s) & M2,
(ca.w() >> s) & M2, (cb.w() >> s) & M2};
#pragma unroll
for (int n = 0; n < 2; ++n) {
if (n == 1 && !on1) break;
int d0[4], d1[4];
mma_s8(d0, a0, bq[n][0].x(), bq[n][0].y());
mma_s8(d1, a1, bq[n][1].x(), bq[n][1].y());
#pragma unroll
for (int q = 0; q < 4; ++q) {
const sycl::float4 z = sc[n][q & 1];
const float u0 = sycl::bit_cast<float>(MAGIC + d0[q]) +
z.y(),
u1 = sycl::bit_cast<float>(MAGIC + d1[q]) +
z.w();
acc[i][n][q] = sycl::fma(
(float)(q < 2 ? da : db),
sycl::fma((float)(z.z()), (float)u1, z.x() * u0),
acc[i][n][q]);
}
}
}
}
if (!on0) continue;
if (GU) {
// SwiGLU of feature 32 wf + 8 i + g (lane g of m tile i), tile rows nb0 + 8 n + 2 tig + c; H to int8 per
// row over the warp's 32 features (= one 32-value half-block of the down product's K)
const int blk = 4 * fb + wf;
#pragma unroll
for (int n = 0; n < 2; ++n) {
if (n == 1 && !on1) break;
#pragma unroll
for (int c = 0; c < 2; ++c) {
float h[4], am = 0.0f;
#pragma unroll
for (int i = 0; i < 4; ++i) {
const float gt = acc[i][n][c], up = acc[i][n][2 + c];
h[i] = gt / (1.0f + sycl::native::exp(-gt)) * up;
am = sycl::fmax(am, sycl::fabs(h[i]));
}
#pragma unroll
/*
DPCT1108: '__shfl_xor_sync' was migrated with the
experimental feature masked sub_group function which may not
be supported by all compilers or runtimes. You may need to
adjust the code.
*/
for (int o = 4; o < 32; o <<= 1) am = sycl::fmax(
am,
dpct::experimental::permute_sub_group_by_xor(
0xffffffffu,
sycl::ext::oneapi::this_work_item::get_sub_group(),
am, o));
const float inv = am > 0.0f ? 127.0f / am : 0.0f;
int qv[4], sum = 0;
#pragma unroll
for (int i = 0; i < 4; ++i) {
qv[i] =
sycl::vec<float, 1>{(h[i] * inv)}
.convert<int, sycl::rounding_mode::rte>()[0];
sum += qv[i];
}
#pragma unroll
/*
DPCT1108: '__shfl_xor_sync' was migrated with the
experimental feature masked sub_group function which may not
be supported by all compilers or runtimes. You may need to
adjust the code.
*/
for (int o = 4; o < 32; o <<= 1) sum +=
dpct::experimental::permute_sub_group_by_xor(
0xffffffffu,
sycl::ext::oneapi::this_work_item::get_sub_group(),
sum, o);
const int r = nb0 + 8 * n + 2 * tig + c;
if (r < nrows) {
uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (blk >> 1) * AB;
const int hh = blk & 1;
#pragma unroll
for (int i = 0; i < 4; ++i) o[32 * hh + perm32(8 * i + g)] = (uint8_t) (int8_t) qv[i];
if (g == 0) * (sycl::float2 *)(o + 64 + 8 * hh) =
sycl::float2(am / 127.0f, -(MAGICF + (float)sum));
}
}
}
} else {
#pragma unroll
for (int i = 0; i < 4; ++i)
#pragma unroll
for (int n = 0; n < 2; ++n) {
if (n == 1 && !on1) break;
#pragma unroll
for (int q = 0; q < 4; ++q) {
const int r = nb0 + 8 * n + 2 * tig + (q & 1);
if (r < nrows) dm[(size_t) (row0 + r) * 2560 + rbase + (q < 2 ? ra[i] : rb[i])] = acc[i][n][q];
}
}
}
}
#endif
}
#if defined(__HIPCC__)
// ---- Aurora (S23): the same products on gfx11 (RDNA3 / RDNA3.5) matrix cores, v_wmma_i32_16x16x16_iu8 (wave32).
// Fragments (gfx11): A lane l holds the 16 k of row l % 16 (lanes 16..31 repeat lanes 0..15), B lane l the 16 k of
// column l % 16, C/D lane l holds D[2i + l / 16][l % 16], i = 0..7. A = 16 weight rows, B = 16 tile rows (tokens):
// a lane's eight results are eight weight rows of ONE tile row, whose activation scales it already holds.
// The K order of a 16-weight word (one k-step): A register t = (word >> 2t) & 0x03030303 holds weights t, t+4, t+8,
// t+12, so k index p = 4t + b stands for weight 4b + t; the activations are stored in that order (act_pos). The
// integer dot of each 32-value half (two k-steps) is exact and the epilogue is the CUDA kernel's, so the numbers are
// those of expert_kernel (another K order inside exact integer sums).
// No LDS staging: a wave reads its weight rows (16 B per row and 64-weight block) and its tile rows' activation
// blocks (80 B) straight from global memory (L2: an expert's rows are read by every tile, a token's by every weight
// block), one block ahead in registers; the work item's weight scales go to LDS once.
#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1150__) || defined(__gfx1151__)
#define STRATA_FUSED_W11 1
#else
#define STRATA_FUSED_W11 0
#endif
typedef int w11_i4 __attribute__((ext_vector_type(4)));
typedef int w11_i8 __attribute__((ext_vector_type(8)));
constexpr int W_ROWS = 128; // weight rows per work item (GU: 64 features = one H block)
constexpr int W_THREADS = 256; // 8 waves: 4 along the weight rows (32 each) x 2 along the tile (32)
__device__ __forceinline__ w11_i8 wmma_iu8(w11_i4 a, w11_i4 b, w11_i8 c) {
#if STRATA_FUSED_W11
return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(false, a, true, b, c, false);
#else
__builtin_trap();
return c;
#endif
}
template <bool GU>
__global__ void __launch_bounds__(W_THREADS)
expert_w11_kernel(const Batch b, const Tables tb, const uint8_t* __restrict__ act, const int32_t* __restrict__ src,
uint8_t* __restrict__ out, float* __restrict__ dm) {
#if STRATA_FUSED_W11
constexpr int NKB = GU ? 40 : 10; // 64-weight blocks along K
constexpr int NFB = (GU ? 1280 : 2560) / W_ROWS; // work items per tile
constexpr int CODE_LD = GU ? 640 : 160; // code bytes per weight row
constexpr size_t O_CODES = GU ? O_GU_CODES : O_D_CODES, O_SC = GU ? O_GU_SC : O_D_SC;
constexpr int ACT_LD = NKB * AB; // bytes per activation row
constexpr unsigned M2 = 0x03030303u;
__shared__ __align__(16) uint16_t wsc[W_ROWS * NKB]; // the item's weight scales (fp16), [row][kb]
__shared__ int srow[kTileRows];
__shared__ float hs[GU ? kTileRows : 1][GU ? 65 : 1]; // GU: H of the item's 64 features, [tile row][feature]
const int tid = threadIdx.x, lane = tid & 31, wave = tid >> 5;
const int wm = wave & 3, wn = wave >> 2, l16 = lane & 15, hi = lane >> 4;
const int t0 = tb.ts[b.e0], nwork = (tb.ts[b.e1] - t0) * NFB;
for (int w = blockIdx.x; w < nwork; w += gridDim.x) {
const int2 tl = tb.tiles[t0 + w / NFB];
const int fb = w % NFB, e = tl.x, row0 = tl.y, nrows = min(kTileRows, tb.off[e + 1] - row0);
const uint8_t* blob = b.blob[e - b.e0];
const int rbase = fb * W_ROWS;
__syncthreads(); // the previous item is done with srow / wsc / hs
if (tid < kTileRows) {
const int r = row0 + min(tid, nrows - 1); // rows past the tile's end repeat its last one
srow[tid] = GU ? src[r] : r;
}
{
const uint4* s4 = reinterpret_cast<const uint4*>(blob + O_SC + (size_t) rbase * NKB * 2);
for (int i = tid; i < W_ROWS * NKB / 8; i += W_THREADS) reinterpret_cast<uint4*>(wsc)[i] = s4[i];
}
__syncthreads();
const bool on = 32 * wn < nrows; // a wave past the tile's rows only stages
float acc[2][2][8];
#pragma unroll
for (int mt = 0; mt < 2; ++mt)
#pragma unroll
for (int nt = 0; nt < 2; ++nt)
#pragma unroll
for (int i = 0; i < 8; ++i) acc[mt][nt][i] = 0.0f;
if (on) {
const uint8_t* arow[2];
const uint8_t* brow[2];
#pragma unroll
for (int mt = 0; mt < 2; ++mt) arow[mt] = blob + O_CODES + (size_t) (rbase + 32 * wm + 16 * mt + l16) * CODE_LD;
#pragma unroll
for (int nt = 0; nt < 2; ++nt) brow[nt] = act + (size_t) srow[32 * wn + 16 * nt + l16] * ACT_LD;
uint4 ac[2], bq[2][4];
float4 bs[2];
auto fetch = [&](int kb, uint4 (&a)[2], uint4 (&q)[2][4], float4 (&s)[2]) {
#pragma unroll
for (int mt = 0; mt < 2; ++mt) a[mt] = *reinterpret_cast<const uint4*>(arow[mt] + kb * 16);
#pragma unroll
for (int nt = 0; nt < 2; ++nt) {
const uint4* p = reinterpret_cast<const uint4*>(brow[nt] + kb * AB);
q[nt][0] = p[0]; q[nt][1] = p[1]; q[nt][2] = p[2]; q[nt][3] = p[3];
s[nt] = *reinterpret_cast<const float4*>(brow[nt] + kb * AB + 64);
}
};
fetch(0, ac, bq, bs);
for (int kb = 0; kb < NKB; ++kb) {
uint4 nac[2], nbq[2][4];
float4 nbs[2];
if (kb + 1 < NKB) fetch(kb + 1, nac, nbq, nbs);
w11_i8 c[2][2][2]; // [half][mt][nt]
#pragma unroll
for (int h = 0; h < 2; ++h)
#pragma unroll
for (int mt = 0; mt < 2; ++mt)
#pragma unroll
for (int nt = 0; nt < 2; ++nt) c[h][mt][nt] = w11_i8{0, 0, 0, 0, 0, 0, 0, 0};
#pragma unroll
for (int ks = 0; ks < 4; ++ks) { // k-step = 16-weight word ks of the block
w11_i4 A[2];
#pragma unroll
for (int mt = 0; mt < 2; ++mt) {
const unsigned wd = ks == 0 ? ac[mt].x : ks == 1 ? ac[mt].y : ks == 2 ? ac[mt].z : ac[mt].w;
A[mt] = w11_i4{(int) (wd & M2), (int) ((wd >> 2) & M2), (int) ((wd >> 4) & M2),
(int) ((wd >> 6) & M2)};
}
#pragma unroll
for (int nt = 0; nt < 2; ++nt) {
const uint4 q = bq[nt][ks];
const w11_i4 B = w11_i4{(int) q.x, (int) q.y, (int) q.z, (int) q.w};
#pragma unroll
for (int mt = 0; mt < 2; ++mt) c[ks >> 1][mt][nt] = wmma_iu8(A[mt], B, c[ks >> 1][mt][nt]);
}
}
// the epilogue: per 64-weight block acc += d_w * (d_x0 * (dot0 - sum0) + d_x1 * (dot1 - sum1))
#pragma unroll
for (int mt = 0; mt < 2; ++mt) {
float dw[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
const int m = 32 * wm + 16 * mt + 2 * i + hi;
dw[i] = __half2float(__ushort_as_half(wsc[m * NKB + kb]));
}
#pragma unroll
for (int nt = 0; nt < 2; ++nt) {
const float4 z = bs[nt];
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float u0 = __int_as_float(MAGIC + c[0][mt][nt][i]) + z.y;
const float u1 = __int_as_float(MAGIC + c[1][mt][nt][i]) + z.w;
acc[mt][nt][i] = fmaf(dw[i], fmaf(z.z, u1, z.x * u0), acc[mt][nt][i]);
}
}
}
if (kb + 1 < NKB) {
#pragma unroll
for (int mt = 0; mt < 2; ++mt) ac[mt] = nac[mt];
#pragma unroll
for (int nt = 0; nt < 2; ++nt) {
bs[nt] = nbs[nt];
#pragma unroll
for (int j = 0; j < 4; ++j) bq[nt][j] = nbq[nt][j];
}
}
}
}
if constexpr (GU) {
// SwiGLU: row 2f is feature f's gate (lanes 0..15), 2f + 1 its up (lanes 16..31) - the same (mt, i)
if (on) {
#pragma unroll
for (int mt = 0; mt < 2; ++mt)
#pragma unroll
for (int nt = 0; nt < 2; ++nt)
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float up = __shfl_xor_sync(0xffffffffu, acc[mt][nt][i], 16);
const float gt = acc[mt][nt][i];
if (hi == 0) hs[32 * wn + 16 * nt + l16][16 * wm + 8 * mt + i] = gt / (1.0f + __expf(-gt)) * up;
}
}
__syncthreads();
// H block fb (features 64 fb .. 64 fb + 63) to int8: a thread per (tile row, half)
if (tid < 2 * kTileRows) {
const int r = tid >> 1, hh = tid & 1;
if (r < nrows) {
float am = 0.0f;
#pragma unroll 8
for (int j = 0; j < 32; ++j) am = fmaxf(am, fabsf(hs[r][32 * hh + j]));
const float inv = am > 0.0f ? 127.0f / am : 0.0f;
uint8_t* o = out + (size_t) (row0 + r) * (10 * AB) + (size_t) fb * AB;
int sum = 0;
uint32_t wd[8] = {};
#pragma unroll
for (int j = 0; j < 32; ++j) {
const int qv = __float2int_rn(hs[r][32 * hh + j] * inv);
sum += qv;
const int p = act_pos(j);
wd[p >> 2] |= (uint32_t) (uint8_t) (int8_t) qv << (8 * (p & 3));
}
uint4* o4 = reinterpret_cast<uint4*>(o + 32 * hh);
o4[0] = make_uint4(wd[0], wd[1], wd[2], wd[3]);
o4[1] = make_uint4(wd[4], wd[5], wd[6], wd[7]);
*reinterpret_cast<float2*>(o + 64 + 8 * hh) = make_float2(am / 127.0f, -(MAGICF + (float) sum));
}
}
} else if (on) {
#pragma unroll
for (int mt = 0; mt < 2; ++mt)
#pragma unroll
for (int nt = 0; nt < 2; ++nt) {
const int r = 32 * wn + 16 * nt + l16;
if (r < nrows) {
float* d = dm + (size_t) (row0 + r) * 2560 + rbase + 32 * wm + 16 * mt + hi;
#pragma unroll
for (int i = 0; i < 8; ++i) d[2 * i] = acc[mt][nt][i];
}
}
}
}
#else
__builtin_trap();
#endif
}
#endif // __HIPCC__
unsigned blocks(int64_t n, int per) { return (unsigned) ((n + per - 1) / per); }
struct DevInfo {
bool done = false;
int cc = 0, sms = 0, occ_gu = 0, occ_d = 0;
};
std::mutex g_mu;
DevInfo g_dev[32];
#if defined(__HIPCC__)
// Resident blocks per multiprocessor for the gfx11 WMMA prompt-expert kernels' persistent grids. HIP counts a gfx11
// WGP (two CUs, which these kernels use as one in the default WGP mode) as one multiprocessor, and the occupancy query
// answers 1 block for these kernels where 2 run side by side - so the grid was half the GPU's room. gfx1151 (20
// WGPs), 8192 tokens x top 10, IQ3_S / IQ4_NL, one layer: 1 block per WGP 60.7 ms, 2 44.7, 3 49.9, 4 45.2 (the same
// work items and arithmetic: the results do not change). STRATA_PF_OCC=N sets the blocks per WGP (an experiment knob).
static int wgp_blocks(int occ) {
static const int env = [] {
const char* v = std::getenv("STRATA_PF_OCC");
return v != nullptr ? std::atoi(v) : 0;
}();
return env > 0 ? env : 2 * std::max(occ, 1);
}
#endif
const DevInfo &dev_info() try {
int dev = 0;
dev = dpct::get_current_device_id();
std::lock_guard<std::mutex> lk(g_mu);
DevInfo& d = g_dev[dev & 31];
if (d.done) return d;
d.done = true;
#if defined(STRATA_HIP_GFX906)
// gfx906 reports compute capability 9.0 through HIP, but the mma.sync bodies above are CUDA sm_80+ only and
// empty in a hipcc build: never available here (the prompt path keeps its own expert GEMMs)
return d;
#elif defined(__HIPCC__)
{ // gfx11 (RDNA3 / RDNA3.5) with this build's code for it: the WMMA kernels
cudaDeviceProp prop;
cudaDeviceGetAttribute(&d.sms, cudaDevAttrMultiProcessorCount, dev);
hipFuncAttributes fa{};
if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess || std::strncmp(prop.gcnArchName, "gfx11", 5) != 0 ||
hipFuncGetAttributes(&fa, reinterpret_cast<const void*>(expert_w11_kernel<true>)) != hipSuccess) {
cudaGetLastError();
return d;
}
if (hipOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_gu, expert_w11_kernel<true>, W_THREADS, 0) != hipSuccess ||
hipOccupancyMaxActiveBlocksPerMultiprocessor(&d.occ_d, expert_w11_kernel<false>, W_THREADS, 0) != hipSuccess)
d.occ_gu = d.occ_d = 0;
d.cc = d.occ_gu >= 1 && d.occ_d >= 1 ? 80 : 0;
cudaGetLastError();
return d;
}
#else
int major = 0, minor = 0;
major = dpct::get_device(dev).get_major_version();
minor = dpct::get_device(dev).get_minor_version();
d.sms = dpct::get_device(dev).get_max_compute_units();
d.cc = 10 * major + minor;
if (d.cc < 80) return d;
// the device code of this card's arch must have the kernels (a build without an sm_80+ target has an empty body)
dpct::kernel_function_info fa{};
if (DPCT_CHECK_ERROR(dpct::get_kernel_function_info(
&fa, (const void *)expert_kernel<true>)) != 0 ||
fa.ptxVersion < 80) {
/*
DPCT1026: The call to cudaGetLastError was removed because this
functionality is redundant in SYCL.
*/
d.cc = 0;
return d;
}
/*
DPCT1027: The call to cudaFuncSetAttribute was replaced with 0 because
SYCL currently does not support corresponding setting.
*/
ck(0, "smem gu");
/*
DPCT1027: The call to cudaFuncSetAttribute was replaced with 0 because
SYCL currently does not support corresponding setting.
*/
ck(0, "smem down");
/*
DPCT1111: Please verify the input arguments of
"dpct::experimental::calculate_max_active_wg_per_xecore" base on the target
function "expert_kernel<true>".
*/
dpct::experimental::calculate_max_active_wg_per_xecore(
&d.occ_gu, THREADS,
smem_bytes(true) +
dpct_placeholder /* total share local memory size */);
/*
DPCT1111: Please verify the input arguments of
"dpct::experimental::calculate_max_active_wg_per_xecore" base on the target
function "expert_kernel<false>".
*/
dpct::experimental::calculate_max_active_wg_per_xecore(
&d.occ_d, THREADS,
smem_bytes(false) +
dpct_placeholder /* total share local memory size */);
if (d.occ_gu < 1 || d.occ_d < 1) d.cc = 0; // does not fit this card: the MMQ path
/*
DPCT1026: The call to cudaGetLastError was removed because this
functionality is redundant in SYCL.
*/
#endif
return d;
}
catch (sycl::exception const &exc) {
std::cerr << exc.what() << "Exception caught at file:" << __FILE__
<< ", line:" << __LINE__ << std::endl;
std::exit(1);
}
} // namespace
bool built() { return true; }
bool available() { return dev_info().cc >= 80; }
// 0.1.36: on by default for the Q2_0 pack (+13-23% prompts on an RTX 5070; teacher-forced against the FP16 path as
// close as MMQ at 8K and closer at 32K: phaseA-tests s18-tf); STRATA_PF_FUSED=0 keeps MMQ. The native packs' kernels
// (moe_fused_iq) stay opt-in: STRATA_PF_FUSED=1 (requested()).
bool enabled() {
static const bool env = [] {
const char* v = std::getenv("STRATA_PF_FUSED");
#if defined(__HIPCC__)
return v != nullptr && v[0] == '1'; // Aurora: the gfx11 WMMA kernels are opt-in (STRATA_PF_FUSED=1)
#else
return v == nullptr || v[0] != '0';
#endif
}();
return env && available();
}
bool requested() {
static const bool env = [] {
const char* v = std::getenv("STRATA_PF_FUSED");
return v != nullptr && v[0] == '1';
}();
return env && available();
}
size_t act_bytes(int64_t rows, int64_t cols) { return (size_t) rows * (size_t) (cols / 64) * AB; }
size_t group_bytes(int64_t n, int n_expert) {
return tiles_at(n_expert) +
((size_t)(n + kTileRows - 1) / kTileRows + (size_t)n_expert) *
sizeof(sycl::int2);
}
void quantize_act(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
if (rows <= 0) return;
const int64_t nblk = rows * (cols / 64);
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
strata::q_of(stream)
->parallel_for<dpct_kernel_name<class quant_act_kernel_da6c97>>(
sycl::nd_range<3>(sycl::range(1, 1, blocks(nblk, 8)) *
sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
quant_act_kernel(x, nblk, (uint8_t *)xa);
});
}
/*
DPCT1010: SYCL uses exceptions to report errors and does not use the
error codes. The cudaGetLastError function call was replaced with 0. You
need to rewrite this code.
*/
ck(0, "quantize_act");
}
void group(const int32_t* ids, int64_t n, int k, int n_expert, void* scratch, int32_t* slot, int32_t* src,
void* stream) {
if (n_expert < 1 || n_expert > 1024) {
std::fprintf(stderr, "prefill fused experts: %d experts (the grouping takes 1 to 1024)\n", n_expert);
std::exit(1);
}
const dpct::queue_ptr s = strata::q_of(stream);
const Tables tb = tables(scratch, n_expert);
ck(DPCT_CHECK_ERROR(s->memset(tb.cnt, 0, (size_t)n_expert * 4)),
"group memset");
if (n > 0) {
const unsigned nb = std::min<unsigned>(blocks(n, 1024), 4u * (unsigned) std::max(dev_info().sms, 1));
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed
the limit. To get the device limit, query
info::device::max_work_group_size. Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
s->submit([&](sycl::handler &cgh) {
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
sycl::range((size_t)n_expert * 4), cgh);
cgh.parallel_for<
dpct_kernel_name<class count_kernel_9919e0>>(
sycl::nd_range<3>(sycl::range(1, 1, nb) *
sycl::range(1, 1, 1024),
sycl::range(1, 1, 1024)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
count_kernel(ids, n, n_expert, tb.cnt,
dpct_local_acc_ct1
.get_multi_ptr<
sycl::access::decorated::no>()
.get());
});
});
}
}
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed the
limit. To get the device limit, query info::device::max_work_group_size.
Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
s->parallel_for<dpct_kernel_name<class scan_kernel_7ed98e>>(
sycl::nd_range<3>(
sycl::range(1, 1, (unsigned)((n_expert + 31) / 32 * 32)),
sycl::range(1, 1, (unsigned)((n_expert + 31) / 32 * 32))),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
scan_kernel(tb, n_expert);
});
}
if (n > 0) {
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
s->parallel_for<dpct_kernel_name<class place_kernel_cf784d>>(
sycl::nd_range<3>(sycl::range(1, 1, blocks(n, 256)) *
sycl::range(1, 1, 256),
sycl::range(1, 1, 256)),
exp_props, [=](sycl::nd_item<3> item_ct1) {
place_kernel(ids, n, k, n_expert, tb, slot, src);
});
}
/*
DPCT1010: SYCL uses exceptions to report errors and does not use the
error codes. The cudaGetLastError function call was replaced with 0. You
need to rewrite this code.
*/
ck(0, "group");
}
void experts(const Batch& b, int n_expert, int64_t n, const void* scratch, const void* xa, const int32_t* src,
void* ha, float* dm, void* stream) {
if (b.e1 <= b.e0 || n <= 0) return;
const DevInfo& d = dev_info();
const dpct::queue_ptr s = strata::q_of(stream);
const Tables tb = tables(const_cast<void*>(scratch), n_expert);
// the most tiles the batch can have: every row in it, plus a partial tile per expert
const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
#if defined(__HIPCC__)
const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * (1280 / W_ROWS), (int64_t) d.sms * wgp_blocks(d.occ_gu));
const unsigned g_d = (unsigned) std::min<int64_t>(tiles * (2560 / W_ROWS), (int64_t) d.sms * wgp_blocks(d.occ_d));
expert_w11_kernel<true><<<g_gu, W_THREADS, 0, s>>>(b, tb, (const uint8_t*) xa, src, (uint8_t*) ha, nullptr);
expert_w11_kernel<false><<<g_d, W_THREADS, 0, s>>>(b, tb, (const uint8_t*) ha, src, nullptr, dm);
#else
const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * 5, (int64_t) d.sms * d.occ_gu);
const unsigned g_d = (unsigned) std::min<int64_t>(tiles * 10, (int64_t) d.sms * d.occ_d);
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed the
limit. To get the device limit, query info::device::max_work_group_size.
Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
dpct::has_capability_or_fail(s->get_device(), {sycl::aspect::fp16});
s->submit([&](sycl::handler &cgh) {
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
sycl::range(smem_bytes(true)), cgh);
cgh.parallel_for<dpct_kernel_name<class expert_kernel_f38977,
dpct_kernel_scalar<true>>>(
sycl::nd_range<3>(sycl::range(1, 1, g_gu) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
expert_kernel<true>(
b, tb, (const uint8_t *)xa, src, (uint8_t *)ha,
nullptr,
dpct_local_acc_ct1
.get_multi_ptr<sycl::access::decorated::no>()
.get());
});
});
}
/*
DPCT1049: The work-group size passed to the SYCL kernel may exceed the
limit. To get the device limit, query info::device::max_work_group_size.
Adjust the work-group size if needed.
*/
{
auto exp_props = sycl::ext::oneapi::experimental::properties{
sycl::ext::oneapi::experimental::use_root_sync};
dpct::has_capability_or_fail(s->get_device(), {sycl::aspect::fp16});
s->submit([&](sycl::handler &cgh) {
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
sycl::range(smem_bytes(false)), cgh);
cgh.parallel_for<dpct_kernel_name<class expert_kernel_e2d106,
dpct_kernel_scalar<false>>>(
sycl::nd_range<3>(sycl::range(1, 1, g_d) *
sycl::range(1, 1, THREADS),
sycl::range(1, 1, THREADS)),
exp_props,
[=](sycl::nd_item<3> item_ct1)
[[sycl::reqd_sub_group_size(32)]] {
expert_kernel<false>(
b, tb, (const uint8_t *)ha, src, nullptr, dm,
dpct_local_acc_ct1
.get_multi_ptr<sycl::access::decorated::no>()
.get());
});
});
}
#endif
/*
DPCT1010: SYCL uses exceptions to report errors and does not use the
error codes. The cudaGetLastError function call was replaced with 0. You
need to rewrite this code.
*/
ck(0, "experts");
}
} // namespace strata::prefill::fused