Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-145-pp-volta into f-146-volta (0.1.46): the community Volta/Pascal perf-PR group (35 commits)

Niko12218b3a96779e88
diff --cc CMakeLists.txt
index d900326e3,1fae7183c..758655a8a
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -983,21 -891,14 +996,29 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_
target_include_directories(native_grouped_parity PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(native_grouped_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME native_grouped_parity COMMAND native_grouped_parity)
+ add_executable(mmvf_rows_parity src/kernels/mmvf_rows_parity.cpp)
+ target_include_directories(mmvf_rows_parity PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(mmvf_rows_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+ add_test(NAME mmvf_rows_parity COMMAND mmvf_rows_parity)
+ set_tests_properties(mmvf_rows_parity PROPERTIES ENVIRONMENT "STRATA_MMVF_ROWS=1" TIMEOUT 120)
+ if(STRATA_HIP_GFX906)
+ add_test(NAME native_grouped_fused_parity COMMAND native_grouped_parity)
+ set_tests_properties(native_grouped_fused_parity PROPERTIES
+ ENVIRONMENT "STRATA_HIP_SWIGLU_FUSED=1;STRATA_EXP_MODE=7" TIMEOUT 120)
+ add_executable(gfx906_gdn_rec_head tests/hip/gdn_rec_head.cpp)
+ target_include_directories(gfx906_gdn_rec_head PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(gfx906_gdn_rec_head PRIVATE strata_prefill ${_strata_gpu_runtime_target})
+ add_test(NAME gfx906_gdn_rec_head COMMAND gfx906_gdn_rec_head)
+ set_tests_properties(gfx906_gdn_rec_head PROPERTIES TIMEOUT 120)
+ endif()
+ if(STRATA_ENABLE_CUDA)
+ # the Volta prompt experts (gemm_iq_f16_grouped, STRATA_PF_WMMA) against a double reference; skipped below sm_70
+ add_executable(pf_wmma_parity src/kernels/pf_wmma_parity.cpp)
+ target_include_directories(pf_wmma_parity PRIVATE ${_strata_gpu_include_directories})
+ target_link_libraries(pf_wmma_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
+ add_test(NAME pf_wmma_parity COMMAND pf_wmma_parity)
+ set_tests_properties(pf_wmma_parity PROPERTIES SKIP_RETURN_CODE 77)
+ endif()
add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)
target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
diff --cc src/kernels/cuda/iq_kernels.cu
index 3334d8404,a42cc4e5f..3ae7e3355
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@@ -1299,6 -1255,6 +1312,36 @@@ __device__ __forceinline__ void row_dot
}
}
++// The sign-mask tables that follow the codebook words in a staged IQ3_XXS / IQ3_S table (see stage_iq_grid): the byte
++// masks of every 7-bit sign field (IQ3_XXS, 2 x 128 words at +256) and of a sign byte's bits 0-3 and 4-7 (IQ3_S, 2 x 256
++// words at +512). Split<18>::load<true> and Split<21>::load<true> read them, so every kernel that stages these two
++// codebooks (the default kernels through stage_iq_grid, the STRATA_EXPDEC window kernels through EdGrid) fills them too.
++__device__ __forceinline__ void stage_iq3xxs_masks(uint32_t* s_buf, int tid, int nthreads) {
++ for (int u = tid; u < 128; u += nthreads) {
++ const uint32_t s = (uint32_t) u | ((__popc((uint32_t) u) & 1u) << 7); // 8th sign = parity of the 7
++ uint32_t ml = 0, mh = 0;
++#pragma unroll
++ for (int i = 0; i < 4; ++i) {
++ ml |= ((s >> i) & 1u) * (0xffu << (8 * i));
++ mh |= ((s >> (4 + i)) & 1u) * (0xffu << (8 * i));
++ }
++ s_buf[256 + u] = ml;
++ s_buf[384 + u] = mh;
++ }
++}
++__device__ __forceinline__ void stage_iq3s_masks(uint32_t* s_buf, int tid, int nthreads) {
++ for (int u = tid; u < 256; u += nthreads) {
++ uint32_t ml = 0, mh = 0;
++#pragma unroll
++ for (int i = 0; i < 4; ++i) {
++ ml |= (((uint32_t) u >> i) & 1u) * (0xffu << (8 * i));
++ mh |= (((uint32_t) u >> (4 + i)) & 1u) * (0xffu << (8 * i));
++ }
++ s_buf[512 + u] = ml;
++ s_buf[768 + u] = mh;
++ }
++}
++
template<int TY, bool STAGE_GRID = kStageIqGrid<TY>>
__device__ __forceinline__ const uint32_t* stage_iq_grid(uint32_t* s_buf, int tid, int nthreads) {
if constexpr (!STAGE_GRID) {
@@@ -1327,6 -1283,44 +1370,25 @@@
for (int i = tid; i < 1024; i += nthreads) dst[i] = src[i];
__syncthreads();
return s_buf;
+ } else if constexpr (TY == 18) {
+ // 256 codebook words, then two 128-word tables of the per-byte sign masks (signs 0-3, signs 4-7). A
+ // mask depends only on the 7 bits it is unpacked from, and iq3xxs_grid's bytes are all >= 4, so the
+ // consumer applies it as (g ^ mask) + (mask & 0x01010101): a negate per set byte, with no carry into
+ // the next. One pass per block replaces the per-lookup unpack_ksigns / __vcmpne4 / __vsub4 chain.
+ // Ported from shinbunbun/llama-cpp-p100-patches (MIT), patches 29 `mmvq-iq3xxs-grid-smem` and 30
+ // `mmvq-ksigns-smem`, measured there on a Tesla P100 (sm_60).
+ for (int i = tid; i < 256; i += nthreads) s_buf[i] = iq3xxs_grid[i];
- for (int u = tid; u < 128; u += nthreads) {
- const uint32_t s = (uint32_t) u | ((__popc((uint32_t) u) & 1u) << 7); // 8th sign = parity of the 7
- uint32_t ml = 0, mh = 0;
-#pragma unroll
- for (int i = 0; i < 4; ++i) {
- ml |= ((s >> i) & 1u) * (0xffu << (8 * i));
- mh |= ((s >> (4 + i)) & 1u) * (0xffu << (8 * i));
- }
- s_buf[256 + u] = ml;
- s_buf[384 + u] = mh;
- }
++ stage_iq3xxs_masks(s_buf, tid, nthreads);
+ __syncthreads();
+ return s_buf;
+ } else if constexpr (TY == 21) {
+ // 512 codebook words, then two 256-word tables: the byte masks of a sign byte's bits 0-3 and of its bits
+ // 4-7 (IQ3_S stores one sign byte per 8 values). As for IQ3_XXS, one pass per block replaces the
+ // per-lookup __vcmpne4 / __vsub4 chain, and the codebook reads leave global memory.
+ for (int i = tid; i < 512; i += nthreads) s_buf[i] = iq3s_grid[i];
- for (int u = tid; u < 256; u += nthreads) {
- uint32_t ml = 0, mh = 0;
-#pragma unroll
- for (int i = 0; i < 4; ++i) {
- ml |= (((uint32_t) u >> i) & 1u) * (0xffu << (8 * i));
- mh |= (((uint32_t) u >> (4 + i)) & 1u) * (0xffu << (8 * i));
- }
- s_buf[512 + u] = ml;
- s_buf[768 + u] = mh;
- }
++ stage_iq3s_masks(s_buf, tid, nthreads);
+ __syncthreads();
+ return s_buf;
} else {
return nullptr;
}
@@@ -1721,369 -1715,6 +1783,376 @@@ __global__ void __launch_bounds__(256)
}
}
+// ---------------------------------------------------------------- EXPDEC (STRATA_EXPDEC=1): the group's activations
+// in shared memory, R rows per lane
+//
+// native_gu_multi_kernel / native_down_multi_kernel decode each weight part once and apply it to every entry of the
+// group, but the gate/up apply re-reads the entry's q8_1 block from global memory with 9 scattered 4-byte loads
+// (36-byte blocks: a warp's load touches 32 sectors), so an entry costs about 60% of a whole expert (nsys, 2080 Ti,
+// L02 at 131k: ~2.6 us per expert and ~1.6 us per entry and layer) although it reads nothing from DRAM. Here a
+// block stages its group's entries (ED_NC at a time, as the default) in shared memory laid out so that a call's
+// words are two 16-byte loads (gate/up, Q2_0 down: the block's two halves in separate planes, conflict-free) or two
+// 8-byte loads (IQ4_NL down), the fp16 block scale already a float; every lane computes R rows at once, so the
+// activation words are loaded once per R rows; the IQ3_XXS / IQ3_S codebooks are staged too (IQ2_* already were),
+// and a block takes `rb` rows (auto: the most that keep ~STRATA_EXPDEC_MINBLK blocks in flight) instead of 16.
+// Each (row, entry) sum is the default kernel's sum tree (row_dot_80_sub16 / row_dot_40_sub8 / row_dot_20_sub16: the
+// same lane does the same calls in the same order with the same operations, `apply_u` = `apply` on the block's
+// words), so the output is bitwise the same (expdec_bench checks it; docs/EXPERTDEC.md).
+#ifndef STRATA_EXPDEC_MINB
+#define STRATA_EXPDEC_MINB 4 // gate/up __launch_bounds__ min blocks per SM by default (STRATA_EXPDEC_MINB at runtime): 4 = up to 64
+ // registers, as the default kernels (bench: equal on the 2080 Ti, 0.87-0.95 on the 3060); 2 = up to 128
+#endif
+constexpr int ED_NC = GRP_NC; // entries staged at once
+constexpr int ED_XB = 80; // q8_1 blocks of a gate/up row (n_embd 2560)
+constexpr int ED_HB = 20; // q8_1 blocks of a down row (n_ff 640)
+
+// the codebook of a gate/up format in shared memory: words, and the copy (every thread of the block; sync after)
+template<int TG> struct EdGrid;
+template<> struct EdGrid<16> { static constexpr int words = 512;
+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2xxs_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 256; i += 256) dst[i] = src[i]; } };
+template<> struct EdGrid<17> { static constexpr int words = 1024;
+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2xs_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 512; i += 256) dst[i] = src[i]; } };
+template<> struct EdGrid<22> { static constexpr int words = 2048;
+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2s_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 1024; i += 256) dst[i] = src[i]; } };
- template<> struct EdGrid<18> { static constexpr int words = 256;
- __device__ static void copy(uint32_t* s, int tid) { for (int i = tid; i < 256; i += 256) s[i] = iq3xxs_grid[i]; } };
- template<> struct EdGrid<21> { static constexpr int words = 512;
- __device__ static void copy(uint32_t* s, int tid) { for (int i = tid; i < 512; i += 256) s[i] = iq3s_grid[i]; } };
++// (IQ3_XXS / IQ3_S: the codebook and, behind it, the sign-mask tables Split<TG>::load<true> reads - stage_iq_grid's layout)
++template<> struct EdGrid<18> { static constexpr int words = 512;
++ __device__ static void copy(uint32_t* s, int tid) {
++ for (int i = tid; i < 256; i += 256) s[i] = iq3xxs_grid[i];
++ stage_iq3xxs_masks(s, tid, 256);
++ } };
++template<> struct EdGrid<21> { static constexpr int words = 1024;
++ __device__ static void copy(uint32_t* s, int tid) {
++ for (int i = tid; i < 512; i += 256) s[i] = iq3s_grid[i];
++ stage_iq3s_masks(s, tid, 256);
++ } };
+
+// One entry's q8_1 row (XB blocks of 36 bytes, 4-byte aligned) into shared memory as entry c: the 8 words of block k
+// in planes (HALVES: s_u[(2 c + h) XB + k] = words 4h..4h+3, two 16-byte loads per call, conflict-free over 16
+// consecutive k; else s_u[(c XB + k) 2 + h], the block's 32 bytes together, for the IQ4_NL pairs) and the fp16 scale
+// as s_ds[c XB + k] = __low2float(ds), the value `apply` computes. All 256 threads.
+template<int XB, bool HALVES>
+__device__ __forceinline__ void ed_stage_entry(int4* __restrict__ s_u, float* __restrict__ s_ds, int c,
+ const block_q8_1* __restrict__ src, int tid) {
+ int* su = (int*) s_u;
+ const int* w32 = (const int*) src;
+ for (int w = tid; w < XB * 9; w += 256) {
+ const int k = w / 9, j = w - k * 9;
+ const int v = w32[w];
+ if (j == 0) {
+ const half2 h = *reinterpret_cast<const half2*>(&v);
+ s_ds[c * XB + k] = __low2float(h);
+ } else {
+ const int q = j - 1;
+ if constexpr (HALVES) su[((c * 2 + (q >> 2)) * XB + k) * 4 + (q & 3)] = v;
+ else su[(c * XB + k) * 8 + q] = v;
+ }
+ }
+}
+template<int XB>
+__device__ __forceinline__ void ed_load_u(const int4* __restrict__ s_u, int c, int k, int (&u)[8]) {
+ const int4 a = s_u[(c * 2 + 0) * XB + k], b = s_u[(c * 2 + 1) * XB + k];
+ u[0] = a.x; u[1] = a.y; u[2] = a.z; u[3] = a.w;
+ u[4] = b.x; u[5] = b.y; u[6] = b.z; u[7] = b.w;
+}
+
+// R gate/up rows against the NC staged entries, lane t of a 16-lane sub-warp: row_dot_80_sub16's tree (calls t,
+// t+32, t+64 into s; t+16, t+48 into s16; s = __fadd_rn(s, s16); xor 8-4-2-1), every call's weight part decoded once
+// per row and the entry's block loaded once per NC x R products.
+template<int TG, int R, int NC>
+__device__ __forceinline__ void ed_gu_rows(const uint8_t* const* wr, const int4* __restrict__ s_u,
+ const float* __restrict__ s_ds, const uint32_t* __restrict__ s_grid, int t,
+ float (&s)[R][NC]) {
+ using F = Fmt<TG>;
+ using S = Split<TG>;
+ float s16[R][NC];
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) { s[r][c] = 0.0f; s16[r][c] = 0.0f; }
+#pragma unroll
+ for (int j = 0; j < 3; ++j) {
+ const int k = t + 32 * j;
+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);
+ typename S::W w[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) w[r] = S::template load<true>(wr[r], kbx, iqs, s_grid);
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ int u[8];
+ ed_load_u<ED_XB>(s_u, c, k, u);
+ const float d8 = s_ds[c * ED_XB + k];
+#pragma unroll
+ for (int r = 0; r < R; ++r) s[r][c] += S::apply_u(w[r], u, d8);
+ }
+ }
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ const int k = t + 16 + 32 * j;
+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);
+ typename S::W w[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) w[r] = S::template load<true>(wr[r], kbx, iqs, s_grid);
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ int u[8];
+ ed_load_u<ED_XB>(s_u, c, k, u);
+ const float d8 = s_ds[c * ED_XB + k];
+#pragma unroll
+ for (int r = 0; r < R; ++r) s16[r][c] += S::apply_u(w[r], u, d8);
+ }
+ }
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ s[r][c] = __fadd_rn(s[r][c], s16[r][c]);
+#pragma unroll
+ for (int o = 8; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);
+ }
+}
+
+// gate/up: 16 sub-warps of 16 lanes, R rows each = 16 R rows per pass, `rb` rows per block (a multiple of 16 R; 2 n_ff
+// is one too, so a pass is whole or absent for the entire block). Group stride as native_gu_multi_kernel.
+template<int TG, int R, int MINB>
+__global__ void __launch_bounds__(256, MINB) expdec_gu_kernel(const unsigned long long* __restrict__ grp_ptr,
+ const int32_t* __restrict__ grp_start,
+ const int32_t* __restrict__ n_groups,
+ const int32_t* __restrict__ ent_tok,
+ const block_q8_1* __restrict__ xq, NativeExpertLayout L,
+ float* __restrict__ gate, float* __restrict__ up, int rb) {
+ const int ng = *n_groups;
+ if (blockIdx.y >= ng) return;
+ __shared__ alignas(16) int4 s_u[ED_NC * 2 * ED_XB];
+ __shared__ float s_ds[ED_NC * ED_XB];
+ STRATA_SHARED_ALIGN16_U32(s_grid_buf, EdGrid<TG>::words);
+ const int tid = threadIdx.x;
+ EdGrid<TG>::copy(s_grid_buf, tid); // visible after the staging's __syncthreads
+ const int sw = tid >> 4, t = tid & 15;
+ const int nrow = 2 * (int) L.n_ff, n_ff = (int) L.n_ff;
+ const int row_base = blockIdx.x * rb;
+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+ const uint8_t* blob = (const uint8_t*) grp_ptr[g];
+ const int e0 = grp_start[g], e1 = grp_start[g + 1];
+ for (int e = e0; e < e1; e += ED_NC) {
+ const int n = min(ED_NC, e1 - e);
+ __syncthreads();
+ for (int c = 0; c < n; ++c) ed_stage_entry<ED_XB, true>(s_u, s_ds, c, xq + (size_t) ent_tok[e + c] * ED_XB, tid);
+ __syncthreads();
+ for (int p = 0; p * 16 * R < rb; ++p) {
+ const int row0 = row_base + p * 16 * R + sw * R;
+ if (row0 >= nrow) break; // uniform over the block
+ const bool is_up = row0 >= n_ff;
+ const int r0 = is_up ? row0 - n_ff : row0;
+ const uint8_t* wr[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) wr[r] = blob + (is_up ? L.up_off : 0) + (size_t) (r0 + r) * L.gu_row;
+ float* dst = is_up ? up : gate;
+ auto run = [&](auto nc) {
+ constexpr int NC = decltype(nc)::value;
+ float s[R][NC];
+ ed_gu_rows<TG, R, NC>(wr, s_u, s_ds, s_grid_buf, t, s);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (t == c)
+#pragma unroll
+ for (int r = 0; r < R; ++r) dst[(size_t) (e + c) * n_ff + r0 + r] = s[r][c];
+ };
+ switch (n) {
+ case 1: run(std::integral_constant<int, 1>{}); break;
+ case 2: run(std::integral_constant<int, 2>{}); break;
+ case 3: run(std::integral_constant<int, 3>{}); break;
+ default: run(std::integral_constant<int, 4>{}); break;
+ }
+ }
+ }
+ }
+}
+
+// R IQ4_NL down rows, lane t of an 8-lane sub-warp: row_dot_40_sub8's tree. s_p: the block's 8 words together
+// (ed_stage_entry<ED_HB, false>), read as the int2 pairs (iqs, iqs + 1) and (iqs + 4, iqs + 5).
+template<int R, int NC>
+__device__ __forceinline__ void ed_d20_rows(const uint8_t* const* wr, const int2* __restrict__ s_p,
+ const float* __restrict__ s_ds, int t, float (&s)[R][NC]) {
+ using F = Fmt<20>;
+ using S = Split<20>;
+ auto term = [&](int k, int r, int c, const typename S::W& w) -> float {
+ const int kbx = k / F::ipb, par = k & 1;
+ const int2 p0 = s_p[(c * ED_HB + kbx) * 4 + par], p1 = s_p[(c * ED_HB + kbx) * 4 + 2 + par];
+ return S::apply_u(w, p0, p1, s_ds[c * ED_HB + kbx]);
+ };
+ auto loadw = [&](int k, typename S::W (&w)[R]) {
+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);
+#pragma unroll
+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);
+ };
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) s[r][c] = 0.0f;
+#pragma unroll
+ for (int j = 0; j < 2; ++j) { // k = t, t + 32
+ const int k = t + 32 * j;
+ typename S::W w[R];
+ loadw(k, w);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+#pragma unroll
+ for (int r = 0; r < R; ++r) s[r][c] += term(k, r, c, w[r]);
+ }
+ {
+ const int k = t + 16;
+ typename S::W w[R];
+ loadw(k, w);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+#pragma unroll
+ for (int r = 0; r < R; ++r) {
+ float s16 = 0.0f;
+ s16 += term(k, r, c, w[r]);
+ s[r][c] = __fadd_rn(s[r][c], s16);
+ }
+ }
+ float s8[R][NC];
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) s8[r][c] = 0.0f;
+ {
+ const int k = t + 8;
+ typename S::W w[R];
+ loadw(k, w);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+#pragma unroll
+ for (int r = 0; r < R; ++r) s8[r][c] += term(k, r, c, w[r]);
+ }
+ {
+ const int k = t + 24;
+ typename S::W w[R];
+ loadw(k, w);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+#pragma unroll
+ for (int r = 0; r < R; ++r) {
+ float s24 = 0.0f;
+ s24 += term(k, r, c, w[r]);
+ s[r][c] = __fadd_rn(s[r][c], __fadd_rn(s8[r][c], s24));
+ }
+ }
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+#pragma unroll
+ for (int o = 4; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);
+}
+
+// R Q2_0 down rows, lane t of a 16-lane sub-warp: row_dot_20_sub16's tree (call t; call t + 16 for t < 4 into s16).
+template<int R, int NC>
+__device__ __forceinline__ void ed_d42_rows(const uint8_t* const* wr, const int4* __restrict__ s_u,
+ const float* __restrict__ s_ds, int t, float (&s)[R][NC]) {
+ using F = Fmt<42>;
+ using S = Split<42>;
+ {
+ const int k = t, kbx = k / F::ipb, iqs = F::step * (k % F::ipb);
+ typename S::W w[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ int u[8];
+ ed_load_u<ED_HB>(s_u, c, k, u);
+ const float d8 = s_ds[c * ED_HB + k];
+#pragma unroll
+ for (int r = 0; r < R; ++r) { s[r][c] = 0.0f; s[r][c] += S::apply_u(w[r], u, d8); }
+ }
+ }
+ float s16[R][NC];
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) s16[r][c] = 0.0f;
+ if (t < 4) {
+ const int k = t + 16, kbx = k / F::ipb, iqs = F::step * (k % F::ipb);
+ typename S::W w[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ int u[8];
+ ed_load_u<ED_HB>(s_u, c, k, u);
+ const float d8 = s_ds[c * ED_HB + k];
+#pragma unroll
+ for (int r = 0; r < R; ++r) s16[r][c] += S::apply_u(w[r], u, d8);
+ }
+ }
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ s[r][c] = __fadd_rn(s[r][c], s16[r][c]);
+#pragma unroll
+ for (int o = 8; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);
+ }
+}
+
+// down (IQ4_NL: 32 sub-warps of 8 lanes; Q2_0: 16 of 16), R rows per lane, `rb` rows per block (a multiple of the
+// rows per pass; n_embd too). The entries' h rows are contiguous in hq (entry e at e * ED_HB).
+template<int TD, int R>
+__global__ void __launch_bounds__(256, STRATA_EXPDEC_MINB) expdec_down_kernel(const unsigned long long* __restrict__ grp_ptr,
+ const int32_t* __restrict__ grp_start,
+ const int32_t* __restrict__ n_groups,
+ const int32_t* __restrict__ ent_dst,
+ const block_q8_1* __restrict__ hq, NativeExpertLayout L,
+ float* __restrict__ out, int rb) {
+ const int ng = *n_groups;
+ if (blockIdx.y >= ng) return;
+ constexpr int LANES = TD == 20 ? 8 : 16;
+ constexpr int RPP = (256 / LANES) * R; // rows per pass
+ __shared__ alignas(16) int4 s_u[ED_NC * 2 * ED_HB];
+ __shared__ float s_ds[ED_NC * ED_HB];
+ const int tid = threadIdx.x;
+ const int sw = tid / LANES, t = tid % LANES;
+ const int nrow = (int) L.n_embd;
+ const int row_base = blockIdx.x * rb;
+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+ const uint8_t* blob = (const uint8_t*) grp_ptr[g] + L.down_off;
+ const int e0 = grp_start[g], e1 = grp_start[g + 1];
+ for (int e = e0; e < e1; e += ED_NC) {
+ const int n = min(ED_NC, e1 - e);
+ __syncthreads();
+ for (int c = 0; c < n; ++c) ed_stage_entry<ED_HB, TD == 42>(s_u, s_ds, c, hq + (size_t) (e + c) * ED_HB, tid);
+ __syncthreads();
+ for (int p = 0; p * RPP < rb; ++p) {
+ const int row0 = row_base + p * RPP + sw * R;
+ if (row0 >= nrow) break; // uniform over the block
+ const uint8_t* wr[R];
+#pragma unroll
+ for (int r = 0; r < R; ++r) wr[r] = blob + (size_t) (row0 + r) * L.d_row;
+ auto run = [&](auto nc) {
+ constexpr int NC = decltype(nc)::value;
+ float s[R][NC];
+ if constexpr (TD == 20) ed_d20_rows<R, NC>(wr, (const int2*) s_u, s_ds, t, s);
+ else ed_d42_rows<R, NC>(wr, s_u, s_ds, t, s);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (t == c) {
+ const size_t o = (size_t) ent_dst[e + c] * L.n_embd + row0;
+#pragma unroll
+ for (int r = 0; r < R; ++r) out[o + r] = s[r][c];
+ }
+ };
+ switch (n) {
+ case 1: run(std::integral_constant<int, 1>{}); break;
+ case 2: run(std::integral_constant<int, 2>{}); break;
+ case 3: run(std::integral_constant<int, 3>{}); break;
+ default: run(std::integral_constant<int, 4>{}); break;
+ }
+ }
+ }
+ }
+}
+
// ---------------------------------------------------------------- q8_1 (quantize.cu)
// value i of a q8_1 row set (the warp holds block i / 32, lane = i % 32)
__device__ __forceinline__ void q8_1_store(const float xi, block_q8_1* __restrict__ y, const long long i) {
diff --cc src/prefill/kernels.cu
index a637797ae,d6c6b99a0..a4f56f10c
--- a/src/prefill/kernels.cu
+++ b/src/prefill/kernels.cu
@@@ -2529,6 -2114,45 +2662,44 @@@ void gdn_recurrence_variant(int variant
}
#else
(void) variant; // the quad recurrence kernels are AMD-only
+ #endif
- static const bool serial = std::getenv("STRATA_GDN_REC_HEADS") != nullptr; // the one-block-per-head kernel (A/B)
+ #if defined(STRATA_EXPERIMENTAL_SM60)
+ // v100/gdn-chunk: the chunked recurrence (src/prefill/gdn_chunk.cu) on Volta, where the token chain is the
+ // cost. STRATA_GDN_CHUNK=0 forces the recurrence, =1 forces the chunked path (the parity test's A/B switch);
+ // unset: cc == 7.0 only, every other architecture keeps the recurrence bit-for-bit as before. Read per call
+ // on purpose - cheap next to the work, and the tests flip it between two launches.
+ const char* chunk_v = std::getenv("STRATA_GDN_CHUNK");
+ const int chunk_env = chunk_v && *chunk_v ? std::atoi(chunk_v) : -1;
+ static const bool volta = gdn_chunk_available(); // runtime cc == 7.0 (the kernels are compiled for sm_70 only)
+ const bool chunked = !serial && T > 0 && (chunk_env >= 0 ? chunk_env != 0 : false);
+ // v100/gdn-chunk: the sync-free warp-reduction recurrence (bit-exact vs gdn_rec_cols_pipe) is the
+ // measured-fast path on Volta; the chunked recurrence is opt-in (STRATA_GDN_CHUNK=1/2) - correct and
+ // measured, but slower than this on the V100 (bench/gdn_chunk_bench). STRATA_GDN_REC_WARP=0 restores
+ // the old pipelined kernel bit-for-bit.
+ // STRATA_GDN_REC_WARP (Volta only, opt-in): unset or 0 -> the pipelined kernel (the default; the chain split changes the
+ // summation order, so it is not the default without a V100 check here), 2 -> the chain-split kernel (measured +6..9%, FP32-level), the
+ // old pipelined kernel bit-for-bit is 0, 1 -> the sync-free bit-exact control.
+ const char* warp_v = std::getenv("STRATA_GDN_REC_WARP");
+ const int warp_env = warp_v && *warp_v ? std::atoi(warp_v) : 0;
+ const bool warp_rec = !serial && !chunked && T > 0 && volta && warp_env != 0;
+ if (chunked && T >= 64) {
+ // the chunked prefix + the recurrence over the tail (< 64 tokens), then the out norm over all of T
+ const int64_t T64 = T - T % 64;
+ gdn_chunk_recurrence(state, h, gate, beta, y, T64, stream);
+ if (T > T64)
+ gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(
+ state, h + T64 * C, gate + T64 * HV, beta + T64 * HV, y + T64 * (size_t) HV * S, T - T64);
+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);
+ } else if (warp_rec) {
+ // =2: the 4-accumulator chain split on the pipelined structure (FP32-level, not bit-exact);
+ // =1: the sync-free bit-exact warp reduction (measured slower - kept as the bitwise control)
+ if (warp_env == 2)
+ gdn_rec_cols_pipe_fast_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta,
+ y, T);
+ else
+ gdn_rec_cols_warp_kernel<<<HV * NCB, S, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);
+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);
+ } else
#endif
if (serial || T <= 0 || variant == 2) {
gdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T, ld16);
diff --cc src/prefill/prefill.cpp
index 17428925e,ccce1faa5..bceacbda3
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -1118,14 -1009,31 +1119,38 @@@ struct MmqPlan
std::vector<char> fo; // per layer: no MMQ here but the native fused kernels cover its formats
// (gfx11: UD-Q4_K_XL's Q4_K / Q5_K and Q5_1 / Q8_0 experts, STRATA_PF_FUSED=1)
size_t gu_max = 0, d_max = 0;
+ // STRATA_MMQ_DIRECT=1: per layer, whether resident experts are read in their cache slots (pointer-table MMQ),
+ // and the bytes the down product reads past its matrix there (zeroed in the slot's guard before each group)
+ std::vector<char> direct;
+ std::vector<uint32_t> d_over;
+ // STRATA_MMQ_GROUP_SORT=1 (without STRATA_MMQ_RESIDENT_SORT_NE): per MMQ layer, whether both products' 16-expert
+ // launches run whole tiles on every visible GPU (mmq::whole_tiles), so its full groups may be taken in row-count order
+ std::vector<char> group_sort;
};
+ // Volta (sm_70): MMQ runs on dp4a there (no int8 tensor cores). pf_wmma(): the layers MMQ would take, and the IQ1_M
+ // ones the FP16 path would, run as one launch per expert group on FP16 tensor-core tiles instead
+ // (strata::kernels::gemm_iq_f16_grouped: each weight superblock dequantized once into shared memory by the FP16 path's
+ // own formulas, FP16 activations, FP32 sums). OPT-IN: STRATA_PF_WMMA=1 (the activations are FP16 rather than MMQ's
+ // q8_1, so the answers differ from the default). Only when EVERY visible CUDA card is compute capability 7.0 or
+ // newer: the tiles need tensor cores, and a Volta + Pascal rig (one process, one decision) keeps MMQ.
+ bool pf_wmma() {
+ #if defined(__HIPCC__)
+ return false;
+ #else
+ static const bool on = [] {
+ const char* e = std::getenv("STRATA_PF_WMMA");
+ if (e == nullptr || e[0] != '1') return false;
+ int n = 0;
+ if (cudaGetDeviceCount(&n) != cudaSuccess || n <= 0) return false;
+ for (int d = 0; d < n; ++d) {
+ int major = 0;
+ if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, d) != cudaSuccess || major < 7) return false;
+ }
+ return true;
+ }();
+ return on;
+ #endif
+ }
const MmqPlan& mmq_plan() {
static const MmqPlan plan = [] {
MmqPlan p;
@@@ -3803,13 -3498,9 +3848,14 @@@ bool Prefill::run_impl(const int64_t* t
const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;
pt.mark(kPfGather, cs);
if (use_mmq) {
- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`
- if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows
- else mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
+ // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`:
+ // each token quantized once and written to its K rows (slot_dev, the inverse of src_dev).
+ // NVFP4 w4a4 activations carry a per-row scale (Xscale): that mode keeps the gather.
- if (mmq::fp4_activations(mmq_gt))
++ if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows (pf_wmma)
++ else if (mmq::fp4_activations(mmq_gt))
+ mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
+ else
+ mmq::quantize_scatter(m.mixed, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down
// reads the group's own quantized H)
const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
@@@ -4200,15 -3876,13 +4246,15 @@@
mmq::GatherGroup gg;
int gg_slots[MMQ_GROUP];
int gg_nslots = 0; // ring slots gathered by the next flush
- // `zero`: the MMQ tail zeroed after the gathered experts (the group's last flush)
+ // STRATA_MMQ_DIRECT=1: a resident expert with guard room in its slot is not gathered - the
+ // group's pointer tables give MMQ each expert's own address (group position q), the others are
+ // gathered at compact positions c = 0..ns-1 of the group buffers, as before (same bytes)
- const bool direct = use_mmq && lay.native && m.cache && mmq_plan().direct[(size_t) l];
++ const bool direct = use_mmq && lay.native && m.cache && mmq_plan().direct[(size_t) l] && !wmma_l;
+ mmq::WeightTab tgu, td;
+ mmq::GuardTab gd;
+ int ns = 0;
+ // `zero`: the MMQ tail zeroed after the gathered experts (the group's last flush); a direct
+ // group's tail is zeroed by prep_group
auto flush = [&](size_t zero) {
if (gg.n <= gg.first) return;
const auto& f = lay.fmt[(size_t) l];
@@@ -4310,13 -3951,19 +4356,26 @@@
int64_t maxr = 0;
for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, m.cnt[(size_t) order[i]]);
pt.mark(kPfGemmGU, cs);
+ if (wmma_l) { // pf_wmma: gate/up, SwiGLU to FP16, down - the group in 2 tile launches
+ const int32_t* dnb = m.bounds_dev + n + 1 + g * (MMQ_GROUP + 1);
+ const bool ok1 = strata::kernels::gemm_iq_f16_grouped(
+ mmq_gt, m.grp_gu, mmq_gub, 1280, (int) N, m.Xs, (int) N, m.bounds_dev + j0, ngx,
+ (int) maxr, m.GU, 1280, m.cs);
+ swiglu_split_f16(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, 640, !lay.native, m.cs);
+ pt.mark(kPfGemmD, cs);
+ const bool ok2 = strata::kernels::gemm_iq_f16_grouped(
+ mmq_dt, m.grp_d, mmq_db, (int) N, 640, m.Hh + r0 * 640, 640, dnb, ngx, (int) maxr,
+ m.Dm + r0 * N, (int) N, m.cs);
+ if (!ok1 || !ok2) { err = "prefill: the WMMA expert tiles could not launch"; return false; }
+ return true;
+ }
+ if (direct) {
+ // one launch: the direct experts' guards (their down over-read) and the zeroed tail
+ // after the last gathered expert (see MMQ_TAIL)
+ mmq::prep_group(gd, mmq_plan().d_over[(size_t) l],
+ ns ? m.grp_gu + (size_t) ns * mmq_gub : nullptr,
+ ns ? m.grp_d + (size_t) ns * mmq_db : nullptr, MMQ_TAIL, m.cs);
+ }
mmq::Product gu;
gu.w = m.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;
gu.n = ngx; gu.xq = m.Xq; gu.bounds = m.bounds_dev + j0; gu.ids = m.ids_identity;