Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-145-pp-volta into f-146-volta (0.1.46): the community Volta/Pascal perf-PR group (35 commits)
Niko12218b3a96779e88
diff --cc CMakeLists.txtindex d900326e3,1fae7183c..758655a8a--- a/CMakeLists.txt+++ b/CMakeLists.txt@@@ -983,21 -891,14 +996,29 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_target_include_directories(native_grouped_parity PRIVATE ${_strata_gpu_include_directories})target_link_libraries(native_grouped_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME native_grouped_parity COMMAND native_grouped_parity)+ add_executable(mmvf_rows_parity src/kernels/mmvf_rows_parity.cpp)+ target_include_directories(mmvf_rows_parity PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(mmvf_rows_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})+ add_test(NAME mmvf_rows_parity COMMAND mmvf_rows_parity)+ set_tests_properties(mmvf_rows_parity PROPERTIES ENVIRONMENT "STRATA_MMVF_ROWS=1" TIMEOUT 120)+ if(STRATA_HIP_GFX906)+ add_test(NAME native_grouped_fused_parity COMMAND native_grouped_parity)+ set_tests_properties(native_grouped_fused_parity PROPERTIES+ ENVIRONMENT "STRATA_HIP_SWIGLU_FUSED=1;STRATA_EXP_MODE=7" TIMEOUT 120)+ add_executable(gfx906_gdn_rec_head tests/hip/gdn_rec_head.cpp)+ target_include_directories(gfx906_gdn_rec_head PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(gfx906_gdn_rec_head PRIVATE strata_prefill ${_strata_gpu_runtime_target})+ add_test(NAME gfx906_gdn_rec_head COMMAND gfx906_gdn_rec_head)+ set_tests_properties(gfx906_gdn_rec_head PROPERTIES TIMEOUT 120)+ endif()+ if(STRATA_ENABLE_CUDA)+ # the Volta prompt experts (gemm_iq_f16_grouped, STRATA_PF_WMMA) against a double reference; skipped below sm_70+ add_executable(pf_wmma_parity src/kernels/pf_wmma_parity.cpp)+ target_include_directories(pf_wmma_parity PRIVATE ${_strata_gpu_include_directories})+ target_link_libraries(pf_wmma_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})+ add_test(NAME pf_wmma_parity COMMAND pf_wmma_parity)+ set_tests_properties(pf_wmma_parity PROPERTIES SKIP_RETURN_CODE 77)+ endif()add_executable(shared_expert_parity src/kernels/shared_expert_parity.cpp)target_include_directories(shared_expert_parity PRIVATE ${_strata_gpu_include_directories})target_link_libraries(shared_expert_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})diff --cc src/kernels/cuda/iq_kernels.cuindex 3334d8404,a42cc4e5f..3ae7e3355--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@@ -1299,6 -1255,6 +1312,36 @@@ __device__ __forceinline__ void row_dot}}++// The sign-mask tables that follow the codebook words in a staged IQ3_XXS / IQ3_S table (see stage_iq_grid): the byte++// masks of every 7-bit sign field (IQ3_XXS, 2 x 128 words at +256) and of a sign byte's bits 0-3 and 4-7 (IQ3_S, 2 x 256++// words at +512). Split<18>::load<true> and Split<21>::load<true> read them, so every kernel that stages these two++// codebooks (the default kernels through stage_iq_grid, the STRATA_EXPDEC window kernels through EdGrid) fills them too.++__device__ __forceinline__ void stage_iq3xxs_masks(uint32_t* s_buf, int tid, int nthreads) {++ for (int u = tid; u < 128; u += nthreads) {++ const uint32_t s = (uint32_t) u | ((__popc((uint32_t) u) & 1u) << 7); // 8th sign = parity of the 7++ uint32_t ml = 0, mh = 0;++#pragma unroll++ for (int i = 0; i < 4; ++i) {++ ml |= ((s >> i) & 1u) * (0xffu << (8 * i));++ mh |= ((s >> (4 + i)) & 1u) * (0xffu << (8 * i));++ }++ s_buf[256 + u] = ml;++ s_buf[384 + u] = mh;++ }++}++__device__ __forceinline__ void stage_iq3s_masks(uint32_t* s_buf, int tid, int nthreads) {++ for (int u = tid; u < 256; u += nthreads) {++ uint32_t ml = 0, mh = 0;++#pragma unroll++ for (int i = 0; i < 4; ++i) {++ ml |= (((uint32_t) u >> i) & 1u) * (0xffu << (8 * i));++ mh |= (((uint32_t) u >> (4 + i)) & 1u) * (0xffu << (8 * i));++ }++ s_buf[512 + u] = ml;++ s_buf[768 + u] = mh;++ }++}++template<int TY, bool STAGE_GRID = kStageIqGrid<TY>>__device__ __forceinline__ const uint32_t* stage_iq_grid(uint32_t* s_buf, int tid, int nthreads) {if constexpr (!STAGE_GRID) {@@@ -1327,6 -1283,44 +1370,25 @@@for (int i = tid; i < 1024; i += nthreads) dst[i] = src[i];__syncthreads();return s_buf;+ } else if constexpr (TY == 18) {+ // 256 codebook words, then two 128-word tables of the per-byte sign masks (signs 0-3, signs 4-7). A+ // mask depends only on the 7 bits it is unpacked from, and iq3xxs_grid's bytes are all >= 4, so the+ // consumer applies it as (g ^ mask) + (mask & 0x01010101): a negate per set byte, with no carry into+ // the next. One pass per block replaces the per-lookup unpack_ksigns / __vcmpne4 / __vsub4 chain.+ // Ported from shinbunbun/llama-cpp-p100-patches (MIT), patches 29 `mmvq-iq3xxs-grid-smem` and 30+ // `mmvq-ksigns-smem`, measured there on a Tesla P100 (sm_60).+ for (int i = tid; i < 256; i += nthreads) s_buf[i] = iq3xxs_grid[i];- for (int u = tid; u < 128; u += nthreads) {- const uint32_t s = (uint32_t) u | ((__popc((uint32_t) u) & 1u) << 7); // 8th sign = parity of the 7- uint32_t ml = 0, mh = 0;-#pragma unroll- for (int i = 0; i < 4; ++i) {- ml |= ((s >> i) & 1u) * (0xffu << (8 * i));- mh |= ((s >> (4 + i)) & 1u) * (0xffu << (8 * i));- }- s_buf[256 + u] = ml;- s_buf[384 + u] = mh;- }++ stage_iq3xxs_masks(s_buf, tid, nthreads);+ __syncthreads();+ return s_buf;+ } else if constexpr (TY == 21) {+ // 512 codebook words, then two 256-word tables: the byte masks of a sign byte's bits 0-3 and of its bits+ // 4-7 (IQ3_S stores one sign byte per 8 values). As for IQ3_XXS, one pass per block replaces the+ // per-lookup __vcmpne4 / __vsub4 chain, and the codebook reads leave global memory.+ for (int i = tid; i < 512; i += nthreads) s_buf[i] = iq3s_grid[i];- for (int u = tid; u < 256; u += nthreads) {- uint32_t ml = 0, mh = 0;-#pragma unroll- for (int i = 0; i < 4; ++i) {- ml |= (((uint32_t) u >> i) & 1u) * (0xffu << (8 * i));- mh |= (((uint32_t) u >> (4 + i)) & 1u) * (0xffu << (8 * i));- }- s_buf[512 + u] = ml;- s_buf[768 + u] = mh;- }++ stage_iq3s_masks(s_buf, tid, nthreads);+ __syncthreads();+ return s_buf;} else {return nullptr;}@@@ -1721,369 -1715,6 +1783,376 @@@ __global__ void __launch_bounds__(256)}}+// ---------------------------------------------------------------- EXPDEC (STRATA_EXPDEC=1): the group's activations+// in shared memory, R rows per lane+//+// native_gu_multi_kernel / native_down_multi_kernel decode each weight part once and apply it to every entry of the+// group, but the gate/up apply re-reads the entry's q8_1 block from global memory with 9 scattered 4-byte loads+// (36-byte blocks: a warp's load touches 32 sectors), so an entry costs about 60% of a whole expert (nsys, 2080 Ti,+// L02 at 131k: ~2.6 us per expert and ~1.6 us per entry and layer) although it reads nothing from DRAM. Here a+// block stages its group's entries (ED_NC at a time, as the default) in shared memory laid out so that a call's+// words are two 16-byte loads (gate/up, Q2_0 down: the block's two halves in separate planes, conflict-free) or two+// 8-byte loads (IQ4_NL down), the fp16 block scale already a float; every lane computes R rows at once, so the+// activation words are loaded once per R rows; the IQ3_XXS / IQ3_S codebooks are staged too (IQ2_* already were),+// and a block takes `rb` rows (auto: the most that keep ~STRATA_EXPDEC_MINBLK blocks in flight) instead of 16.+// Each (row, entry) sum is the default kernel's sum tree (row_dot_80_sub16 / row_dot_40_sub8 / row_dot_20_sub16: the+// same lane does the same calls in the same order with the same operations, `apply_u` = `apply` on the block's+// words), so the output is bitwise the same (expdec_bench checks it; docs/EXPERTDEC.md).+#ifndef STRATA_EXPDEC_MINB+#define STRATA_EXPDEC_MINB 4 // gate/up __launch_bounds__ min blocks per SM by default (STRATA_EXPDEC_MINB at runtime): 4 = up to 64+ // registers, as the default kernels (bench: equal on the 2080 Ti, 0.87-0.95 on the 3060); 2 = up to 128+#endif+constexpr int ED_NC = GRP_NC; // entries staged at once+constexpr int ED_XB = 80; // q8_1 blocks of a gate/up row (n_embd 2560)+constexpr int ED_HB = 20; // q8_1 blocks of a down row (n_ff 640)++// the codebook of a gate/up format in shared memory: words, and the copy (every thread of the block; sync after)+template<int TG> struct EdGrid;+template<> struct EdGrid<16> { static constexpr int words = 512;+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2xxs_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 256; i += 256) dst[i] = src[i]; } };+template<> struct EdGrid<17> { static constexpr int words = 1024;+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2xs_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 512; i += 256) dst[i] = src[i]; } };+template<> struct EdGrid<22> { static constexpr int words = 2048;+ __device__ static void copy(uint32_t* s, int tid) { const uint2* src = (const uint2*) iq2s_grid; uint2* dst = (uint2*) s; for (int i = tid; i < 1024; i += 256) dst[i] = src[i]; } };- template<> struct EdGrid<18> { static constexpr int words = 256;- __device__ static void copy(uint32_t* s, int tid) { for (int i = tid; i < 256; i += 256) s[i] = iq3xxs_grid[i]; } };- template<> struct EdGrid<21> { static constexpr int words = 512;- __device__ static void copy(uint32_t* s, int tid) { for (int i = tid; i < 512; i += 256) s[i] = iq3s_grid[i]; } };++// (IQ3_XXS / IQ3_S: the codebook and, behind it, the sign-mask tables Split<TG>::load<true> reads - stage_iq_grid's layout)++template<> struct EdGrid<18> { static constexpr int words = 512;++ __device__ static void copy(uint32_t* s, int tid) {++ for (int i = tid; i < 256; i += 256) s[i] = iq3xxs_grid[i];++ stage_iq3xxs_masks(s, tid, 256);++ } };++template<> struct EdGrid<21> { static constexpr int words = 1024;++ __device__ static void copy(uint32_t* s, int tid) {++ for (int i = tid; i < 512; i += 256) s[i] = iq3s_grid[i];++ stage_iq3s_masks(s, tid, 256);++ } };++// One entry's q8_1 row (XB blocks of 36 bytes, 4-byte aligned) into shared memory as entry c: the 8 words of block k+// in planes (HALVES: s_u[(2 c + h) XB + k] = words 4h..4h+3, two 16-byte loads per call, conflict-free over 16+// consecutive k; else s_u[(c XB + k) 2 + h], the block's 32 bytes together, for the IQ4_NL pairs) and the fp16 scale+// as s_ds[c XB + k] = __low2float(ds), the value `apply` computes. All 256 threads.+template<int XB, bool HALVES>+__device__ __forceinline__ void ed_stage_entry(int4* __restrict__ s_u, float* __restrict__ s_ds, int c,+ const block_q8_1* __restrict__ src, int tid) {+ int* su = (int*) s_u;+ const int* w32 = (const int*) src;+ for (int w = tid; w < XB * 9; w += 256) {+ const int k = w / 9, j = w - k * 9;+ const int v = w32[w];+ if (j == 0) {+ const half2 h = *reinterpret_cast<const half2*>(&v);+ s_ds[c * XB + k] = __low2float(h);+ } else {+ const int q = j - 1;+ if constexpr (HALVES) su[((c * 2 + (q >> 2)) * XB + k) * 4 + (q & 3)] = v;+ else su[(c * XB + k) * 8 + q] = v;+ }+ }+}+template<int XB>+__device__ __forceinline__ void ed_load_u(const int4* __restrict__ s_u, int c, int k, int (&u)[8]) {+ const int4 a = s_u[(c * 2 + 0) * XB + k], b = s_u[(c * 2 + 1) * XB + k];+ u[0] = a.x; u[1] = a.y; u[2] = a.z; u[3] = a.w;+ u[4] = b.x; u[5] = b.y; u[6] = b.z; u[7] = b.w;+}++// R gate/up rows against the NC staged entries, lane t of a 16-lane sub-warp: row_dot_80_sub16's tree (calls t,+// t+32, t+64 into s; t+16, t+48 into s16; s = __fadd_rn(s, s16); xor 8-4-2-1), every call's weight part decoded once+// per row and the entry's block loaded once per NC x R products.+template<int TG, int R, int NC>+__device__ __forceinline__ void ed_gu_rows(const uint8_t* const* wr, const int4* __restrict__ s_u,+ const float* __restrict__ s_ds, const uint32_t* __restrict__ s_grid, int t,+ float (&s)[R][NC]) {+ using F = Fmt<TG>;+ using S = Split<TG>;+ float s16[R][NC];+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) { s[r][c] = 0.0f; s16[r][c] = 0.0f; }+#pragma unroll+ for (int j = 0; j < 3; ++j) {+ const int k = t + 32 * j;+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ typename S::W w[R];+#pragma unroll+ for (int r = 0; r < R; ++r) w[r] = S::template load<true>(wr[r], kbx, iqs, s_grid);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ int u[8];+ ed_load_u<ED_XB>(s_u, c, k, u);+ const float d8 = s_ds[c * ED_XB + k];+#pragma unroll+ for (int r = 0; r < R; ++r) s[r][c] += S::apply_u(w[r], u, d8);+ }+ }+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ const int k = t + 16 + 32 * j;+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ typename S::W w[R];+#pragma unroll+ for (int r = 0; r < R; ++r) w[r] = S::template load<true>(wr[r], kbx, iqs, s_grid);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ int u[8];+ ed_load_u<ED_XB>(s_u, c, k, u);+ const float d8 = s_ds[c * ED_XB + k];+#pragma unroll+ for (int r = 0; r < R; ++r) s16[r][c] += S::apply_u(w[r], u, d8);+ }+ }+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ s[r][c] = __fadd_rn(s[r][c], s16[r][c]);+#pragma unroll+ for (int o = 8; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);+ }+}++// gate/up: 16 sub-warps of 16 lanes, R rows each = 16 R rows per pass, `rb` rows per block (a multiple of 16 R; 2 n_ff+// is one too, so a pass is whole or absent for the entire block). Group stride as native_gu_multi_kernel.+template<int TG, int R, int MINB>+__global__ void __launch_bounds__(256, MINB) expdec_gu_kernel(const unsigned long long* __restrict__ grp_ptr,+ const int32_t* __restrict__ grp_start,+ const int32_t* __restrict__ n_groups,+ const int32_t* __restrict__ ent_tok,+ const block_q8_1* __restrict__ xq, NativeExpertLayout L,+ float* __restrict__ gate, float* __restrict__ up, int rb) {+ const int ng = *n_groups;+ if (blockIdx.y >= ng) return;+ __shared__ alignas(16) int4 s_u[ED_NC * 2 * ED_XB];+ __shared__ float s_ds[ED_NC * ED_XB];+ STRATA_SHARED_ALIGN16_U32(s_grid_buf, EdGrid<TG>::words);+ const int tid = threadIdx.x;+ EdGrid<TG>::copy(s_grid_buf, tid); // visible after the staging's __syncthreads+ const int sw = tid >> 4, t = tid & 15;+ const int nrow = 2 * (int) L.n_ff, n_ff = (int) L.n_ff;+ const int row_base = blockIdx.x * rb;+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {+ const uint8_t* blob = (const uint8_t*) grp_ptr[g];+ const int e0 = grp_start[g], e1 = grp_start[g + 1];+ for (int e = e0; e < e1; e += ED_NC) {+ const int n = min(ED_NC, e1 - e);+ __syncthreads();+ for (int c = 0; c < n; ++c) ed_stage_entry<ED_XB, true>(s_u, s_ds, c, xq + (size_t) ent_tok[e + c] * ED_XB, tid);+ __syncthreads();+ for (int p = 0; p * 16 * R < rb; ++p) {+ const int row0 = row_base + p * 16 * R + sw * R;+ if (row0 >= nrow) break; // uniform over the block+ const bool is_up = row0 >= n_ff;+ const int r0 = is_up ? row0 - n_ff : row0;+ const uint8_t* wr[R];+#pragma unroll+ for (int r = 0; r < R; ++r) wr[r] = blob + (is_up ? L.up_off : 0) + (size_t) (r0 + r) * L.gu_row;+ float* dst = is_up ? up : gate;+ auto run = [&](auto nc) {+ constexpr int NC = decltype(nc)::value;+ float s[R][NC];+ ed_gu_rows<TG, R, NC>(wr, s_u, s_ds, s_grid_buf, t, s);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (t == c)+#pragma unroll+ for (int r = 0; r < R; ++r) dst[(size_t) (e + c) * n_ff + r0 + r] = s[r][c];+ };+ switch (n) {+ case 1: run(std::integral_constant<int, 1>{}); break;+ case 2: run(std::integral_constant<int, 2>{}); break;+ case 3: run(std::integral_constant<int, 3>{}); break;+ default: run(std::integral_constant<int, 4>{}); break;+ }+ }+ }+ }+}++// R IQ4_NL down rows, lane t of an 8-lane sub-warp: row_dot_40_sub8's tree. s_p: the block's 8 words together+// (ed_stage_entry<ED_HB, false>), read as the int2 pairs (iqs, iqs + 1) and (iqs + 4, iqs + 5).+template<int R, int NC>+__device__ __forceinline__ void ed_d20_rows(const uint8_t* const* wr, const int2* __restrict__ s_p,+ const float* __restrict__ s_ds, int t, float (&s)[R][NC]) {+ using F = Fmt<20>;+ using S = Split<20>;+ auto term = [&](int k, int r, int c, const typename S::W& w) -> float {+ const int kbx = k / F::ipb, par = k & 1;+ const int2 p0 = s_p[(c * ED_HB + kbx) * 4 + par], p1 = s_p[(c * ED_HB + kbx) * 4 + 2 + par];+ return S::apply_u(w, p0, p1, s_ds[c * ED_HB + kbx]);+ };+ auto loadw = [&](int k, typename S::W (&w)[R]) {+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+#pragma unroll+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);+ };+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) s[r][c] = 0.0f;+#pragma unroll+ for (int j = 0; j < 2; ++j) { // k = t, t + 32+ const int k = t + 32 * j;+ typename S::W w[R];+ loadw(k, w);+#pragma unroll+ for (int c = 0; c < NC; ++c)+#pragma unroll+ for (int r = 0; r < R; ++r) s[r][c] += term(k, r, c, w[r]);+ }+ {+ const int k = t + 16;+ typename S::W w[R];+ loadw(k, w);+#pragma unroll+ for (int c = 0; c < NC; ++c)+#pragma unroll+ for (int r = 0; r < R; ++r) {+ float s16 = 0.0f;+ s16 += term(k, r, c, w[r]);+ s[r][c] = __fadd_rn(s[r][c], s16);+ }+ }+ float s8[R][NC];+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) s8[r][c] = 0.0f;+ {+ const int k = t + 8;+ typename S::W w[R];+ loadw(k, w);+#pragma unroll+ for (int c = 0; c < NC; ++c)+#pragma unroll+ for (int r = 0; r < R; ++r) s8[r][c] += term(k, r, c, w[r]);+ }+ {+ const int k = t + 24;+ typename S::W w[R];+ loadw(k, w);+#pragma unroll+ for (int c = 0; c < NC; ++c)+#pragma unroll+ for (int r = 0; r < R; ++r) {+ float s24 = 0.0f;+ s24 += term(k, r, c, w[r]);+ s[r][c] = __fadd_rn(s[r][c], __fadd_rn(s8[r][c], s24));+ }+ }+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c)+#pragma unroll+ for (int o = 4; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);+}++// R Q2_0 down rows, lane t of a 16-lane sub-warp: row_dot_20_sub16's tree (call t; call t + 16 for t < 4 into s16).+template<int R, int NC>+__device__ __forceinline__ void ed_d42_rows(const uint8_t* const* wr, const int4* __restrict__ s_u,+ const float* __restrict__ s_ds, int t, float (&s)[R][NC]) {+ using F = Fmt<42>;+ using S = Split<42>;+ {+ const int k = t, kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ typename S::W w[R];+#pragma unroll+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ int u[8];+ ed_load_u<ED_HB>(s_u, c, k, u);+ const float d8 = s_ds[c * ED_HB + k];+#pragma unroll+ for (int r = 0; r < R; ++r) { s[r][c] = 0.0f; s[r][c] += S::apply_u(w[r], u, d8); }+ }+ }+ float s16[R][NC];+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) s16[r][c] = 0.0f;+ if (t < 4) {+ const int k = t + 16, kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ typename S::W w[R];+#pragma unroll+ for (int r = 0; r < R; ++r) w[r] = S::load(wr[r], kbx, iqs);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ int u[8];+ ed_load_u<ED_HB>(s_u, c, k, u);+ const float d8 = s_ds[c * ED_HB + k];+#pragma unroll+ for (int r = 0; r < R; ++r) s16[r][c] += S::apply_u(w[r], u, d8);+ }+ }+#pragma unroll+ for (int r = 0; r < R; ++r)+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ s[r][c] = __fadd_rn(s[r][c], s16[r][c]);+#pragma unroll+ for (int o = 8; o > 0; o >>= 1) s[r][c] += __shfl_xor_sync(0xffffffffu, s[r][c], o);+ }+}++// down (IQ4_NL: 32 sub-warps of 8 lanes; Q2_0: 16 of 16), R rows per lane, `rb` rows per block (a multiple of the+// rows per pass; n_embd too). The entries' h rows are contiguous in hq (entry e at e * ED_HB).+template<int TD, int R>+__global__ void __launch_bounds__(256, STRATA_EXPDEC_MINB) expdec_down_kernel(const unsigned long long* __restrict__ grp_ptr,+ const int32_t* __restrict__ grp_start,+ const int32_t* __restrict__ n_groups,+ const int32_t* __restrict__ ent_dst,+ const block_q8_1* __restrict__ hq, NativeExpertLayout L,+ float* __restrict__ out, int rb) {+ const int ng = *n_groups;+ if (blockIdx.y >= ng) return;+ constexpr int LANES = TD == 20 ? 8 : 16;+ constexpr int RPP = (256 / LANES) * R; // rows per pass+ __shared__ alignas(16) int4 s_u[ED_NC * 2 * ED_HB];+ __shared__ float s_ds[ED_NC * ED_HB];+ const int tid = threadIdx.x;+ const int sw = tid / LANES, t = tid % LANES;+ const int nrow = (int) L.n_embd;+ const int row_base = blockIdx.x * rb;+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {+ const uint8_t* blob = (const uint8_t*) grp_ptr[g] + L.down_off;+ const int e0 = grp_start[g], e1 = grp_start[g + 1];+ for (int e = e0; e < e1; e += ED_NC) {+ const int n = min(ED_NC, e1 - e);+ __syncthreads();+ for (int c = 0; c < n; ++c) ed_stage_entry<ED_HB, TD == 42>(s_u, s_ds, c, hq + (size_t) (e + c) * ED_HB, tid);+ __syncthreads();+ for (int p = 0; p * RPP < rb; ++p) {+ const int row0 = row_base + p * RPP + sw * R;+ if (row0 >= nrow) break; // uniform over the block+ const uint8_t* wr[R];+#pragma unroll+ for (int r = 0; r < R; ++r) wr[r] = blob + (size_t) (row0 + r) * L.d_row;+ auto run = [&](auto nc) {+ constexpr int NC = decltype(nc)::value;+ float s[R][NC];+ if constexpr (TD == 20) ed_d20_rows<R, NC>(wr, (const int2*) s_u, s_ds, t, s);+ else ed_d42_rows<R, NC>(wr, s_u, s_ds, t, s);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (t == c) {+ const size_t o = (size_t) ent_dst[e + c] * L.n_embd + row0;+#pragma unroll+ for (int r = 0; r < R; ++r) out[o + r] = s[r][c];+ }+ };+ switch (n) {+ case 1: run(std::integral_constant<int, 1>{}); break;+ case 2: run(std::integral_constant<int, 2>{}); break;+ case 3: run(std::integral_constant<int, 3>{}); break;+ default: run(std::integral_constant<int, 4>{}); break;+ }+ }+ }+ }+}+// ---------------------------------------------------------------- q8_1 (quantize.cu)// value i of a q8_1 row set (the warp holds block i / 32, lane = i % 32)__device__ __forceinline__ void q8_1_store(const float xi, block_q8_1* __restrict__ y, const long long i) {diff --cc src/prefill/kernels.cuindex a637797ae,d6c6b99a0..a4f56f10c--- a/src/prefill/kernels.cu+++ b/src/prefill/kernels.cu@@@ -2529,6 -2114,45 +2662,44 @@@ void gdn_recurrence_variant(int variant}#else(void) variant; // the quad recurrence kernels are AMD-only+ #endif- static const bool serial = std::getenv("STRATA_GDN_REC_HEADS") != nullptr; // the one-block-per-head kernel (A/B)+ #if defined(STRATA_EXPERIMENTAL_SM60)+ // v100/gdn-chunk: the chunked recurrence (src/prefill/gdn_chunk.cu) on Volta, where the token chain is the+ // cost. STRATA_GDN_CHUNK=0 forces the recurrence, =1 forces the chunked path (the parity test's A/B switch);+ // unset: cc == 7.0 only, every other architecture keeps the recurrence bit-for-bit as before. Read per call+ // on purpose - cheap next to the work, and the tests flip it between two launches.+ const char* chunk_v = std::getenv("STRATA_GDN_CHUNK");+ const int chunk_env = chunk_v && *chunk_v ? std::atoi(chunk_v) : -1;+ static const bool volta = gdn_chunk_available(); // runtime cc == 7.0 (the kernels are compiled for sm_70 only)+ const bool chunked = !serial && T > 0 && (chunk_env >= 0 ? chunk_env != 0 : false);+ // v100/gdn-chunk: the sync-free warp-reduction recurrence (bit-exact vs gdn_rec_cols_pipe) is the+ // measured-fast path on Volta; the chunked recurrence is opt-in (STRATA_GDN_CHUNK=1/2) - correct and+ // measured, but slower than this on the V100 (bench/gdn_chunk_bench). STRATA_GDN_REC_WARP=0 restores+ // the old pipelined kernel bit-for-bit.+ // STRATA_GDN_REC_WARP (Volta only, opt-in): unset or 0 -> the pipelined kernel (the default; the chain split changes the+ // summation order, so it is not the default without a V100 check here), 2 -> the chain-split kernel (measured +6..9%, FP32-level), the+ // old pipelined kernel bit-for-bit is 0, 1 -> the sync-free bit-exact control.+ const char* warp_v = std::getenv("STRATA_GDN_REC_WARP");+ const int warp_env = warp_v && *warp_v ? std::atoi(warp_v) : 0;+ const bool warp_rec = !serial && !chunked && T > 0 && volta && warp_env != 0;+ if (chunked && T >= 64) {+ // the chunked prefix + the recurrence over the tail (< 64 tokens), then the out norm over all of T+ const int64_t T64 = T - T % 64;+ gdn_chunk_recurrence(state, h, gate, beta, y, T64, stream);+ if (T > T64)+ gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(+ state, h + T64 * C, gate + T64 * HV, beta + T64 * HV, y + T64 * (size_t) HV * S, T - T64);+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ } else if (warp_rec) {+ // =2: the 4-accumulator chain split on the pipelined structure (FP32-level, not bit-exact);+ // =1: the sync-free bit-exact warp reduction (measured slower - kept as the bitwise control)+ if (warp_env == 2)+ gdn_rec_cols_pipe_fast_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta,+ y, T);+ else+ gdn_rec_cols_warp_kernel<<<HV * NCB, S, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ } else#endifif (serial || T <= 0 || variant == 2) {gdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T, ld16);diff --cc src/prefill/prefill.cppindex 17428925e,ccce1faa5..bceacbda3--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@@ -1118,14 -1009,31 +1119,38 @@@ struct MmqPlanstd::vector<char> fo; // per layer: no MMQ here but the native fused kernels cover its formats// (gfx11: UD-Q4_K_XL's Q4_K / Q5_K and Q5_1 / Q8_0 experts, STRATA_PF_FUSED=1)size_t gu_max = 0, d_max = 0;+ // STRATA_MMQ_DIRECT=1: per layer, whether resident experts are read in their cache slots (pointer-table MMQ),+ // and the bytes the down product reads past its matrix there (zeroed in the slot's guard before each group)+ std::vector<char> direct;+ std::vector<uint32_t> d_over;+ // STRATA_MMQ_GROUP_SORT=1 (without STRATA_MMQ_RESIDENT_SORT_NE): per MMQ layer, whether both products' 16-expert+ // launches run whole tiles on every visible GPU (mmq::whole_tiles), so its full groups may be taken in row-count order+ std::vector<char> group_sort;};+ // Volta (sm_70): MMQ runs on dp4a there (no int8 tensor cores). pf_wmma(): the layers MMQ would take, and the IQ1_M+ // ones the FP16 path would, run as one launch per expert group on FP16 tensor-core tiles instead+ // (strata::kernels::gemm_iq_f16_grouped: each weight superblock dequantized once into shared memory by the FP16 path's+ // own formulas, FP16 activations, FP32 sums). OPT-IN: STRATA_PF_WMMA=1 (the activations are FP16 rather than MMQ's+ // q8_1, so the answers differ from the default). Only when EVERY visible CUDA card is compute capability 7.0 or+ // newer: the tiles need tensor cores, and a Volta + Pascal rig (one process, one decision) keeps MMQ.+ bool pf_wmma() {+ #if defined(__HIPCC__)+ return false;+ #else+ static const bool on = [] {+ const char* e = std::getenv("STRATA_PF_WMMA");+ if (e == nullptr || e[0] != '1') return false;+ int n = 0;+ if (cudaGetDeviceCount(&n) != cudaSuccess || n <= 0) return false;+ for (int d = 0; d < n; ++d) {+ int major = 0;+ if (cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, d) != cudaSuccess || major < 7) return false;+ }+ return true;+ }();+ return on;+ #endif+ }const MmqPlan& mmq_plan() {static const MmqPlan plan = [] {MmqPlan p;@@@ -3803,13 -3498,9 +3848,14 @@@ bool Prefill::run_impl(const int64_t* tconst size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;pt.mark(kPfGather, cs);if (use_mmq) {- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`- if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows- else mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);+ // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`:+ // each token quantized once and written to its K rows (slot_dev, the inverse of src_dev).+ // NVFP4 w4a4 activations carry a per-row scale (Xscale): that mode keeps the gather.- if (mmq::fp4_activations(mmq_gt))++ if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows (pf_wmma)++ else if (mmq::fp4_activations(mmq_gt))+ mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);+ else+ mmq::quantize_scatter(m.mixed, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down// reads the group's own quantized H)const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;@@@ -4200,15 -3876,13 +4246,15 @@@mmq::GatherGroup gg;int gg_slots[MMQ_GROUP];int gg_nslots = 0; // ring slots gathered by the next flush- // `zero`: the MMQ tail zeroed after the gathered experts (the group's last flush)+ // STRATA_MMQ_DIRECT=1: a resident expert with guard room in its slot is not gathered - the+ // group's pointer tables give MMQ each expert's own address (group position q), the others are+ // gathered at compact positions c = 0..ns-1 of the group buffers, as before (same bytes)- const bool direct = use_mmq && lay.native && m.cache && mmq_plan().direct[(size_t) l];++ const bool direct = use_mmq && lay.native && m.cache && mmq_plan().direct[(size_t) l] && !wmma_l;+ mmq::WeightTab tgu, td;+ mmq::GuardTab gd;+ int ns = 0;+ // `zero`: the MMQ tail zeroed after the gathered experts (the group's last flush); a direct+ // group's tail is zeroed by prep_groupauto flush = [&](size_t zero) {if (gg.n <= gg.first) return;const auto& f = lay.fmt[(size_t) l];@@@ -4310,13 -3951,19 +4356,26 @@@int64_t maxr = 0;for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, m.cnt[(size_t) order[i]]);pt.mark(kPfGemmGU, cs);+ if (wmma_l) { // pf_wmma: gate/up, SwiGLU to FP16, down - the group in 2 tile launches+ const int32_t* dnb = m.bounds_dev + n + 1 + g * (MMQ_GROUP + 1);+ const bool ok1 = strata::kernels::gemm_iq_f16_grouped(+ mmq_gt, m.grp_gu, mmq_gub, 1280, (int) N, m.Xs, (int) N, m.bounds_dev + j0, ngx,+ (int) maxr, m.GU, 1280, m.cs);+ swiglu_split_f16(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, 640, !lay.native, m.cs);+ pt.mark(kPfGemmD, cs);+ const bool ok2 = strata::kernels::gemm_iq_f16_grouped(+ mmq_dt, m.grp_d, mmq_db, (int) N, 640, m.Hh + r0 * 640, 640, dnb, ngx, (int) maxr,+ m.Dm + r0 * N, (int) N, m.cs);+ if (!ok1 || !ok2) { err = "prefill: the WMMA expert tiles could not launch"; return false; }+ return true;+ }+ if (direct) {+ // one launch: the direct experts' guards (their down over-read) and the zeroed tail+ // after the last gathered expert (see MMQ_TAIL)+ mmq::prep_group(gd, mmq_plan().d_over[(size_t) l],+ ns ? m.grp_gu + (size_t) ns * mmq_gub : nullptr,+ ns ? m.grp_d + (size_t) ns * mmq_db : nullptr, MMQ_TAIL, m.cs);+ }mmq::Product gu;gu.w = m.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;gu.n = ngx; gu.xq = m.Xq; gu.bounds = m.bounds_dev + j0; gu.ids = m.ids_identity;