Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
experts: gate/up, SwiGLU and q8_1 in one launch (bitwise; default on sm_120)
Niko122175afe3f4a084
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hppindex 31c031ca7..66b2c7940 100644--- a/include/strata/core/sm120_series.hpp+++ b/include/strata/core/sm120_series.hpp@@ -8,6 +8,9 @@// STRATA_ATTN_DAILY=1 the Daily's decode attention (attn_chunk_kernel_pre75 + merge v2)// STRATA_GR_UP75=1 the hyper-connection up projection with its rows' loads hoisted// STRATA_GR_DOWN_TG=1 the hyper-connection down projection with one token a block row (T times the blocks)+// STRATA_GU_SWIGLU=1 gate/up, SwiGLU and the q8_1 blocks of the routed experts in one launch+// and one that rounds differently (KL-checked, the AMD gfx12 rule): STRATA_PF_FUSED (the native packs' prompt experts on the+// fused int8 kernels, IQ2_S / IQ3_XXS / IQ3_S / IQ4_XS gate/up only; =0 keeps MMQ, =1 takes every covered format).//// STRATA_SM120_PERF=0 turns the whole set off on a 12.0 card (the defaults every other card has).// Each switch's own variable still wins in both directions (=0 off, =1 / =2 on), on any card, as before.diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex 3ae7e3355..1332335ac 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -6,6 +6,7 @@// included unchanged.#include "strata/kernels/iq_kernels.hpp"#include "strata/core/sm75_series.hpp"+#include "strata/core/sm120_series.hpp"#include "strata/kernels/gfx_arch.hpp"#include "strata/kernels/dp4a.hpp"#include "strata/kernels/q8_1_finite.hpp"@@ -1312,6 +1313,55 @@ __device__ __forceinline__ void row_dot_80_sub16(const uint8_t* row, const block}}+// 147-verify: row_dot_80_sub16 for two rows (a gate row and its up row) at once, both rows' weight loads of a step issued+// before either is applied; each row's sums are row_dot_80_sub16's, in its order (the same bits).+template<int TY, int NC, bool EXACT_N = false, bool STAGE_GRID = false>+__device__ __forceinline__ void row_dot2_80_sub16(const uint8_t* ra, const uint8_t* rb, const block_q8_1* x, const int (&off)[NC],+ int n, int t, float (&sa)[NC], float (&sb)[NC],+ const uint32_t* __restrict__ s_grid = nullptr) {+ using F = Fmt<TY>;+ using S = Split<TY>;+#pragma unroll+ for (int c = 0; c < NC; ++c) { sa[c] = 0.0f; sb[c] = 0.0f; }+ for (int k = t; k < 80; k += 32) {+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ const typename S::W wa = S::template load<STAGE_GRID>(ra, kbx, iqs, s_grid);+ const typename S::W wb = S::template load<STAGE_GRID>(rb, kbx, iqs, s_grid);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (EXACT_N || c < n) {+ sa[c] += S::apply(wa, x + off[c] + kbx * (F::qk / 32), iqs);+ sb[c] += S::apply(wb, x + off[c] + kbx * (F::qk / 32), iqs);+ }+ }+ float a16[NC], b16[NC];+#pragma unroll+ for (int c = 0; c < NC; ++c) { a16[c] = 0.0f; b16[c] = 0.0f; }+ for (int k = t + 16; k < 64; k += 32) {+ const int kbx = k / F::ipb, iqs = F::step * (k % F::ipb);+ const typename S::W wa = S::template load<STAGE_GRID>(ra, kbx, iqs, s_grid);+ const typename S::W wb = S::template load<STAGE_GRID>(rb, kbx, iqs, s_grid);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (EXACT_N || c < n) {+ a16[c] += S::apply(wa, x + off[c] + kbx * (F::qk / 32), iqs);+ b16[c] += S::apply(wb, x + off[c] + kbx * (F::qk / 32), iqs);+ }+ }+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ if (EXACT_N || c < n) {+ sa[c] = __fadd_rn(sa[c], a16[c]);+ sb[c] = __fadd_rn(sb[c], b16[c]);+#pragma unroll+ for (int o = 8; o > 0; o >>= 1) {+ sa[c] += __shfl_xor_sync(0xffffffffu, sa[c], o);+ sb[c] += __shfl_xor_sync(0xffffffffu, sb[c], o);+ }+ }+ }+}+// The sign-mask tables that follow the codebook words in a staged IQ3_XXS / IQ3_S table (see stage_iq_grid): the byte// masks of every 7-bit sign field (IQ3_XXS, 2 x 128 words at +256) and of a sign byte's bits 0-3 and 4-7 (IQ3_S, 2 x 256// words at +512). Split<18>::load<true> and Split<21>::load<true> read them, so every kernel that stages these two@@ -2203,6 +2253,82 @@ __global__ void __launch_bounds__(256) swiglu_q8_1_entries_kernel(const float* _}}+// 147-verify (STRATA_GU_SWIGLU=1): native_gu_multi_kernel's 16-lane gate and up rows, SwiGLU and the q8_1 blocks in ONE+// launch. A block owns 32 consecutive hidden columns r of a group: its 32 sub-warps each compute gate row r and up row+// r (the same row_dot_80_sub16 calls, entries GRP_NC at a time, the same NVFP4 tail scale), so the product+// h = silu(gate) * up is formed in the block and warp k quantizes entry k's 32 values with q8_1_store - the q8_1 block+// of those columns. Every value is the separate kernels': the dots are the same calls, gate and up are the same floats+// (written to memory before), the product is rounded on its own as in swiglu_q8_1_entries_kernel, and q8_1_store is the+// same. It saves the gate/up round trip through memory and one launch per layer.+template<int TG, bool STAGE_GRID>+__global__ void __launch_bounds__(512) native_gu_swiglu_kernel(const unsigned long long* __restrict__ grp_ptr,+ const int32_t* __restrict__ grp_start,+ const int32_t* __restrict__ n_groups,+ const int32_t* __restrict__ ent_tok,+ const block_q8_1* __restrict__ xq, NativeExpertLayout L,+ block_q8_1* __restrict__ hq) {+#if defined(__HIPCC__)+#pragma clang fp contract(off)+#endif+ const int ng = *n_groups;+ if ((int) blockIdx.y >= ng) return;+ STRATA_SHARED_ALIGN16_U32(s_grid_buf, IqGridWords<TG, STAGE_GRID>::value);+ const uint32_t* s_grid = stage_iq_grid<TG, STAGE_GRID>(s_grid_buf, threadIdx.x, 512);+ __shared__ float sh[GRP_NC][32];+ const int sw = threadIdx.x >> 4, t = threadIdx.x & 15, warp = threadIdx.x >> 5, lane = threadIdx.x & 31;+ const int r = blockIdx.x * 32 + sw; // the hidden column (gate row r, up row r)+ const size_t g_off = (size_t) r * L.gu_row, u_off = L.up_off + (size_t) r * L.gu_row;+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {+ const uint8_t* blob = (const uint8_t*) grp_ptr[g];+ const float sg = L.tail_off ? ((const float*) (blob + L.tail_off))[0] : 1.0f; // NVFP4: s_gate / s_up+ const float su = L.tail_off ? ((const float*) (blob + L.tail_off))[1] : 1.0f;+ const int e0 = grp_start[g], e1 = grp_start[g + 1];+ for (int e = e0; e < e1; e += GRP_NC) {+ const int n = min(GRP_NC, e1 - e);+ float gv[GRP_NC], uv[GRP_NC];+ if (n == 1) {+ const int off1[1] = { ent_tok[e] * 80 };+ float a1[1], b1[1];+ row_dot2_80_sub16<TG, 1, true, STAGE_GRID>(blob + g_off, blob + u_off, xq, off1, 1, t, a1, b1, s_grid);+ gv[0] = a1[0]; uv[0] = b1[0];+ } else if (n == 2) {+ const int off2[2] = { ent_tok[e] * 80, ent_tok[e + 1] * 80 };+ float a2[2], b2[2];+ row_dot2_80_sub16<TG, 2, true, STAGE_GRID>(blob + g_off, blob + u_off, xq, off2, 2, t, a2, b2, s_grid);+#pragma unroll+ for (int c = 0; c < 2; ++c) { gv[c] = a2[c]; uv[c] = b2[c]; }+ } else if (n == 3) {+ const int off3[3] = { ent_tok[e] * 80, ent_tok[e + 1] * 80, ent_tok[e + 2] * 80 };+ float a3[3], b3[3];+ row_dot2_80_sub16<TG, 3, true, STAGE_GRID>(blob + g_off, blob + u_off, xq, off3, 3, t, a3, b3, s_grid);+#pragma unroll+ for (int c = 0; c < 3; ++c) { gv[c] = a3[c]; uv[c] = b3[c]; }+ } else {+ const int off4[4] = { ent_tok[e] * 80, ent_tok[e + 1] * 80, ent_tok[e + 2] * 80, ent_tok[e + 3] * 80 };+ float a4[4], b4[4];+ row_dot2_80_sub16<TG, 4, true, STAGE_GRID>(blob + g_off, blob + u_off, xq, off4, 4, t, a4, b4, s_grid);+#pragma unroll+ for (int c = 0; c < 4; ++c) { gv[c] = a4[c]; uv[c] = b4[c]; }+ }+ // sub-lane c holds entry c's sums (as the separate kernel's store): h of column r, entry c+#pragma unroll+ for (int c = 0; c < GRP_NC; ++c) {+ if (c < n && t == c) {+ const float gg = gv[c] * sg, uu = uv[c] * su;+#if defined(__HIPCC__)+ sh[c][sw] = (gg / (1.0f + __expf(-gg))) * uu;+#else+ sh[c][sw] = __fmul_rn(gg / (1.0f + __expf(-gg)), uu);+#endif+ }+ }+ __syncthreads();+ if (warp < n) q8_1_store(sh[warp][lane], hq, (long long) (e + warp) * L.n_ff + (long long) blockIdx.x * 32 + lane);+ __syncthreads();+ }+ }+}+// ---------------------------------------------------------------- dequant (dequantize.cuh)template<typename dst_t> __device__ __forceinline__ dst_t cvt(float v);template<> __device__ __forceinline__ float cvt<float>(float v) { return v; }@@ -3506,6 +3632,50 @@ void launch_gu(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, con}}+// 147-verify: STRATA_GU_SWIGLU=1 runs gate/up, SwiGLU and the q8_1 blocks as native_gu_swiglu_kernel, where launch_gu would+// run native_gu_multi_kernel's 16-lane path (n_embd 2560, the split formats); bitwise the same hq. Unset: on for a compute+// capability 12.0 card (sm120_series.hpp; RTX PRO 5000, 6 pairs: decode +0.6% story, +0.9% code, +0.5% 8K, 6/6 each, the+// same tokens), off elsewhere; STRATA_GU_SWIGLU=0 turns it off.+bool gu_swiglu_on() {+ static const int env = [] { const char* v = std::getenv("STRATA_GU_SWIGLU"); return v != nullptr ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();+ if (env >= 0) return env != 0;+#if defined(__HIPCC__)+ return false;+#else+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }+ if (!per_dev[dev]) {+ int maj = 0, mnr = 0;+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;+ if (!ok) cudaGetLastError();+ per_dev[dev] = ok && strata::sm120_default(maj, mnr) ? 2 : 1;+ }+ return per_dev[dev] == 2;+#endif+}+template<int TG>+bool launch_gu_swiglu(const NativeExpertLayout& L, int64_t gy, cudaStream_t s, const unsigned long long* grp_ptr,+ const int32_t* grp_start, const int32_t* n_groups, const int32_t* ent_tok, const block_q8_1* X,+ block_q8_1* hq) {+ if constexpr (!kSplit<TG>) {+ return false;+ } else {+ if (g_old_kernels || g_no_sub16_gu || L.n_embd != 2560 || L.n_ff % 32 != 0 || L.n_ff <= 0) return false;+ const dim3 grid((unsigned) (L.n_ff / 32), (unsigned) gy);+ bool stage = false;+ if constexpr (kStageIqGrid<TG>) stage = g_stage_grid && !((TG == 18 || TG == 21) && !stage_tables_on());+ if constexpr (kStageIqGrid<TG>) {+ if (stage) native_gu_swiglu_kernel<TG, true><<<grid, 512, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, hq);+ else native_gu_swiglu_kernel<TG, false><<<grid, 512, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, hq);+ } else {+ native_gu_swiglu_kernel<TG, false><<<grid, 512, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, hq);+ }+ return true;+ }+}+template<int TD>void launch_down(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, const int32_t* grp_start,const int32_t* n_groups, const int32_t* ent_dst, const block_q8_1* hq, const NativeExpertLayout& L,@@ -4430,9 +4600,27 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long}check("native_expert_grouped/gu fused");}- if (g_exp_phase != 2 && !fused_gu) {+#endif+ // 147-verify STRATA_GU_SWIGLU=1: gate/up + SwiGLU + q8_1 in one launch where the CUDA layout's 16-lane kernel would run+ bool gs_done = false;+#if STRATA_EXP_LAYOUTS+ const bool gs_layout = !fused_gu && em0 == 0;+#else+ const bool gs_layout = true;+#endif+ if (gs_layout && g_exp_phase == 0 && !v1 && gu_swiglu_on() && !(g_expdec && !g_old_kernels && expdec_gu_ok(L))) {+ switch (L.gu_type) {+#define STRATA_GS(T) case T: gs_done = launch_gu_swiglu<T>(L, gy, s, grp_ptr, grp_start, n_groups, ent_tok, X, hq); break;+ STRATA_GU_FMTS(STRATA_GS)+#undef STRATA_GS+ default: break;+ }+ if (gs_done) check("native_expert_grouped/gu+swiglu");+ }+#if STRATA_EXP_LAYOUTS+ if (g_exp_phase != 2 && !fused_gu && !gs_done) {#else- if (g_exp_phase != 2) {+ if (g_exp_phase != 2 && !gs_done) {#endif#if STRATA_EXP_LAYOUTSconst bool lds_gu = (em0 == 5 || em0 == 6 || em0 == 7 || em0 == 8) && ((kExpLds16 && L.gu_type == 16) || L.gu_type == 18 || L.gu_type == 21 || L.gu_type == 22 || L.gu_type == 23);