Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Gyro trellis experts on compute capability 6.0: row chunks per block so the staged codebook is paid once per several rows; bench: rows of different lengths
Niko1221c864d804b603
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex 3a6844ccd..a6b45a7a1 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -1879,8 +1879,13 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon}}const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;- const int row = blockIdx.x * GU_ROWS + warp; // 0 .. 2*n_ff- if (row >= 2 * L.n_ff) return;+ // cc 6.0 trellis experts (staged codebook): a block takes the row chunks blockIdx.x, + gridDim.x, ... so the 4 KB+ // codebook staging above is paid once for several chunks (the launch shrinks the grid to match); every other+ // launch has one chunk per block and this is the single pass it always was+ constexpr bool kRowLoop = kTqType<TG> && STAGE_GRID;+ for (int rc = blockIdx.x; ; rc += gridDim.x) {+ const int row = rc * GU_ROWS + warp; // 0 .. 2*n_ff+ if (row >= 2 * L.n_ff) break;const bool is_up = row >= L.n_ff;const int r = is_up ? row - (int) L.n_ff : row;const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;@@ -1915,6 +1920,8 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon}}}+ if constexpr (!kRowLoop) break;+ }}// APR (prism.hadamard): y = H_block (signs * x) per block of each row, block = blockDim.x (a power of two <= 1024),@@ -2000,7 +2007,56 @@ __global__ void __launch_bounds__(256) native_down_multi_kernel(const unsigned lconst uint32_t* s_grid = stage_iq_grid<TD, STAGE_TQ>(s_grid_buf, threadIdx.x, 256);__shared__ uint32_t s_hq_buf[GRP_NC * 20 * 9];const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);- if constexpr (SUB_DOWN && (TD == 20 || TD == 40 || (kTqType<TD> && STAGE_TQ))) { // IQ4_NL (K 40), NVFP4 (K 20), and with the staged codebook the trellis types (K 40 at n_ff 640): 8-lane rows+ if constexpr (SUB_DOWN && kTqType<TD> && STAGE_TQ) {+ // cc 6.0 trellis down rows (K = 40 dots at n_ff 640, the IQ4_NL row shape): 8 lanes a row, 32 rows a chunk, and a+ // block takes the chunks blockIdx.x, + gridDim.x, ... of one staged codebook and one staged set of activations+ // (the launch shrinks the grid to match). The sums are row_dot_40_sub8's, as the IQ4_NL rows.+ if (hb == 20) {+ const int t = threadIdx.x & 7;+ const block_q8_1* s_hq = reinterpret_cast<const block_q8_1*>(s_hq_buf);+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {+ const uint8_t* gp = (const uint8_t*) grp_ptr[g];+ const float sd = L.tail_off ? ((const float*) (grp_ptr[g] + L.tail_off))[2] : 1.0f;+ const int e0 = grp_start[g], e1 = grp_start[g + 1];+ for (int e = e0; e < e1; e += GRP_NC) {+ const int n = min(GRP_NC, e1 - e);+ const uint32_t* src = reinterpret_cast<const uint32_t*>(hq + (size_t) e * 20);+ const int words = n * (20 * 9);+ __syncthreads();+ for (int i = threadIdx.x; i < words; i += 256) s_hq_buf[i] = src[i];+ __syncthreads();+ for (int rc = blockIdx.x; rc * 32 < L.n_embd; rc += gridDim.x) {+ const int r = rc * 32 + (threadIdx.x >> 3);+ if (r >= L.n_embd) continue;+ const uint8_t* wr = gp + L.down_off + (size_t) r * L.d_row;+ if (n == 1) {+ const int off1[1] = { 0 };+ float s1[1];+ row_dot_sub8<TD, 1, true, STAGE_TQ>(wr, s_hq, off1, 1, t, s1, s_grid);+ if (t == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s1[0] * sd;+ } else if (n == 2) {+ const int off2[2] = { 0, 20 };+ float s2[2];+ row_dot_sub8<TD, 2, true, STAGE_TQ>(wr, s_hq, off2, 2, t, s2, s_grid);+ if (t < 2) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s2[t] * sd;+ } else if (n == 3) {+ const int off3[3] = { 0, 20, 40 };+ float s3[3];+ row_dot_sub8<TD, 3, true, STAGE_TQ>(wr, s_hq, off3, 3, t, s3, s_grid);+ if (t < 3) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s3[t] * sd;+ } else {+ const int off4[4] = { 0, 20, 40, 60 };+ float s4[4];+ row_dot_sub8<TD, 4, true, STAGE_TQ>(wr, s_hq, off4, 4, t, s4, s_grid);+ if (t < 4) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s4[t] * sd;+ }+ }+ }+ }+ return;+ }+ } else+ if constexpr (SUB_DOWN && (TD == 20 || TD == 40)) { // IQ4_NL (K 40) and NVFP4 (K 20): 8-lane rowsif (hb == 20) {const int r = blockIdx.x * 32 + (threadIdx.x >> 3);const int t = threadIdx.x & 7;@@ -3139,6 +3195,15 @@ bool tq_stage_on() {return per_dev[dev] == 2;#endif}+// row chunks per block of the staged trellis gate/up and down kernels (STRATA_TQ_ROWCH, default 4)+int tq_row_chunks() {+ static const int n = [] {+ const char* v = std::getenv("STRATA_TQ_ROWCH");+ const int k = v != nullptr ? std::atoi(v) : 4;+ return k < 1 ? 1 : k > 32 ? 32 : k;+ }();+ return n;+}template<int TY, bool STAGE_GRID = kStageIqGrid<TY>>void launch_mmvq_multi_nc(dim3 grid, dim3 block, cudaStream_t s, const uint8_t* W, size_t rb, const block_q8_1* X,@@ -3871,8 +3936,10 @@ void launch_gu(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, conelse native_gu_multi_kernel<TG, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);return;}- if (g_no_sub16_gu) native_gu_multi_kernel<TG, true, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);- else native_gu_multi_kernel<TG, true, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);+ dim3 sg = grid;+ if constexpr (kTqType<TG>) sg.x = (grid.x + tq_row_chunks() - 1) / tq_row_chunks(); // the staged codebook is paid once per chunks+ if (g_no_sub16_gu) native_gu_multi_kernel<TG, true, false><<<sg, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);+ else native_gu_multi_kernel<TG, true, true><<<sg, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);} else {if (g_no_sub16_gu) native_gu_multi_kernel<TG, false, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);else native_gu_multi_kernel<TG, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);@@ -5006,9 +5073,10 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long longcheck("native_expert_grouped/swiglu");}if (g_exp_phase == 1) return;- const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && tq_stage_on(); // the trellis down rows as 8-lane rows (cc 6.0)+ const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && !g_old_kernels && !g_no_sub16_gu && L.n_ff == 640 && tq_stage_on(); // the trellis down rows as 8-lane rows (cc 6.0)const int d_rows = (!g_old_kernels && !g_no_sub16_gu && L.n_ff == 640) ? (L.d_type == 20 || L.d_type == 40 || tq_sub8 ? 32 : (L.d_type == 42 ? 16 : 8)) : 8;- const dim3 gd((unsigned) ((L.n_embd + d_rows - 1) / d_rows), (unsigned) gy);+ const int tq_ch = tq_sub8 ? tq_row_chunks() : 1; // the staged trellis down kernel takes this many 32-row chunks a block+ const dim3 gd((unsigned) (((L.n_embd + d_rows - 1) / d_rows + tq_ch - 1) / tq_ch), (unsigned) gy);#if STRATA_EXP_LAYOUTSconst int em0d = sm75_down_em(em0, cap_groups);if ((em0d == 7 || em0d == 8) && (L.d_type == 20 || L.d_type == 42)) {diff --git a/src/kernels/tq_pascal_bench.cpp b/src/kernels/tq_pascal_bench.cppindex 38c6394cd..e6d2c7cfc 100644--- a/src/kernels/tq_pascal_bench.cpp+++ b/src/kernels/tq_pascal_bench.cpp@@ -245,6 +245,24 @@ void bench(cudaStream_t s, std::mt19937& rng) {std::printf("\n");cudaFree(dw); cudaFree(dx); cudaFree(dxq); cudaFree(dy);}+ // the same 21M weights as rows of different lengths: a per-row cost shows as a falling time with longer rows+ for (int t : {145}) {+ for (auto rc : {std::pair<int, int>{8192, 2560}, {4096, 5120}, {2048, 10240}, {1024, 20480}}) {+ const int ni = rc.second, no = rc.first;+ const auto w = random_rows(t, no, ni, rng);+ uint8_t* dw = dalloc<uint8_t>(w.size());+ ck(cudaMemcpy(dw, w.data(), w.size(), cudaMemcpyHostToDevice), "w");+ const auto x = random_x((size_t) 8 * ni, rng);+ float* dx = dalloc<float>(x.size());+ ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");+ uint8_t* dxq = dalloc<uint8_t>((size_t) 8 * (ni / 32) * 36);+ k::quantize_q8_1_rows(dx, 8, ni, dxq, s);+ float* dy = dalloc<float>((size_t) 8 * no);+ const float us = 1e3f * time_ms(s, it, [&] { k::iq_mmvq(t, dw, dxq, dy, ni, no, 1, s); });+ std::printf("BENCH rows %-6s %5d x %5d nc1 %.1f us\n", name_of(t), no, ni, us);+ cudaFree(dw); cudaFree(dx); cudaFree(dxq); cudaFree(dy);+ }+ }// the verify window's shape: 2560 x 640 experts, 16 groups of m entries eachfor (auto gd : {std::pair<int, int>{145, 146}, {145, 144}, {144, 144}, {144, 146}}) {std::printf("BENCH grouped %-6s/%-6s 16 groups of m entries:", name_of(gd.first), name_of(gd.second));