Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Gyro trellis experts on compute capability 6.0: row chunks per block so the staged codebook is paid once per several rows; bench: rows of different lengths

Niko1221c864d804b603
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index 3a6844ccd..a6b45a7a1 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -1879,8 +1879,13 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon
}
}
const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
- const int row = blockIdx.x * GU_ROWS + warp; // 0 .. 2*n_ff
- if (row >= 2 * L.n_ff) return;
+ // cc 6.0 trellis experts (staged codebook): a block takes the row chunks blockIdx.x, + gridDim.x, ... so the 4 KB
+ // codebook staging above is paid once for several chunks (the launch shrinks the grid to match); every other
+ // launch has one chunk per block and this is the single pass it always was
+ constexpr bool kRowLoop = kTqType<TG> && STAGE_GRID;
+ for (int rc = blockIdx.x; ; rc += gridDim.x) {
+ const int row = rc * GU_ROWS + warp; // 0 .. 2*n_ff
+ if (row >= 2 * L.n_ff) break;
const bool is_up = row >= L.n_ff;
const int r = is_up ? row - (int) L.n_ff : row;
const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
@@ -1915,6 +1920,8 @@ __global__ void __launch_bounds__(256) native_gu_multi_kernel(const unsigned lon
}
}
}
+ if constexpr (!kRowLoop) break;
+ }
}
// APR (prism.hadamard): y = H_block (signs * x) per block of each row, block = blockDim.x (a power of two <= 1024),
@@ -2000,7 +2007,56 @@ __global__ void __launch_bounds__(256) native_down_multi_kernel(const unsigned l
const uint32_t* s_grid = stage_iq_grid<TD, STAGE_TQ>(s_grid_buf, threadIdx.x, 256);
__shared__ uint32_t s_hq_buf[GRP_NC * 20 * 9];
const int nb = (int) (L.n_ff / Fmt<TD>::qk), hb = (int) (L.n_ff / 32);
- if constexpr (SUB_DOWN && (TD == 20 || TD == 40 || (kTqType<TD> && STAGE_TQ))) { // IQ4_NL (K 40), NVFP4 (K 20), and with the staged codebook the trellis types (K 40 at n_ff 640): 8-lane rows
+ if constexpr (SUB_DOWN && kTqType<TD> && STAGE_TQ) {
+ // cc 6.0 trellis down rows (K = 40 dots at n_ff 640, the IQ4_NL row shape): 8 lanes a row, 32 rows a chunk, and a
+ // block takes the chunks blockIdx.x, + gridDim.x, ... of one staged codebook and one staged set of activations
+ // (the launch shrinks the grid to match). The sums are row_dot_40_sub8's, as the IQ4_NL rows.
+ if (hb == 20) {
+ const int t = threadIdx.x & 7;
+ const block_q8_1* s_hq = reinterpret_cast<const block_q8_1*>(s_hq_buf);
+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+ const uint8_t* gp = (const uint8_t*) grp_ptr[g];
+ const float sd = L.tail_off ? ((const float*) (grp_ptr[g] + L.tail_off))[2] : 1.0f;
+ const int e0 = grp_start[g], e1 = grp_start[g + 1];
+ for (int e = e0; e < e1; e += GRP_NC) {
+ const int n = min(GRP_NC, e1 - e);
+ const uint32_t* src = reinterpret_cast<const uint32_t*>(hq + (size_t) e * 20);
+ const int words = n * (20 * 9);
+ __syncthreads();
+ for (int i = threadIdx.x; i < words; i += 256) s_hq_buf[i] = src[i];
+ __syncthreads();
+ for (int rc = blockIdx.x; rc * 32 < L.n_embd; rc += gridDim.x) {
+ const int r = rc * 32 + (threadIdx.x >> 3);
+ if (r >= L.n_embd) continue;
+ const uint8_t* wr = gp + L.down_off + (size_t) r * L.d_row;
+ if (n == 1) {
+ const int off1[1] = { 0 };
+ float s1[1];
+ row_dot_sub8<TD, 1, true, STAGE_TQ>(wr, s_hq, off1, 1, t, s1, s_grid);
+ if (t == 0) out[(size_t) ent_dst[e] * L.n_embd + r] = s1[0] * sd;
+ } else if (n == 2) {
+ const int off2[2] = { 0, 20 };
+ float s2[2];
+ row_dot_sub8<TD, 2, true, STAGE_TQ>(wr, s_hq, off2, 2, t, s2, s_grid);
+ if (t < 2) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s2[t] * sd;
+ } else if (n == 3) {
+ const int off3[3] = { 0, 20, 40 };
+ float s3[3];
+ row_dot_sub8<TD, 3, true, STAGE_TQ>(wr, s_hq, off3, 3, t, s3, s_grid);
+ if (t < 3) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s3[t] * sd;
+ } else {
+ const int off4[4] = { 0, 20, 40, 60 };
+ float s4[4];
+ row_dot_sub8<TD, 4, true, STAGE_TQ>(wr, s_hq, off4, 4, t, s4, s_grid);
+ if (t < 4) out[(size_t) ent_dst[e + t] * L.n_embd + r] = s4[t] * sd;
+ }
+ }
+ }
+ }
+ return;
+ }
+ } else
+ if constexpr (SUB_DOWN && (TD == 20 || TD == 40)) { // IQ4_NL (K 40) and NVFP4 (K 20): 8-lane rows
if (hb == 20) {
const int r = blockIdx.x * 32 + (threadIdx.x >> 3);
const int t = threadIdx.x & 7;
@@ -3139,6 +3195,15 @@ bool tq_stage_on() {
return per_dev[dev] == 2;
#endif
}
+// row chunks per block of the staged trellis gate/up and down kernels (STRATA_TQ_ROWCH, default 4)
+int tq_row_chunks() {
+ static const int n = [] {
+ const char* v = std::getenv("STRATA_TQ_ROWCH");
+ const int k = v != nullptr ? std::atoi(v) : 4;
+ return k < 1 ? 1 : k > 32 ? 32 : k;
+ }();
+ return n;
+}
template<int TY, bool STAGE_GRID = kStageIqGrid<TY>>
void launch_mmvq_multi_nc(dim3 grid, dim3 block, cudaStream_t s, const uint8_t* W, size_t rb, const block_q8_1* X,
@@ -3871,8 +3936,10 @@ void launch_gu(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, con
else native_gu_multi_kernel<TG, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
return;
}
- if (g_no_sub16_gu) native_gu_multi_kernel<TG, true, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
- else native_gu_multi_kernel<TG, true, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
+ dim3 sg = grid;
+ if constexpr (kTqType<TG>) sg.x = (grid.x + tq_row_chunks() - 1) / tq_row_chunks(); // the staged codebook is paid once per chunks
+ if (g_no_sub16_gu) native_gu_multi_kernel<TG, true, false><<<sg, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
+ else native_gu_multi_kernel<TG, true, true><<<sg, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
} else {
if (g_no_sub16_gu) native_gu_multi_kernel<TG, false, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
else native_gu_multi_kernel<TG, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
@@ -5006,9 +5073,10 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
check("native_expert_grouped/swiglu");
}
if (g_exp_phase == 1) return;
- const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && tq_stage_on(); // the trellis down rows as 8-lane rows (cc 6.0)
+ const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && !g_old_kernels && !g_no_sub16_gu && L.n_ff == 640 && tq_stage_on(); // the trellis down rows as 8-lane rows (cc 6.0)
const int d_rows = (!g_old_kernels && !g_no_sub16_gu && L.n_ff == 640) ? (L.d_type == 20 || L.d_type == 40 || tq_sub8 ? 32 : (L.d_type == 42 ? 16 : 8)) : 8;
- const dim3 gd((unsigned) ((L.n_embd + d_rows - 1) / d_rows), (unsigned) gy);
+ const int tq_ch = tq_sub8 ? tq_row_chunks() : 1; // the staged trellis down kernel takes this many 32-row chunks a block
+ const dim3 gd((unsigned) (((L.n_embd + d_rows - 1) / d_rows + tq_ch - 1) / tq_ch), (unsigned) gy);
#if STRATA_EXP_LAYOUTS
const int em0d = sm75_down_em(em0, cap_groups);
if ((em0d == 7 || em0d == 8) && (L.d_type == 20 || L.d_type == 42)) {
diff --git a/src/kernels/tq_pascal_bench.cpp b/src/kernels/tq_pascal_bench.cpp
index 38c6394cd..e6d2c7cfc 100644
--- a/src/kernels/tq_pascal_bench.cpp
+++ b/src/kernels/tq_pascal_bench.cpp
@@ -245,6 +245,24 @@ void bench(cudaStream_t s, std::mt19937& rng) {
std::printf("\n");
cudaFree(dw); cudaFree(dx); cudaFree(dxq); cudaFree(dy);
}
+ // the same 21M weights as rows of different lengths: a per-row cost shows as a falling time with longer rows
+ for (int t : {145}) {
+ for (auto rc : {std::pair<int, int>{8192, 2560}, {4096, 5120}, {2048, 10240}, {1024, 20480}}) {
+ const int ni = rc.second, no = rc.first;
+ const auto w = random_rows(t, no, ni, rng);
+ uint8_t* dw = dalloc<uint8_t>(w.size());
+ ck(cudaMemcpy(dw, w.data(), w.size(), cudaMemcpyHostToDevice), "w");
+ const auto x = random_x((size_t) 8 * ni, rng);
+ float* dx = dalloc<float>(x.size());
+ ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
+ uint8_t* dxq = dalloc<uint8_t>((size_t) 8 * (ni / 32) * 36);
+ k::quantize_q8_1_rows(dx, 8, ni, dxq, s);
+ float* dy = dalloc<float>((size_t) 8 * no);
+ const float us = 1e3f * time_ms(s, it, [&] { k::iq_mmvq(t, dw, dxq, dy, ni, no, 1, s); });
+ std::printf("BENCH rows %-6s %5d x %5d nc1 %.1f us\n", name_of(t), no, ni, us);
+ cudaFree(dw); cudaFree(dx); cudaFree(dxq); cudaFree(dy);
+ }
+ }
// the verify window's shape: 2560 x 640 experts, 16 groups of m entries each
for (auto gd : {std::pair<int, int>{145, 146}, {145, 144}, {144, 144}, {144, 146}}) {
std::printf("BENCH grouped %-6s/%-6s 16 groups of m entries:", name_of(gd.first), name_of(gd.second));