Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-022-int into f-022-gyrocuda: Pascal path kept for cc 6.0, 8-lane trellis rows for every other CUDA card

Niko122101f8c32559e6
diff --cc src/kernels/cuda/iq_kernels.cu
index 769b674a8,54fe7070f..f39e9a953
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@@ -1588,43 -1591,6 +1591,43 @@@ __device__ __forceinline__ void row_dot
}
}
+// 8-lane sub-warp row dot for K = nb * ipb == 160 (the APR trellis gate/up types at n_embd 2560): 4 rows per warp (32 per
+// 256-thread block). Bitwise row_dot_multi's tree: its lane L sums the calls k = L, L + 32, ... L + 128 in that order, then
+// warp_sum adds lane + 16, then + 8, then 4, 2, 1. Here lane t of 8 keeps the four sums of the lanes t, t + 8, t + 16 and
+// t + 24 (v = 0..3) and adds them (v0 + v2) + (v1 + v3), which is what the first two steps give; 3 steps follow. A trellis
+// block is 8 lanes, so lane t always takes lane t of block v + 4 j: its bit offsets are the same for all 20 calls.
+template<int TY, int NC, bool EXACT_N = false, bool STAGE_GRID = false>
+__device__ __forceinline__ void row_dot_160_sub8(const uint8_t* row, const block_q8_1* x, const int (&off)[NC], int n,
+ int t, float (&s)[NC], const uint32_t* __restrict__ s_grid = nullptr) {
+ using F = Fmt<TY>;
+ using S = Split<TY>;
+ static_assert(F::ipb == 8 && F::step == 4, "one trellis block is 8 lanes of 4 steps");
+ float p[4][NC];
+#pragma unroll
+ for (int v = 0; v < 4; ++v)
+#pragma unroll
+ for (int c = 0; c < NC; ++c) p[v][c] = 0.0f;
+#pragma unroll
+ for (int j = 0; j < 5; ++j) {
+#pragma unroll
+ for (int v = 0; v < 4; ++v) {
+ const int kbx = v + 4 * j, iqs = F::step * t;
- const typename S::W w = split_load_st<S, STAGE_GRID>(row, kbx, iqs, s_grid);
++ const typename S::W w = S::template load<STAGE_GRID>(row, kbx, iqs, s_grid);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (EXACT_N || c < n) p[v][c] += S::apply(w, x + off[c] + kbx * (F::qk / 32), iqs);
+ }
+ }
+#pragma unroll
+ for (int c = 0; c < NC; ++c) {
+ if (EXACT_N || c < n) {
+ s[c] = __fadd_rn(__fadd_rn(p[0][c], p[2][c]), __fadd_rn(p[1][c], p[3][c]));
+#pragma unroll
+ for (int o = 4; o > 0; o >>= 1) s[c] += __shfl_xor_sync(0xffffffffu, s[c], o);
+ }
+ }
+}
+
// 147-verify: row_dot_80_sub16 for two rows (a gate row and its up row) at once, both rows' weight loads of a step issued
// before either is applied; each row's sums are row_dot_80_sub16's, in its order (the same bits).
template<int TY, int NC, bool EXACT_N = false, bool STAGE_GRID = false>
@@@ -1870,47 -1836,7 +1873,47 @@@ __global__ void __launch_bounds__(256)
STRATA_SHARED_ALIGN16_U32(s_grid_buf, IqGridWords<TG, STAGE_GRID>::value);
const uint32_t* s_grid = stage_iq_grid<TG, STAGE_GRID>(s_grid_buf, threadIdx.x, 256);
const int nb = (int) (L.n_embd / Fmt<TG>::qk), xb = (int) (L.n_embd / 32);
- if constexpr (SUB16) {
+ if constexpr (SUB16 && (TG == 144 || TG == 145)) {
- if (nb * Fmt<TG>::ipb == 160) { // the trellis rows: 160 dots at n_embd 2560, 8 lanes a row
++ if (nb * Fmt<TG>::ipb == 160) { // the trellis rows (tq_sub8_on): 160 dots at n_embd 2560, 8 lanes a row
+ const int subwarp = threadIdx.x >> 3, t = threadIdx.x & 7;
+ const int row = blockIdx.x * 32 + subwarp; // 0 .. 2*n_ff
+ if (row >= 2 * L.n_ff) return;
+ const bool is_up = row >= L.n_ff;
+ const int r = is_up ? row - (int) L.n_ff : row;
+ const size_t w_off = (is_up ? L.up_off : 0) + (size_t) r * L.gu_row;
+ float* dst = is_up ? up : gate;
+ for (int g = blockIdx.y; g < ng; g += gridDim.y) {
+ const uint8_t* wr = (const uint8_t*) grp_ptr[g] + w_off;
+ const float sc = L.tail_off ? ((const float*) (grp_ptr[g] + L.tail_off))[is_up ? 1 : 0] : 1.0f;
+ const int e0 = grp_start[g], e1 = grp_start[g + 1];
+ for (int e = e0; e < e1; e += GRP_NC) {
+ const int n = min(GRP_NC, e1 - e);
+ if (n == 1) {
+ const int off1[1] = { ent_tok[e] * 80 };
+ float s1[1];
+ row_dot_160_sub8<TG, 1, true, STAGE_GRID>(wr, xq, off1, 1, t, s1, s_grid);
+ if (t == 0) dst[(size_t) e * L.n_ff + r] = s1[0] * sc;
+ } else if (n == 2) {
+ const int off2[2] = { ent_tok[e] * 80, ent_tok[e + 1] * 80 };
+ float s2[2];
+ row_dot_160_sub8<TG, 2, true, STAGE_GRID>(wr, xq, off2, 2, t, s2, s_grid);
+ if (t < 2) dst[(size_t) (e + t) * L.n_ff + r] = s2[t] * sc;
+ } else if (n == 3) {
+ const int off3[3] = { ent_tok[e] * 80, ent_tok[e + 1] * 80, ent_tok[e + 2] * 80 };
+ float s3[3];
+ row_dot_160_sub8<TG, 3, true, STAGE_GRID>(wr, xq, off3, 3, t, s3, s_grid);
+ if (t < 3) dst[(size_t) (e + t) * L.n_ff + r] = s3[t] * sc;
+ } else {
+ const int off4[4] = { ent_tok[e] * 80, ent_tok[e + 1] * 80, ent_tok[e + 2] * 80, ent_tok[e + 3] * 80 };
+ float s4[4];
+ row_dot_160_sub8<TG, 4, true, STAGE_GRID>(wr, xq, off4, 4, t, s4, s_grid);
+ if (t < 4) dst[(size_t) (e + t) * L.n_ff + r] = s4[t] * sc;
+ }
+ }
+ }
+ return;
+ }
+ } else if constexpr (SUB16) {
if (xb == 80 && nb * Fmt<TG>::ipb == 80) { // row_dot_80_sub16 covers exactly 80 dots (not the 128-value TQ blocks)
const int subwarp = threadIdx.x >> 4, t = threadIdx.x & 15;
const int row = blockIdx.x * 16 + subwarp; // 0 .. 2*n_ff
@@@ -3189,6 -3226,40 +3303,57 @@@ bool stage_tables_on()
return on;
}
+ // The trellis types (Gyro: TQ2_T / TQK6 / TQK7) read a 4 KB int8 codebook with 8 random lookups per 16 weights. The
+ // grouped expert kernels stage it in shared memory on compute capability 6.0 (Tesla P100: no dp4a, ~28 ALU ops per
+ // weight in the decode, the global lookups are replayed per distinct line) and leave it in global memory elsewhere
+ // (the newer cards' L1 absorbs it). STRATA_TQ_STAGE=1 / 0 forces it either way; bitwise the same result both ways.
+ bool tq_stage_on() {
+ static const int env = [] { const char* v = std::getenv("STRATA_TQ_STAGE"); return v != nullptr && v[0] != '\0' ? (v[0] != '0' ? 1 : 0) : -1; }();
+ if (env >= 0) return env == 1 && g_stage_grid;
+ #if defined(__HIPCC__)
+ return false;
+ #else
+ if (!g_stage_grid) return false;
+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+ if (!per_dev[dev]) {
+ int maj = 0, mnr = 0;
+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;
+ if (!ok) cudaGetLastError();
+ per_dev[dev] = ok && maj * 10 + mnr < 61 ? 2 : 1;
+ }
+ return per_dev[dev] == 2;
+ #endif
+ }
++// Everywhere but cc 6.0 (tq_stage_on) the trellis kernels stage the codebook too, and the 160-dot gate/up rows are 8-lane rows (4 per
++// warp, 32 per block, row_dot_160_sub8) while the down rows are the staged 8-lane rows above: on an RTX 3060 (Gyro-M, 24 groups of one
++// token) gate/up 221 -> 140 us and down 182 -> 86 us. Bitwise the same sums as before. STRATA_TQ_STAGE=0 or STRATA_TQ_SUB8=0 turn it off
++// (CUDA only: the AMD builds keep the old kernels), STRATA_IQ_STAGE_GRID=0 as for every staged table.
++bool tq_sub8_on() {
++#if defined(__HIPCC__)
++ return false;
++#else
++ static const bool on = [] {
++ const char* v = std::getenv("STRATA_TQ_STAGE");
++ if (v != nullptr && v[0] == '0') return false;
++ v = std::getenv("STRATA_TQ_SUB8");
++ return !(v != nullptr && v[0] == '0');
++ }();
++ return on && g_stage_grid && !g_no_sub16_gu && !tq_stage_on();
++#endif
++}
+ // row chunks per block of the staged trellis gate/up and down kernels (STRATA_TQ_ROWCH, default 4)
+ int tq_row_chunks() {
+ static const int n = [] {
+ const char* v = std::getenv("STRATA_TQ_ROWCH");
+ const int k = v != nullptr ? std::atoi(v) : 4;
+ return k < 1 ? 1 : k > 32 ? 32 : k;
+ }();
+ return n;
+ }
+
template<int TY, bool STAGE_GRID = kStageIqGrid<TY>>
void launch_mmvq_multi_nc(dim3 grid, dim3 block, cudaStream_t s, const uint8_t* W, size_t rb, const block_q8_1* X,
float* y, int n_in, int n_out, int ncols) {
@@@ -3918,7 -3986,7 +4080,15 @@@ void launch_gu(dim3 grid, cudaStream_t
if constexpr (!kSplit<TG>) native_gu_kernel<TG><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
else if (g_old_kernels || split_off<TG>()) native_gu_kernel<TG><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
else if constexpr (kStageIqGrid<TG>) {
- if (!g_stage_grid || ((TG == 18 || TG == 21) && !stage_tables_on())) {
++ if constexpr (TG == 144 || TG == 145) { // tq_sub8_on: the 8-lane trellis rows, 32 rows a block
++ if (tq_sub8_on() && (L.n_embd / Fmt<TG>::qk) * Fmt<TG>::ipb == 160) {
++ dim3 g8 = grid;
++ g8.x = (unsigned) ((2 * L.n_ff + 31) / 32);
++ native_gu_multi_kernel<TG, true, true><<<g8, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
++ return;
++ }
++ }
+ if (!g_stage_grid || ((TG == 18 || TG == 21) && !stage_tables_on()) || (kTqType<TG> && !tq_stage_on())) {
if (g_no_sub16_gu) native_gu_multi_kernel<TG, false, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
else native_gu_multi_kernel<TG, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);
return;
@@@ -3985,14 -4055,12 +4157,12 @@@ void launch_down(dim3 grid, cudaStream_
float* out) {
if constexpr (!kSplit<TD>) native_down_kernel<TD><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
else if (g_old_kernels || split_off<TD>()) native_down_kernel<TD><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
- else if constexpr (TD == 144 || TD == 146) { // the trellis codebook in shared memory (STRATA_IQ_STAGE_GRID=0: read in place)
- if (g_no_sub16_gu) {
- if (g_stage_grid) native_down_multi_kernel<TD, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
- else native_down_multi_kernel<TD, false, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
- } else {
- if (g_stage_grid) native_down_multi_kernel<TD, true, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
- else native_down_multi_kernel<TD, true, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
- else if constexpr (kTqType<TD>) { // the trellis down types: the codebook staged where tq_stage_on()
- if (tq_stage_on()) {
++ else if constexpr (kTqType<TD>) { // the trellis down types: the codebook staged where tq_stage_on() or tq_sub8_on()
++ if (tq_stage_on() || tq_sub8_on()) {
+ if (g_no_sub16_gu) native_down_multi_kernel<TD, false, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
+ else native_down_multi_kernel<TD, true, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
}
+ else native_down_multi_kernel<TD, true, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
}
else if (g_no_sub16_gu) native_down_multi_kernel<TD, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
else native_down_multi_kernel<TD, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
@@@ -5060,8 -5128,10 +5230,10 @@@ void native_expert_grouped(const Native
check("native_expert_grouped/swiglu");
}
if (g_exp_phase == 1) return;
- const int d_rows = (!g_old_kernels && !g_no_sub16_gu && L.n_ff == 640) ? (L.d_type == 20 || L.d_type == 40 || L.d_type == 144 || L.d_type == 146 ? 32 : (L.d_type == 42 ? 16 : 8)) : 8;
- const dim3 gd((unsigned) ((L.n_embd + d_rows - 1) / d_rows), (unsigned) gy);
- const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && !g_old_kernels && !g_no_sub16_gu && L.n_ff == 640 && tq_stage_on(); // the trellis down rows as 8-lane rows (cc 6.0)
++ const bool tq_sub8 = (L.d_type == 144 || L.d_type == 146) && !g_old_kernels && !g_no_sub16_gu && L.n_ff == 640 && (tq_stage_on() || tq_sub8_on()); // the trellis down rows as 8-lane rows
+ const int d_rows = (!g_old_kernels && !g_no_sub16_gu && L.n_ff == 640) ? (L.d_type == 20 || L.d_type == 40 || tq_sub8 ? 32 : (L.d_type == 42 ? 16 : 8)) : 8;
+ const int tq_ch = tq_sub8 ? tq_row_chunks() : 1; // the staged trellis down kernel takes this many 32-row chunks a block
+ const dim3 gd((unsigned) (((L.n_embd + d_rows - 1) / d_rows + tq_ch - 1) / tq_ch), (unsigned) gy);
#if STRATA_EXP_LAYOUTS
const int em0d = sm75_down_em(em0, cap_groups);
if ((em0d == 7 || em0d == 8) && (L.d_type == 20 || L.d_type == 42)) {