Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: trellis gate/up fused in one pass sharing the activation fragments (token path 454 -> ~355 us a layer, outputs unchanged: ds4_tq_test ALL OK); optional 32-bit stream loads (no gain)
Niko12211c2388876c69
diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cuindex e96298716..04b994d40 100644--- a/src/dsv41/ds4_tq.cu+++ b/src/dsv41/ds4_tq.cu@@ -125,96 +125,150 @@ struct ActF16 { // F16 rows: entry m = row rows[m] (ld halves a row)}};-// ---------------------------------------------------------------- one 16-row tile over up to 16 entries-// acc[i] (i < 8) = D[entry 8 kg + i][row r] of the tile: sum over the row's k values of x_entry * w, k % 128 == 0.-// wrow: the tile's row r (this lane's) in the blob; nblk blocks.-template <int K, bool COMPUTED, class Act>-__device__ __forceinline__ void tile_dot(const uint8_t* __restrict__ wrow, int nblk, const uint32_t* lut, const Act& act, int ne,- float* acc_out) {+// ---------------------------------------------------------------- the stream window by 32-bit loads+// The same 128-bit window as stream_window, from aligned 32-bit loads (a record is 2 + 4 K bytes: its scale, then the stream, at+// any 2-byte alignment): the array starts at the 4-byte floor of the record, the stream at bit 16 or 32 of it. Loads past the+// end of the record are skipped (the window bits above what a lane needs are junk anyway).+template <int K>+__device__ __forceinline__ void stream_window_u32(const uint8_t* rec, int kg, uint64_t& lo, uint64_t& hi) {+ constexpr int bsz = 2 + 4 * K;+ const uintptr_t ad = (uintptr_t) rec;+ const uint32_t* pa = (const uint32_t*) (ad & ~(uintptr_t) 3);+ const int off = 16 + 8 * (int) (ad & 3); // the stream's first bit in the array: 16 or 32+ const int start = off + (kg ? 0 : 16 * K);+ const int w0 = start >> 5, s = start & 31;+ const int nwords = (int) (((ad & 3) + bsz + 3) >> 2); // words that hold the record+ uint32_t v[5];+#pragma unroll+ for (int i = 0; i < 5; ++i) v[i] = (w0 + i < nwords) ? pa[w0 + i] : 0u;+ uint32_t r[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) r[i] = (uint32_t) ((((uint64_t) v[i + 1] << 32) | v[i]) >> s);+ lo = ((uint64_t) r[1] << 32) | r[0];+ hi = ((uint64_t) r[3] << 32) | r[2];+ if (!kg) { // the upper half [16 K, 32 K) of the stream (16 K >= 64 bits); the bits past its end wrap to the stream's start+ hi &= (1ull << (16 * K - 64)) - 1ull;+ const uint32_t first = off == 16 ? (pa[0] >> 16) : pa[1];+ const uint64_t wrap = (uint64_t) first & ((1ull << (15 - K)) - 1ull);+ const int p = 16 * K - 64;+ hi |= wrap << p;+ }+}++// ---------------------------------------------------------------- NM 16-row tiles (matrices sharing the activations) over up to 16 entries+// acc_out[mi][i] (i < 8) = D[entry 8 kg + i][row r] of matrix mi's tile: sum over the row's k values of x_entry * w, k % 128 == 0.+// wrows[mi]: the tile's row r (this lane's) of matrix mi in the blob; nblk blocks. The activation fragments are made once for all NM.+template <int K, bool COMPUTED, int NM, bool WORDS, class Act>+__device__ __forceinline__ void tile_dot(const uint8_t* const* wrows, int nblk, const uint32_t* lut, const Act& act, int ne,+ float (*acc_out)[8]) {constexpr int bsz = 2 + 4 * K;const int lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;- float tot[8];+ float tot[NM][8];#pragma unroll- for (int i = 0; i < 8; ++i) tot[i] = 0.0f;+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[mi][i] = 0.0f;const bool have = r < ne; // this lane's A rows (the entry m = r)for (int b = 0; b < nblk; ++b) {- const uint8_t* rec = wrow + (size_t) b * bsz;- const float d = half_bits_f(*(const uint16_t*) rec);- uint64_t lo, hi;- stream_window<K>(rec, kg, lo, hi);+ float d[NM];+ uint64_t lo[NM], hi[NM];+#pragma unroll+ for (int mi = 0; mi < NM; ++mi) {+ const uint8_t* rec = wrows[mi] + (size_t) b * bsz;+ d[mi] = half_bits_f(*(const uint16_t*) rec);+ if (WORDS) stream_window_u32<K>(rec, kg, lo[mi], hi[mi]);+ else stream_window<K>(rec, kg, lo[mi], hi[mi]);+ }#if defined(DS4_TQ_WMMA)- float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};+ float8_t acc[NM];+#pragma unroll+ for (int mi = 0; mi < NM; ++mi) acc[mi] = float8_t{0, 0, 0, 0, 0, 0, 0, 0};#pragma unrollfor (int m = 0; m < 8; ++m) {- half8_t wf, ah, al;-#pragma unroll- for (int s2 = 0; s2 < 2; ++s2) {- const int u = 2 * m + s2;- uint32_t w0, w1;- step_words<COMPUTED>(lut, state_at(lo, hi, (15 - u) * K), w0, w1);- wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));- wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));- wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));- wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));- }- float v[8];- if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);- else {+ half8_t ah, al;+ {+ float v[8];+ if (have) act.load8(r, b * 128 + 64 * kg + 8 * m, v);+ else {+#pragma unroll+ for (int j = 0; j < 8; ++j) v[j] = 0.0f;+ }#pragma unroll- for (int j = 0; j < 8; ++j) v[j] = 0.0f;+ for (int j = 0; j < 8; ++j) {+ const _Float16 h = (_Float16) v[j];+ ah[j] = h;+ al[j] = (_Float16) (v[j] - (float) h);+ }}#pragma unroll- for (int j = 0; j < 8; ++j) {- const _Float16 h = (_Float16) v[j];- ah[j] = h;- al[j] = (_Float16) (v[j] - (float) h);+ for (int mi = 0; mi < NM; ++mi) {+ half8_t wf;+#pragma unroll+ for (int s2 = 0; s2 < 2; ++s2) {+ const int u = 2 * m + s2;+ uint32_t w0, w1;+ step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - u) * K), w0, w1);+ wf[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));+ wf[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));+ wf[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));+ wf[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));+ }+ acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc[mi]); // D[m = entry][n = weight row]+ acc[mi] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc[mi]);}- acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(ah, wf, acc); // D[m = entry][n = weight row]- acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(al, wf, acc);}#pragma unroll- for (int i = 0; i < 8; ++i) tot[i] += d * acc[i];+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[mi][i] += d[mi] * acc[mi][i];#else// plain FMAs: this lane's row r, its 64 weights of the block against every entry (lane r + 16 kg' sums the halves)- float part[kMaxEnt];+ float part[NM][kMaxEnt];+#pragma unroll+ for (int mi = 0; mi < NM; ++mi)#pragma unroll- for (int e = 0; e < kMaxEnt; ++e) part[e] = 0.0f;+ for (int e = 0; e < kMaxEnt; ++e) part[mi][e] = 0.0f;#pragma unrollfor (int m = 0; m < 8; ++m) {- float w[8];-#pragma unroll- for (int s2 = 0; s2 < 2; ++s2) {- uint32_t w0, w1;- step_words<COMPUTED>(lut, state_at(lo, hi, (15 - (2 * m + s2)) * K), w0, w1);- w[4 * s2 + 0] = half_bits_f(w0 & 0xffffu);- w[4 * s2 + 1] = half_bits_f(w0 >> 16);- w[4 * s2 + 2] = half_bits_f(w1 & 0xffffu);- w[4 * s2 + 3] = half_bits_f(w1 >> 16);- }+ float w[NM][8];+#pragma unroll+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int s2 = 0; s2 < 2; ++s2) {+ uint32_t w0, w1;+ step_words<COMPUTED>(lut, state_at(lo[mi], hi[mi], (15 - (2 * m + s2)) * K), w0, w1);+ w[mi][4 * s2 + 0] = half_bits_f(w0 & 0xffffu);+ w[mi][4 * s2 + 1] = half_bits_f(w0 >> 16);+ w[mi][4 * s2 + 2] = half_bits_f(w1 & 0xffffu);+ w[mi][4 * s2 + 3] = half_bits_f(w1 >> 16);+ }#pragma unrollfor (int e = 0; e < kMaxEnt; ++e)if (e < ne) {float v[8];act.load8(e, b * 128 + 64 * kg + 8 * m, v);#pragma unroll- for (int j = 0; j < 8; ++j) part[e] = fmaf(w[j], v[j], part[e]);+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int j = 0; j < 8; ++j) part[mi][e] = fmaf(w[mi][j], v[j], part[mi][e]);}}// entries 0..7 into tot[i] of the kg = 0 lane, 8..15 of the kg = 1 lane, so the output indexing matches the WMMA's#pragma unroll- for (int i = 0; i < 8; ++i) {- const float p0 = part[i], p1 = part[8 + i];- // lane kg holds its half's partial of both; the full sum needs the partner's half (same row, other kg)- const float q0 = __shfl_xor(p0, 16), q1 = __shfl_xor(p1, 16);- tot[i] += d * (kg == 0 ? p0 + q0 : p1 + q1);- }+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const float p0 = part[mi][i], p1 = part[mi][8 + i];+ const float q0 = __shfl_xor(p0, 16), q1 = __shfl_xor(p1, 16);+ tot[mi][i] += d[mi] * (kg == 0 ? p0 + q0 : p1 + q1);+ }#endif}#pragma unroll- for (int i = 0; i < 8; ++i) acc_out[i] = tot[i];+ for (int mi = 0; mi < NM; ++mi)+#pragma unroll+ for (int i = 0; i < 8; ++i) acc_out[mi][i] = tot[mi][i];}-// (the plain form: the block scale d is the row's, which both halves of the row read alike, so d * (sum of the halves) is-// right; the halves' partial sums swap through the shuffle, kg 0 keeping entries 0..7 and kg 1 entries 8..15)__device__ __forceinline__ float swiglu(float gv, float uv, float limit) { // ds4_iq.cu'sif (limit > 0.0f) {@@ -261,7 +315,7 @@ __device__ __forceinline__ bool group_of(const Args& a, int grp, const uint8_t*&}// grid (n_ff / 64, groups): a wave = a 16-row tile of the gate and of the up of the group's expert, 16 entries a pass-template <int K, bool COMPUTED>+template <int K, bool COMPUTED, bool WORDS>__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {__shared__ uint32_t lut[COMPUTED ? 1 : 2048];__shared__ int ent[kMaxEnt], xr[kMaxEnt];@@ -286,19 +340,19 @@ __global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {__syncthreads();if (tile * kTileRows >= a.n_ff) continue;const ActQ8 act{a.xq, a.xd, xr, a.n_embd};- float ag[8], au[8];- tile_dot<K, COMPUTED>(g, a.n_embd / 128, lut, act, ne, ag);- tile_dot<K, COMPUTED>(u, a.n_embd / 128, lut, act, ne, au);+ const uint8_t* wr[2] = {g, u};+ float gu[2][8];+ tile_dot<K, COMPUTED, 2, WORDS>(wr, a.n_embd / 128, lut, act, ne, gu);#pragma unrollfor (int i = 0; i < 8; ++i) {const int m = 8 * kg + i;- if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(ag[i], au[i], a.limit);+ if (m < ne) a.h[(size_t) ent[m] * a.n_ff + row] = swiglu(gu[0][i], gu[1][i], a.limit);}}}// grid (n_embd / 64, groups): the down's tile of a wave against the entries' int8 h-template <int K, bool COMPUTED>+template <int K, bool COMPUTED, bool WORDS>__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {__shared__ uint32_t lut[COMPUTED ? 1 : 2048];__shared__ int ent[kMaxEnt];@@ -318,12 +372,13 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {__syncthreads();if (tile * kTileRows >= a.n_embd) continue;const ActQ8 act{a.hq, a.hd, ent, a.n_ff};- float ad[8];- tile_dot<K, COMPUTED>(d, a.n_ff / 128, lut, act, ne, ad);+ const uint8_t* wr[1] = {d};+ float dd[1][8];+ tile_dot<K, COMPUTED, 1, WORDS>(wr, a.n_ff / 128, lut, act, ne, dd);#pragma unrollfor (int i = 0; i < 8; ++i) {const int m = 8 * kg + i;- if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = ad[i];+ if (m < ne) a.part[(size_t) ent[m] * a.n_embd + row] = dd[0][i];}}}@@ -332,7 +387,7 @@ __global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {// grid (n_ff / 64, chunks of 16 rows, n_exp): expert e = blockIdx.z owns the sorted rows [bounds[e], bounds[e + 1]); local// row = sorted row - row0. gate / up: F32 x[idx[local]] (ldx) -> F16 swiglu h[local * ldo + c]; down: F16 xh[local] (ldx) ->// F32 y[local * ldo + c].-template <int K, bool COMPUTED, bool GATE_UP>+template <int K, bool COMPUTED, bool GATE_UP, bool WORDS>__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {__shared__ uint32_t lut[COMPUTED ? 1 : 2048];__shared__ int rowsel[kMaxEnt];@@ -351,14 +406,15 @@ __global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {if ((int) threadIdx.x < ne) rowsel[threadIdx.x] = GATE_UP ? a.idx[r0 + threadIdx.x] : r0 + (int) threadIdx.x;__syncthreads();if (tile * kTileRows >= N) return;- float acc[8], accu[8];+ float res[2][8];if (GATE_UP) {const ActF32 act{a.xf, rowsel, (size_t) a.ldx};- tile_dot<K, COMPUTED>(blob + a.tr[0] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);- tile_dot<K, COMPUTED>(blob + a.tr[1] + (size_t) row * rb, Kin / 128, lut, act, ne, accu);+ const uint8_t* wr[2] = {blob + a.tr[0] + (size_t) row * rb, blob + a.tr[1] + (size_t) row * rb};+ tile_dot<K, COMPUTED, 2, WORDS>(wr, Kin / 128, lut, act, ne, res);} else {const ActF16 act{a.xh, rowsel, (size_t) a.ldx};- tile_dot<K, COMPUTED>(blob + a.tr[2] + (size_t) row * rb, Kin / 128, lut, act, ne, acc);+ const uint8_t* wr[1] = {blob + a.tr[2] + (size_t) row * rb};+ tile_dot<K, COMPUTED, 1, WORDS>(wr, Kin / 128, lut, act, ne, res);}const int kgl = threadIdx.x >> 4 & 1;#pragma unroll@@ -366,14 +422,14 @@ __global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {const int m = 8 * kgl + i;if (m >= ne) continue;const size_t lr = (size_t) (r0 + m);- if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(acc[i], accu[i], a.limit)));- else a.y[lr * a.ldo + row] = acc[i];+ if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(res[0][i], res[1][i], a.limit)));+ else a.y[lr * a.ldo + row] = res[0][i];}}// ---------------------------------------------------------------- GEMV jobs// grid (n / 64): y[c * ycs + r] = alpha (row r . x_c) + bias[r] for ncol (<= 16) int8 activation columns-template <int K, bool COMPUTED>+template <int K, bool COMPUTED, bool WORDS>__global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {__shared__ uint32_t lut[COMPUTED ? 1 : 2048];__shared__ int cols[kMaxEnt];@@ -385,12 +441,13 @@ __global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {if (tile * kTileRows >= j.n) return;const int row = tile * kTileRows + r;const ActQ8 act{j.xq, j.xd, cols, j.k};- float acc[8];- tile_dot<K, COMPUTED>(j.w + (size_t) row * j.ld, j.k / 128, lut, act, j.ncol, acc);+ const uint8_t* wr[1] = {j.w + (size_t) row * j.ld};+ float accm[1][8];+ tile_dot<K, COMPUTED, 1, WORDS>(wr, j.k / 128, lut, act, j.ncol, accm);#pragma unrollfor (int i = 0; i < 8; ++i) {const int m = 8 * kg + i;- if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * acc[i] + (j.bias != nullptr ? j.bias[row] : 0.0f);+ if (m < j.ncol && row < j.n) j.y[(size_t) m * j.ycs + row] = j.alpha * accm[0][i] + (j.bias != nullptr ? j.bias[row] : 0.0f);}}@@ -406,6 +463,15 @@ __global__ void __launch_bounds__(kThreads) mv_kernel(const MvJob j) {default: return false; \}+// 32-bit stream loads (STRATA_DS4_TQ_WORDS=1) or the 16-bit ones (default)+bool use_words() {+ static const bool on = [] {+ const char* e = std::getenv("STRATA_DS4_TQ_WORDS");+ return e != nullptr && std::atoi(e) != 0; // (no gain measured on the R9700: 13.8 vs 14.2 ms a token)+ }();+ return on;+}+bool bad(const char* what) {std::fprintf(stderr, "ds4 tq: %s\n", what);return false;@@ -413,38 +479,42 @@ bool bad(const char* what) {} // namespacebool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+ const bool w = use_words();(void) ne_max;if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);-#define GU_(K, C) gate_up_kernel<K, C><<<grid, kThreads, 0, s>>>(a)+#define GU_(K, C) { if (w) gate_up_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else gate_up_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }DS4_TQ_SWITCH(a.tg, GU_)#undef GU_return true;}bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+ const bool w = use_words();(void) ne_max;if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);-#define DN_(K, C) down_kernel<K, C><<<grid, kThreads, 0, s>>>(a)+#define DN_(K, C) { if (w) down_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else down_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }DS4_TQ_SWITCH(a.td, DN_)#undef DN_return true;}bool set_gate_up(const SetArgs& a, cudaStream_t s) {+ const bool w = use_words();if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);-#define SG_(K, C) set_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a)+#define SG_(K, C) { if (w) set_kernel<K, C, true, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, true, false><<<grid, kThreads, 0, s>>>(a); }DS4_TQ_SWITCH(a.tg, SG_)#undef SG_return true;}bool set_down(const SetArgs& a, cudaStream_t s) {+ const bool w = use_words();if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);-#define SD_(K, C) set_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a)+#define SD_(K, C) { if (w) set_kernel<K, C, false, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, false, false><<<grid, kThreads, 0, s>>>(a); }DS4_TQ_SWITCH(a.td, SD_)#undef SD_return true;@@ -453,9 +523,10 @@ bool set_down(const SetArgs& a, cudaStream_t s) {bool mv(const MvJob* jobs, int n, cudaStream_t s) {for (int i = 0; i < n; ++i) {const MvJob& j = jobs[i];+ const bool w = use_words();if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));-#define MV_(K, C) mv_kernel<K, C><<<grid, kThreads, 0, s>>>(j)+#define MV_(K, C) { if (w) mv_kernel<K, C, true><<<grid, kThreads, 0, s>>>(j); else mv_kernel<K, C, false><<<grid, kThreads, 0, s>>>(j); }DS4_TQ_SWITCH(j.type, MV_)#undef MV_}