Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the matrix-core attention's fragments load two halves a word (the scores' Q/K tiles and the values' P tile), and the online softmax runs a warp per two heads with a lane a key (16-lane shuffle max/sum) instead of a thread a head walking 16 keys. R9700 12 GB cell, 2.25 pack, 8K warm with DS4_FAST_ATTN=1: 767.7 / 765.8 ppts (10.67/10.70 s), the sparse attention stage 2516 -> 2311 ms, first prompt 465.6, with the same ids (5705e1164a, the agent decode's 81d895366c) and the same decode speed (16.49 tok/s). Quality gate on this binary (4995-position score, 2.25 plain, ref_base1): KL 0.10776 / ppl 3.389 / top1 87.89% against the scalar kernel's 0.10746 / 3.395 / 87.81% (delta 0.0003; the gate is 0.006). The softmax's sum order differs from the previous matrix-core build (a shuffle tree, not a key walk).
Niko122132c686fa67a7
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 307c9cbeb..218fff205 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1215,10 +1215,14 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*const int k0 = wp * 64;for (int kk = k0; kk < k0 + 64; kk += 16) {h16v a, b;+ const uint32_t* qa = (const uint32_t*) (qs + l16 * kFaLd + kk);+ const uint32_t* kb = (const uint32_t*) (ks + l16 * kFaLd + kk);#pragma unroll- for (int i = 0; i < 16; ++i) {- a[i] = __builtin_bit_cast(_Float16, qs[l16 * kFaLd + kk + i]);- b[i] = __builtin_bit_cast(_Float16, ks[l16 * kFaLd + kk + i]);+ for (int i = 0; i < 8; ++i) { // (two halves a load)+ a[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (qa[i] & 0xffffu));+ a[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (qa[i] >> 16));+ b[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (kb[i] & 0xffffu));+ b[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (kb[i] >> 16));}acc = set_wmma(a, b, acc);}@@ -1232,20 +1236,22 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*sc[h][k] = t0 + k < total ? a * scale : -INFINITY;}__syncthreads();- if (tid < kFaHeads) { // the online softmax, a head a thread- float m = -INFINITY;- for (int j = 0; j < nt; ++j) m = fmaxf(m, sc[tid][j]);- const float mo = m_run[tid], mn = fmaxf(mo, m);- float s = 0.0f;- for (int j = 0; j < nt; ++j) {- const float p = expf(sc[tid][j] - mn);- ps[tid][j] = __builtin_bit_cast(uint16_t, (_Float16) p);- s += p;+ { // the online softmax: a warp per 2 heads, a lane a key (16 lanes a head)+ const int h = 2 * wp + (lane >> 4), k = lane & 15;+ float m = sc[h][k];+#pragma unroll+ for (int o = 8; o > 0; o >>= 1) m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, o));+ const float mo = m_run[h], mn = fmaxf(mo, m);+ const float p = expf(sc[h][k] - mn);+ ps[h][k] = __builtin_bit_cast(uint16_t, (_Float16) p);+ float s = p;+#pragma unroll+ for (int o = 8; o > 0; o >>= 1) s += __shfl_xor_sync(0xffffffffu, s, o);+ if (k == 0) {+ ts[h] = expf(mo - mn);+ den[h] = den[h] * ts[h] + s;+ m_run[h] = mn;}- for (int j = nt; j < kFaKeys; ++j) ps[tid][j] = 0; // (the stale rows: the values' product reads them)- ts[tid] = expf(mo - mn);- den[tid] = den[tid] * ts[tid] + s;- m_run[tid] = mn;}__syncthreads();const int dim0 = wp * 64;@@ -1256,10 +1262,13 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*#pragma unrollfor (int f = 0; f < 4; ++f) { // the values: P (16 heads x 16 keys) x V (16 keys x 64 dims this warp)h16v a, b;+ const uint32_t* pp = (const uint32_t*) ps[l16];#pragma unroll- for (int i = 0; i < 16; ++i) {- a[i] = __builtin_bit_cast(_Float16, ps[l16][i]);- b[i] = __builtin_bit_cast(_Float16, ks[i * kFaLd + dim0 + f * 16 + l16]);+ for (int i = 0; i < 8; ++i) {+ a[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (pp[i] & 0xffffu));+ a[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (pp[i] >> 16));+ b[2 * i] = __builtin_bit_cast(_Float16, ks[(2 * i) * kFaLd + dim0 + f * 16 + l16]);+ b[2 * i + 1] = __builtin_bit_cast(_Float16, ks[(2 * i + 1) * kFaLd + dim0 + f * 16 + l16]);}o[f] = set_wmma(a, b, o[f]);}