Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the matrix-core attention's fragments load two halves a word (the scores' Q/K tiles and the values' P tile), and the online softmax runs a warp per two heads with a lane a key (16-lane shuffle max/sum) instead of a thread a head walking 16 keys. R9700 12 GB cell, 2.25 pack, 8K warm with DS4_FAST_ATTN=1: 767.7 / 765.8 ppts (10.67/10.70 s), the sparse attention stage 2516 -> 2311 ms, first prompt 465.6, with the same ids (5705e1164a, the agent decode's 81d895366c) and the same decode speed (16.49 tok/s). Quality gate on this binary (4995-position score, 2.25 plain, ref_base1): KL 0.10776 / ppl 3.389 / top1 87.89% against the scalar kernel's 0.10746 / 3.395 / 87.81% (delta 0.0003; the gate is 0.006). The softmax's sum order differs from the previous matrix-core build (a shuffle tree, not a key walk).

Niko122132c686fa67a7
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 307c9cbeb..218fff205 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1215,10 +1215,14 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*
const int k0 = wp * 64;
for (int kk = k0; kk < k0 + 64; kk += 16) {
h16v a, b;
+ const uint32_t* qa = (const uint32_t*) (qs + l16 * kFaLd + kk);
+ const uint32_t* kb = (const uint32_t*) (ks + l16 * kFaLd + kk);
#pragma unroll
- for (int i = 0; i < 16; ++i) {
- a[i] = __builtin_bit_cast(_Float16, qs[l16 * kFaLd + kk + i]);
- b[i] = __builtin_bit_cast(_Float16, ks[l16 * kFaLd + kk + i]);
+ for (int i = 0; i < 8; ++i) { // (two halves a load)
+ a[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (qa[i] & 0xffffu));
+ a[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (qa[i] >> 16));
+ b[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (kb[i] & 0xffffu));
+ b[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (kb[i] >> 16));
}
acc = set_wmma(a, b, acc);
}
@@ -1232,20 +1236,22 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*
sc[h][k] = t0 + k < total ? a * scale : -INFINITY;
}
__syncthreads();
- if (tid < kFaHeads) { // the online softmax, a head a thread
- float m = -INFINITY;
- for (int j = 0; j < nt; ++j) m = fmaxf(m, sc[tid][j]);
- const float mo = m_run[tid], mn = fmaxf(mo, m);
- float s = 0.0f;
- for (int j = 0; j < nt; ++j) {
- const float p = expf(sc[tid][j] - mn);
- ps[tid][j] = __builtin_bit_cast(uint16_t, (_Float16) p);
- s += p;
+ { // the online softmax: a warp per 2 heads, a lane a key (16 lanes a head)
+ const int h = 2 * wp + (lane >> 4), k = lane & 15;
+ float m = sc[h][k];
+#pragma unroll
+ for (int o = 8; o > 0; o >>= 1) m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, o));
+ const float mo = m_run[h], mn = fmaxf(mo, m);
+ const float p = expf(sc[h][k] - mn);
+ ps[h][k] = __builtin_bit_cast(uint16_t, (_Float16) p);
+ float s = p;
+#pragma unroll
+ for (int o = 8; o > 0; o >>= 1) s += __shfl_xor_sync(0xffffffffu, s, o);
+ if (k == 0) {
+ ts[h] = expf(mo - mn);
+ den[h] = den[h] * ts[h] + s;
+ m_run[h] = mn;
}
- for (int j = nt; j < kFaKeys; ++j) ps[tid][j] = 0; // (the stale rows: the values' product reads them)
- ts[tid] = expf(mo - mn);
- den[tid] = den[tid] * ts[tid] + s;
- m_run[tid] = mn;
}
__syncthreads();
const int dim0 = wp * 64;
@@ -1256,10 +1262,13 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float*
#pragma unroll
for (int f = 0; f < 4; ++f) { // the values: P (16 heads x 16 keys) x V (16 keys x 64 dims this warp)
h16v a, b;
+ const uint32_t* pp = (const uint32_t*) ps[l16];
#pragma unroll
- for (int i = 0; i < 16; ++i) {
- a[i] = __builtin_bit_cast(_Float16, ps[l16][i]);
- b[i] = __builtin_bit_cast(_Float16, ks[i * kFaLd + dim0 + f * 16 + l16]);
+ for (int i = 0; i < 8; ++i) {
+ a[2 * i] = __builtin_bit_cast(_Float16, (uint16_t) (pp[i] & 0xffffu));
+ a[2 * i + 1] = __builtin_bit_cast(_Float16, (uint16_t) (pp[i] >> 16));
+ b[2 * i] = __builtin_bit_cast(_Float16, ks[(2 * i) * kFaLd + dim0 + f * 16 + l16]);
+ b[2 * i + 1] = __builtin_bit_cast(_Float16, ks[(2 * i + 1) * kFaLd + dim0 + f * 16 + l16]);
}
o[f] = set_wmma(a, b, o[f]);
}