Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: DS4_FAST_ATTN=1 runs the chunked prompt's packed-KV attention on the gfx11/gfx12 matrix cores (new sparse_attn_wmma_kernel, opt-in; the decode and verify paths stay on the scalar kernels). The matrix's rows are 16 heads of ONE query, so a block stages a 16-key tile once for all of them: Q (16 heads x 512) is the A operand of the scores, the key tile is B, and the same tile is B of the values' product; the scores' 64-dim slices are computed by the 8 warps and summed in shared, the values by a warp a 64-dim slice, the online softmax a head a thread. R9700 12 GB cell, 2.25 pack, 8K warm: 752.2 / 748.3 ppts (10.89/10.95 s) against 694.2 / 693.0 / 691.2 for the scalar kernel; the sparse attention stage 3750 -> 2516 ms; first prompt 484.8. Quality gate (4995-position score, 2.25 plain, ref_base1): KL 0.10750 / ppl 3.394 / top1 87.83% against the scalar kernel's 0.10746 / 3.395 / 87.81% (delta 0.00004; the gate is 0.006). The ids are the same (5705e1164a) and the 128-token agent decode's ids too (81d895366c); its decode speed unchanged (16.49 vs 16.87 tok/s). The QK/PV sums are the matrix cores' order, hence opt-in (the owner approved the trade for the prefill).

Niko1221f05ccd3b5496
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index f803016b8..307c9cbeb 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -6,6 +6,7 @@
#include "ds4_numerics.cuh"
#include "ds4_q6k.cuh"
#include "ds4_attn_split.cuh"
+#include "ds4_wmma.cuh"
#include <cuda_fp16.h>
@@ -1128,6 +1129,152 @@ __global__ void __launch_bounds__(kThreads) sparse_attn_packed_kernel(const floa
}
}
+// DS4_FAST_ATTN=1: the chunked prompt's packed-KV attention on the matrix cores (gfx11/gfx12 WMMA; the R9700's gfx12).
+// The matrix's rows are 16 heads of ONE query, so a block stages a 16-key tile once for all of them: Q (16 heads x 512
+// dims) is the A operand of the scores' product, the key tile is B, and the same tile is B of the values' product (the
+// heads' softmaxes and their 512-dim outputs are its rows). The QK and PV sums are the matrix cores' order, not
+// sparse_attn_packed_kernel's scalar lanes' - hence opt-in (DS4_FAST_ATTN).
+constexpr int kFaHeads = 16; // heads a tile (the matrix's rows)
+constexpr int kFaKeys = 16; // keys a tile
+constexpr int kFaD = 512; // the head dim
+constexpr int kFaLd = 520; // halves a tile row (512 + 8: the pad keeps the 16 rows off one bank)
+__global__ void __launch_bounds__(kThreads) sparse_attn_wmma_kernel(const float* __restrict__ q,
+ const uint16_t* __restrict__ win,
+ const uint8_t* __restrict__ ckv,
+ const int* __restrict__ sel, int sel_stride,
+ AttnRows ar, const float* __restrict__ sink,
+ int heads, float scale, float* __restrict__ out) {
+ __shared__ __align__(16) uint16_t qs[kFaHeads * kFaLd];
+ __shared__ __align__(16) uint16_t ks[kFaKeys * kFaLd];
+ __shared__ float sc[kFaHeads][kFaKeys];
+ __shared__ float part[8][kFaHeads][kFaKeys]; // (the 8 warps' 64-dim slices of the scores)
+ __shared__ uint16_t ps[kFaHeads][kFaKeys];
+ __shared__ const uint8_t* rows[kFaKeys];
+ __shared__ float m_run[kFaHeads], den[kFaHeads], ts[kFaHeads];
+ const int tid = (int) threadIdx.x, wp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+ const int t = blockIdx.y, hb = blockIdx.x * kFaHeads;
+ int n_win, n_sel, w0;
+ if (ar.gen) {
+ const long long ta = ar.t0 + t, pa = ar.p0 + ta;
+ const long long first = max((long long) ar.win - ar.p0, ta + 1);
+ w0 = (int) first;
+ n_win = (int) (ar.win + ta - first + 1);
+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;
+ } else {
+ n_win = ar.nwin[t], n_sel = ar.nsel[t], w0 = ar.win0[t];
+ }
+ const uint16_t* wbase = win + (size_t) w0 * kFaD;
+ const int* sl = sel + (size_t) t * sel_stride;
+ const size_t crow = kv_row_bytes(kKvFp4, kFaD);
+ // the block's 16 heads' queries as FP16 (the matrix's A operand), and their running softmax state
+ for (int i = tid; i < kFaHeads * kFaD; i += kThreads) {
+ const int h = i / kFaD, c = i % kFaD;
+ qs[h * kFaLd + c] = __builtin_bit_cast(uint16_t, (_Float16) q[((size_t) t * heads + hb + h) * kFaD + c]);
+ }
+ if (tid < kFaHeads) {
+ m_run[tid] = -1e30f;
+ den[tid] = 0.0f;
+ }
+ f8v o[4];
+#pragma unroll
+ for (int f = 0; f < 4; ++f)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) o[f][i] = 0.0f;
+ __syncthreads();
+ auto stage = [&](int t0, int n) { // the tile's n rows into ks (BF16 rows -> FP16)
+ const int sr = tid >> 4, part = tid & 15;
+ if (sr >= n) return;
+ const uint8_t* row = rows[sr];
+ uint16_t tmp[32];
+ if (t0 + sr < n_win) {
+ const uint4* src = (const uint4*) (row + part * 64);
+ const uint4 a = src[0], b = src[1], c = src[2], d = src[3];
+ ((uint4*) tmp)[0] = a;
+ ((uint4*) tmp)[1] = b;
+ ((uint4*) tmp)[2] = c;
+ ((uint4*) tmp)[3] = d;
+ } else {
+ unpack32(row, part, (uint4*) tmp);
+ }
+ uint16_t* dst = ks + sr * kFaLd + part * 32;
+#pragma unroll
+ for (int i = 0; i < 32; ++i) dst[i] = __builtin_bit_cast(uint16_t, (_Float16) bf16f(tmp[i]));
+ };
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += kFaKeys) {
+ const int nt = min(kFaKeys, total - t0);
+ if (tid < nt) {
+ const int j = t0 + tid;
+ rows[tid] = j < n_win ? (const uint8_t*) (wbase + (size_t) j * kFaD) : ckv + (size_t) sl[j - n_win] * crow;
+ }
+ __syncthreads();
+ stage(t0, nt);
+ __syncthreads();
+ { // the scores in 8 slices of 64 dims, a warp a slice, summed in shared
+ f8v acc = {0, 0, 0, 0, 0, 0, 0, 0};
+ const int k0 = wp * 64;
+ for (int kk = k0; kk < k0 + 64; kk += 16) {
+ h16v a, b;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) {
+ a[i] = __builtin_bit_cast(_Float16, qs[l16 * kFaLd + kk + i]);
+ b[i] = __builtin_bit_cast(_Float16, ks[l16 * kFaLd + kk + i]);
+ }
+ acc = set_wmma(a, b, acc);
+ }
+#pragma unroll
+ for (int i = 0; i < 8; ++i) part[wp][set_wmma_row(i, hi)][l16] = acc[i];
+ __syncthreads();
+ const int h = tid >> 4, k = tid & 15;
+ float a = 0.0f;
+#pragma unroll
+ for (int w = 0; w < 8; ++w) a += part[w][h][k];
+ sc[h][k] = t0 + k < total ? a * scale : -INFINITY;
+ }
+ __syncthreads();
+ if (tid < kFaHeads) { // the online softmax, a head a thread
+ float m = -INFINITY;
+ for (int j = 0; j < nt; ++j) m = fmaxf(m, sc[tid][j]);
+ const float mo = m_run[tid], mn = fmaxf(mo, m);
+ float s = 0.0f;
+ for (int j = 0; j < nt; ++j) {
+ const float p = expf(sc[tid][j] - mn);
+ ps[tid][j] = __builtin_bit_cast(uint16_t, (_Float16) p);
+ s += p;
+ }
+ for (int j = nt; j < kFaKeys; ++j) ps[tid][j] = 0; // (the stale rows: the values' product reads them)
+ ts[tid] = expf(mo - mn);
+ den[tid] = den[tid] * ts[tid] + s;
+ m_run[tid] = mn;
+ }
+ __syncthreads();
+ const int dim0 = wp * 64;
+#pragma unroll
+ for (int f = 0; f < 4; ++f)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) o[f][i] *= ts[set_wmma_row(i, hi)];
+#pragma unroll
+ for (int f = 0; f < 4; ++f) { // the values: P (16 heads x 16 keys) x V (16 keys x 64 dims this warp)
+ h16v a, b;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) {
+ a[i] = __builtin_bit_cast(_Float16, ps[l16][i]);
+ b[i] = __builtin_bit_cast(_Float16, ks[i * kFaLd + dim0 + f * 16 + l16]);
+ }
+ o[f] = set_wmma(a, b, o[f]);
+ }
+ __syncthreads();
+ }
+#pragma unroll
+ for (int f = 0; f < 4; ++f)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int h = set_wmma_row(i, hi), dm = wp * 64 + f * 16 + l16;
+ const float dv = den[h] + expf(sink[hb + h] - m_run[h]);
+ out[((size_t) t * heads + hb + h) * kFaD + dm] = o[f][i] / dv;
+ }
+}
+
// the ring's rows are kKvFp8: E4M3 codes, a UE8M0 scale per 32 (decoded: exactly the BF16 values they were made from)
__global__ void ring_to_ext_kernel(const uint8_t* __restrict__ ring, uint16_t* __restrict__ ext, long long p0, int n,
int d) {
@@ -1839,6 +1986,16 @@ bool split_enabled() {
}();
return on;
}
+// DS4_FAST_ATTN=1: the chunked prompt's packed-KV attention on the matrix cores (sparse_attn_wmma_kernel). Opt-in: the
+// QK/PV sums are the matrix cores' order, so the outputs differ from the scalar kernel's in the last places (the owner
+// approved the trade for the prefill; the decode path stays on the scalar kernels).
+bool fast_attn_enabled() {
+ static const bool on = [] {
+ const char* e = std::getenv("DS4_FAST_ATTN");
+ return e != nullptr && std::atoi(e) != 0;
+ }();
+ return on;
+}
attnsplit::AttnSplitScratch split_scratch(cudaStream_t s, int heads) {
static std::mutex mu;
static std::map<std::pair<int, cudaStream_t>, attnsplit::AttnSplitScratch> m;
@@ -2094,9 +2251,15 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
heads, d, scale, out);
return;
}
+ // DS4_FAST_ATTN=1: the chunked prompt's matrix-core path (16 heads a block, a 16-key tile). The decode and the
+ // verify windows keep the scalar kernels: one query's heads are not the matrix's rows there.
+ if (fast_attn_enabled() && a.gen != 0 && T >= kFaKeys && heads % kFaHeads == 0) {
+ sparse_attn_wmma_kernel<<<dim3(heads / kFaHeads, T), kThreads, 0, s>>>(q, win, (const uint8_t*) ckv, sel, sel_stride,
+ a, sink, heads, scale, out);
+ return;
+ }
// packed rows: as many heads a block as still leaves two blocks a multiprocessor (a decoded row serves them all)
- int G = heads_per_block;
- if (G != 1 && G != 2 && G != 4 && G != 8) {
+ int G = heads_per_block; if (G != 1 && G != 2 && G != 4 && G != 8) {
G = 8;
while (G > 1 && (heads % G != 0 || (heads / G) * T < 2 * sm_count())) G /= 2;
}
diff --git a/src/dsv41/ds4_wmma.cuh b/src/dsv41/ds4_wmma.cuh
new file mode 100644
index 000000000..569387f98
--- /dev/null
+++ b/src/dsv41/ds4_wmma.cuh
@@ -0,0 +1,39 @@
+// Strata. The gfx11/gfx12 WMMA fragments one warp's 16x16x16 tile is fed and read through (the expert kernels'
+// mx_set_wmma_kernel and the attention kernel's matrix-core path share this form). Each lane gives its row's (or
+// column's) 16 values of a k-step - on gfx12 its half of them - and holds 8 of the 16x16 results: row set_wmma_row(i, hi),
+// column lane % 16.
+#pragma once
+#include <cstdint>
+#if defined(STRATA_USE_HIP)
+#include <hip/hip_runtime.h>
+
+using h16v = _Float16 __attribute__((ext_vector_type(16)));
+using f8v = float __attribute__((ext_vector_type(8)));
+
+inline __device__ __forceinline__ f8v set_wmma(h16v a, h16v b, f8v c) {
+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \
+ defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__)
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#elif defined(__gfx1200__) || defined(__gfx1201__)
+ using h8 = _Float16 __attribute__((ext_vector_type(8)));
+ const int k0 = (threadIdx.x & 31) / 16 * 8;
+ h8 aa, bb;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ aa[i] = a[k0 + i];
+ bb[i] = b[k0 + i];
+ }
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(aa, bb, c);
+#else
+ __builtin_trap();
+ return c;
+#endif
+}
+inline __device__ __forceinline__ int set_wmma_row(int i, int hi) {
+#if defined(__gfx1200__) || defined(__gfx1201__)
+ return 8 * hi + i;
+#else
+ return 2 * i + hi;
+#endif
+}
+#endif