Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
WIP metal: ds4_sparse_attn_mx (sparse attention on simdgroup matrices, DS4_ATTN_MX=1; =2 compares with the scalar kernel: rms rel 1.8e-6); timing and KL gate pending
Niko1221b6dcb23ec609
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 6173cb277..86a444daa 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -1151,7 +1151,9 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intif (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)- const char* fn = ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"+ static const int attn_mode = std::getenv("DS4_ATTN_MX") ? std::atoi(std::getenv("DS4_ATTN_MX")) : 0; // 1: matrix-unit kernel; 2: both, compare+ const bool use_mx = ckv_fmt == 2 && G == 8 && attn_mode != 0;+ const char* fn = use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4": G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1"): "ds4_sparse_attn_bf16";mtl::Pipeline pl;@@ -1175,6 +1177,31 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intdsp.tg[0] = 256;dsp.tg[1] = dsp.tg[2] = 1;g_cx->run({dsp});+ if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the difference+ mtl::Pipeline ps;+ try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }+ mtl::Buffer bo2 = g_cx->alloc(obytes);+ mtl::Dispatch d2 = dsp;+ d2.p = &ps;+ d2.args.back() = mtl::Arg::B(bo2, 0);+ g_cx->run({d2});+ g_cx->flush_runs();+ const float* a = (const float*) ho;+ const float* c = (const float*) bo2.ptr;+ double mx_abs = 0, mx_ref = 0, sum_sq = 0, sum_ref = 0;+ for (size_t i = 0; i < obytes / 4; ++i) {+ const double df = std::fabs((double) a[i] - (double) c[i]);+ mx_abs = std::max(mx_abs, df);+ mx_ref = std::max(mx_ref, (double) std::fabs(c[i]));+ sum_sq += df * df;+ sum_ref += (double) c[i] * (double) c[i];+ }+ static double worst_rel = 0; static long calls = 0;+ const double rel = std::sqrt(sum_sq / (sum_ref + 1e-30));+ worst_rel = std::max(worst_rel, rel);+ if (++calls % 50 == 1) std::fprintf(stderr, "attn mx vs scalar: call %ld T %d max abs %.3g (max |ref| %.3g) rms rel %.3g worst rms rel %.3g\n", calls, T, mx_abs, mx_ref, rel, worst_rel);+ g_cx->free(bo2);+ }return true;}diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex b78715d59..9ce0f2613 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -1776,6 +1776,131 @@ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)#undef DS4_SPARSE_PACKED_SIG#undef DS4_SPARSE_PACKED_BODY+// ---------------------------------------------------------------- sparse attention on the matrix units (G = 8 heads a threadgroup)+//+// The same attention as ds4_sparse_attn_g8 (a token, eight heads, the window rows plus the selected compressed rows, softmax with the head's+// sink) but the dots and the weighted sum run on simdgroup 8x8 matrices: eight rows a pass are staged once as BF16 (exact), q and the+// softmax weights are split into a BF16 high and low part (so the products keep about 16 bits), accumulation is fp32, and the softmax is the+// online one (the running maximum rescales the accumulators through a diagonal matrix).+kernel void ds4_sparse_attn_mx(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint4 tile4[8u * 64u];+ threadgroup float qbuf[4096]; // q hi / lo as BF16 (8 KB each), later the output rows as float+ threadgroup float part[8u * 64u];+ threadgroup float ssm[64];+ threadgroup bfloat phi[64], plo[64];+ threadgroup float dg[64];+ threadgroup float m_s[8], den_s[8], sc_s[8];+ threadgroup bfloat* qhi = (threadgroup bfloat*) qbuf;+ threadgroup bfloat* qlo = qhi + 4096;+ threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;++ const int hb = int(tgp.x) * 8, t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ for (uint x = 0; x < 16u; ++x) {+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;+ const float v = q[((size_t) t * heads + uint(hb) + h) * 512u + d];+ const bfloat hi = bfloat(v);+ qhi[idx] = hi;+ qlo[idx] = bfloat(v - float(hi));+ }+ if (tid < 8u) {+ m_s[tid] = -1e30f;+ den_s[tid] = 0.0f;+ }+ simdgroup_float8x8 O[8];+ for (int c = 0; c < 8; ++c) O[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ const int sr = int(tid) >> 4, pt = int(tid) & 15;+ const int total = n_win + n_sel;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int t0 = 0; t0 < total; t0 += 8) {+ const int n = min(8, total - t0);+ if (sr < 8) {+ if (sr < n) {+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, 0, n, sr, pt, tile4);+ } else {+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) pt * 4u;+ dst[0] = uint4(0); dst[1] = uint4(0); dst[2] = uint4(0); dst[3] = uint4(0);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ { // S = q . K^T over this simdgroup's eighth of the 512 values+ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {+ simdgroup_bfloat8x8 a_hi, a_lo, b;+ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);+ simdgroup_load(a_hi, qhi, 512, ulong2(ulong(kb) * 8ul, 0ul));+ simdgroup_load(a_lo, qlo, 512, ulong2(ulong(kb) * 8ul, 0ul));+ simdgroup_multiply_accumulate(acc, a_hi, b, acc);+ simdgroup_multiply_accumulate(acc, a_lo, b, acc);+ }+ simdgroup_store(acc, part + uint(sg) * 64u, 8);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid < 64u) {+ float s = 0.0f;+ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];+ ssm[tid] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ { // head sg's online softmax over the pass's rows+ const bool valid = int(lane) < n;+ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;+ const float m = ds4_attn_warp_max(s);+ const float m_old = m_s[sg];+ const float m_new = fmax(m_old, m);+ const float p = valid ? exp(s - m_new) : 0.0f;+ const float ps = ds4_warp_sum(p);+ if (lane < 8u) {+ const bfloat hi = bfloat(p);+ phi[uint(sg) * 8u + uint(lane)] = hi;+ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));+ }+ if (lane == 0u) {+ const float cs = exp(m_old - m_new);+ sc_s[sg] = cs;+ den_s[sg] = den_s[sg] * cs + ps;+ m_s[sg] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[tid >> 3] : 0.0f;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ { // O = diag(rescale) O + P V on this simdgroup's 64 columns+ simdgroup_float8x8 D;+ simdgroup_bfloat8x8 ph, pl;+ simdgroup_load(D, dg, 8);+ simdgroup_load(ph, phi, 8);+ simdgroup_load(pl, plo, 8);+ for (int c = 0; c < 8; ++c) {+ simdgroup_float8x8 tmp;+ simdgroup_multiply(tmp, D, O[c]);+ simdgroup_bfloat8x8 v;+ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));+ simdgroup_multiply_accumulate(tmp, ph, v, tmp);+ simdgroup_multiply_accumulate(tmp, pl, v, tmp);+ O[c] = tmp;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ for (int c = 0; c < 8; ++c) simdgroup_store(O[c], qbuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint x = 0; x < 16u; ++x) {+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;+ const float den = den_s[h] + exp(sink[uint(hb) + h] - m_s[h]);+ out[((size_t) t * heads + uint(hb) + h) * 512u + d] = qbuf[idx] / den;+ }+}+// kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the// softmax, every thread two output values.static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,