Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
19.2 KB
Raw Blame
// metal/kernels/qsa.metal - QSA (sparse full attention) decode for a window of tokens, between the projections.
// Contract of include/strata/kernels/qsa.hpp / qsa_select.hpp / qsa_decode_attn.hpp / kv_q8.hpp (the engine's
// native path), in a Metal layout (no paging: unified memory):
//
//   qsa_prep    per token t (cell pos0 + t): q heads rms_norm * q_norm, NEOX RoPE on 64 dims -> qcur;
//               k rms_norm * k_norm, RoPE -> int8 KV append (fp16 scale per 64 = fp16(amax/127), rint, clamp);
//               v -> int8 append; indexer q rms_norm * iq_norm, RoPE -> qidx.  The indexer KEYS (last
//               threadgroup, all window tokens in order): raw tail [cell % 4]; on a block completion
//               pooled[b] = rope(rms_norm(mean of the block's 4 raw keys) * ik_norm, pos of the block's first cell);
//               cell 0 also sets dead = rms_norm(raw) * ik_norm (the tail block's key).
//   qsa_score   per token, block b <= n_bid: sum over the 4 indexer heads of relu(qidx . key_b); key_b = pooled[b],
//               the tail block n_bid scores `dead` and gets +1e9 when it has cells.
//   qsa_topk    per token: width = min(n_kv, 2051) cells; n_kv <= 2051: all; else the width cells of largest
//               score (a cell inherits its block's), ties to the lowest cell, emitted ascending (radix select).
//   qsa_attn    split-K: (chunk of CH cells, kv head, token): 12 query heads; s = q.k / 16, chunk max m, sum l,
//               acc = p . v  -> partials
//   qsa_merge   per (head, token): log-sum-exp merge, * sigmoid(gate), written as prepared activations (xprep
//               planes) for attn_output.
//
// RoPE: cos/sin from a host table [pos][32] (double precision, as build_rope_table).

#define QH 24
#define QKV 2
#define QD 256
#define QROT 64
#define IH 4
#define ID 128
#define QBUDGET 2051
#define QCH 64

constant int QSA_NTOK [[function_constant(3)]];
// QSA_MROPE (function constant 24): interleaved M-RoPE (the vision path; strata/kernels/mrope.hpp, ggml IMROPE,
// sections 11/11/10): buffer 20 of qsa_prep is a table int32 [cells][3] (t, h, w), and rotary pair i (0..31) of the row
// at (pos_base + cell) rotates by the cos/sin table's row tab[(pos_base + cell) * 3 + i % 3] instead of its own -
// the pooled indexer keys too (position pos_base + 4 b of a block).  Off: exactly the plain RoPE.
constant bool QSA_MROPE [[function_constant(24)]];
constant bool qsa_mrope = is_function_constant_defined(QSA_MROPE) && QSA_MROPE;

struct QsaArgs {
    uint pos0;        // the window's first cell (relative to the sequence)
    uint pos_base;    // absolute position of cell 0 (RoPE)
    uint max_cells;
    uint n_tok;
    float eps;
};

// cache layout per layer
//   kq/vq [cell][kv head][256] int8, ks/vs [cell][kv head][4] half
//   tail [4][128] f32 (slot = cell % 4), dead [128], pooled [max_cells / 4 + 1][128]

static inline float tg_sum256(float v, threadgroup float* part, ushort sg, ushort lane, ushort nsg) {
    v = simd_sum(v);
    if (lane == 0) part[sg] = v;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float t = 0.0f;
    for (int i = 0; i < nsg; ++i) t += part[i];
    threadgroup_barrier(mem_flags::mem_threadgroup);
    return t;
}

// one row (width w <= 256 = threads) : out = rope((rs * x) * g)  (RoPE on the first 64 dims, NEOX pairs i, i+32)
static inline float norm_rope(float x, float xhi_unused, device const float* g, threadgroup float* row,
                              threadgroup float* part, ushort tid, ushort sg, ushort lane, int w, float eps,
                              device const float2* cs, uint absp, bool rope, device const int* mtab) {
    const float ss = tg_sum256(tid < w ? x * x : 0.0f, part, sg, lane, 8);
    const float rs = rsqrt(ss / float(w) + eps);
    const float y = tid < w ? (rs * x) * g[tid] : 0.0f;
    row[tid] = y;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float o = y;
    if (rope && tid < QROT) {
        const int p = tid & 31;
        const uint pp = qsa_mrope ? uint(mtab[absp * 3 + uint(p % 3)]) : absp;
        const float2 c = cs[pp * 32 + p];
        const float a = row[p], b = row[p + 32];
        o = tid < 32 ? a * c.x - b * c.y : a * c.y + b * c.x;
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    (void)xhi_unused;
    return o;
}

// grid (32 rows + 1, n_tok): rows 0..23 q heads, 24..25 k heads, 26..27 v heads, 28..31 indexer q heads;
// threadgroup 32 (x = 32, y = 0) is the indexer-key threadgroup.  256 threads.
kernel void qsa_prep(device const float* qfull [[buffer(0)]], device const float* kcur [[buffer(1)]],
                     device const float* vcur [[buffer(2)]], device const float* iqraw [[buffer(3)]],
                     device const float* ikraw [[buffer(4)]], device const float* qn [[buffer(5)]],
                     device const float* kn [[buffer(6)]], device const float* iqn [[buffer(7)]],
                     device const float* ikn [[buffer(8)]], device const float2* cs [[buffer(9)]],
                     device float* qcur [[buffer(10)]], device float* qidx [[buffer(11)]],
                     device char* kq [[buffer(12)]], device char* vq [[buffer(13)]], device half* ksc [[buffer(14)]],
                     device half* vsc [[buffer(15)]], device float* tail [[buffer(16)]], device float* dead [[buffer(17)]],
                     device float* pooled [[buffer(18)]], constant QsaArgs& a [[buffer(19)]],
                     device const int* mtab [[buffer(20)]],
                     uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]],
                     ushort sg [[simdgroup_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
    threadgroup float part[8];
    threadgroup float row[256];
    threadgroup float amax[8];
    const int r = int(tg.x);
    if (r == 32) {
        // the indexer keys, every window token in order (only threadgroup (32, 0) works)
        if (tg.y != 0) return;
        for (int t = 0; t < int(a.n_tok); ++t) {
            const uint cell = a.pos0 + uint(t);
            const float x = tid < ID ? ikraw[t * ID + tid] : 0.0f;
            if (tid < ID) tail[(cell & 3) * ID + tid] = x;
            threadgroup_barrier(mem_flags::mem_device);
            if (cell == 0) {
                const float o = norm_rope(x, 0.0f, ikn, row, part, tid, sg, lane, ID, a.eps, cs, 0, false, mtab);
                if (tid < ID) dead[tid] = o;
            }
            if ((cell & 3) == 3) {
                float m = 0.0f;
                if (tid < ID) m = (((tail[tid] + tail[ID + tid]) + tail[2 * ID + tid]) + tail[3 * ID + tid]) * 0.25f;
                const uint b = cell >> 2;
                const float o = norm_rope(m, 0.0f, ikn, row, part, tid, sg, lane, ID, a.eps, cs, a.pos_base + 4 * b, true, mtab);
                if (tid < ID) pooled[b * ID + tid] = o;
            }
            threadgroup_barrier(mem_flags::mem_device);
        }
        return;
    }
    const int t = int(tg.y);
    const uint cell = a.pos0 + uint(t), absp = a.pos_base + cell;
    if (r < QH) {
        const float x = qfull[(t * QH + r) * 2 * QD + tid];
        const float o = norm_rope(x, 0.0f, qn, row, part, tid, sg, lane, QD, a.eps, cs, absp, true, mtab);
        qcur[(t * QH + r) * QD + tid] = o;
    } else if (r < QH + 2 * QKV) {
        const bool isv = r >= QH + QKV;
        const int h = r - QH - (isv ? QKV : 0);
        float x = (isv ? vcur : kcur)[(t * QKV + h) * QD + tid];
        if (!isv) x = norm_rope(x, 0.0f, kn, row, part, tid, sg, lane, QD, a.eps, cs, absp, true, mtab);
        // int8 with an fp16 scale per 64 values: groups = simdgroup pairs (64 threads)
        float m = simd_max(fabs(x));
        if (lane == 0) amax[sg] = m;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const int g = tid >> 6;
        m = max(amax[2 * g], amax[2 * g + 1]);
        const half sh = half(m / 127.0f);
        const float sf = float(sh);
        int q = 0;
        if (sf > 0.0f) q = clamp(int(rint(x / sf)), -127, 127);
        const uint rowi = cell * QKV + uint(h);
        (isv ? vq : kq)[rowi * QD + tid] = char(q);
        if ((tid & 63) == 0) (isv ? vsc : ksc)[rowi * 4 + g] = sh;
    } else {
        const int h = r - QH - 2 * QKV;
        const float x = tid < ID ? iqraw[(t * IH + h) * ID + tid] : 0.0f;
        const float o = norm_rope(x, 0.0f, iqn, row, part, tid, sg, lane, ID, a.eps, cs, absp, true, mtab);
        if (tid < ID) qidx[(t * IH + h) * ID + tid] = o;
    }
}

// scores [n_tok][max_blocks]; grid (ceil(max_blocks / 8), n_tok), 256 threads: a simdgroup per block.
kernel void qsa_score(device const float* qidx [[buffer(0)]], device const float* pooled [[buffer(1)]],
                      device const float* dead [[buffer(2)]], device float* scores [[buffer(3)]],
                      constant QsaArgs& a [[buffer(4)]],
                      uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],
                      ushort lane [[thread_index_in_simdgroup]]) {
    const int t = int(tg.y);
    const uint n_kv = a.pos0 + uint(t) + 1, n_bid = n_kv >> 2;
    const uint b = tg.x * 8 + sg;
    if (b > n_bid) return;
    device const float* key = b == n_bid ? dead : pooled + b * ID;
    float k[4];
    for (int i = 0; i < 4; ++i) k[i] = key[lane + 32 * i];
    float s = 0.0f;
    for (int h = 0; h < IH; ++h) {
        float d = 0.0f;
        for (int i = 0; i < 4; ++i) d = fma(qidx[(t * IH + h) * ID + lane + 32 * i], k[i], d);
        d = simd_sum(d);
        s += max(d, 0.0f);
    }
    if (b == n_bid && (n_kv & 3) != 0) s += 1e9f;
    if (lane == 0) scores[uint(t) * (a.max_cells / 4 + 2) + b] = s;
}

// ids [n_tok][QBUDGET], ascending.  One threadgroup of 1024 threads per token.  Order key of a float: larger score =
// larger key; a radix select (4 passes of 8 bits) over the blocks with weights = their cell counts finds the
// threshold key T and how many cells with key == T to take (the lowest cells first).
static inline uint okey(float f) {
    const uint u = as_type<uint>(f);
    return (u & 0x80000000u) ? ~u : (u | 0x80000000u);
}
kernel void qsa_topk(device const float* scores [[buffer(0)]], device int* ids [[buffer(1)]],
                     constant QsaArgs& a [[buffer(2)]],
                     uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]],
                     ushort sg [[simdgroup_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
    threadgroup atomic_uint hist[256];
    threadgroup uint sel[4];
    threadgroup uint pref[32];
    const int t = int(tg.x);
    const uint n_kv = a.pos0 + uint(t) + 1, n_bid = n_kv >> 2, nb = n_bid + 1;
    device int* out = ids + t * QBUDGET;
    device const float* sc = scores + uint(t) * (a.max_cells / 4 + 2);
    if (n_kv <= QBUDGET) {
        for (uint c = tid; c < n_kv; c += 1024) out[c] = int(c);
        return;
    }
    // radix select of the `need`-th largest cell key
    uint prefix = 0, pmask = 0, need = QBUDGET;
    for (int pass = 0; pass < 4; ++pass) {
        const int shift = 24 - 8 * pass;
        if (tid < 256) atomic_store_explicit(&hist[tid], 0u, memory_order_relaxed);
        threadgroup_barrier(mem_flags::mem_threadgroup);
        for (uint b = tid; b < nb; b += 1024) {
            const uint k = okey(sc[b]);
            if ((k & pmask) == prefix) {
                const uint w = b < n_bid ? 4u : (n_kv & 3u);
                if (w) atomic_fetch_add_explicit(&hist[(k >> shift) & 255u], w, memory_order_relaxed);
            }
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == 0) {
            uint acc = 0, d = 0;
            for (int v = 255; v >= 0; --v) {
                const uint h = atomic_load_explicit(&hist[v], memory_order_relaxed);
                if (acc + h >= need) { d = uint(v); break; }
                acc += h;
            }
            sel[0] = d;
            sel[1] = need - acc;
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        prefix |= sel[0] << shift;
        pmask |= 255u << shift;
        need = sel[1];
        threadgroup_barrier(mem_flags::mem_threadgroup);
    }
    // T = prefix; take every cell with key > T and the first `need` cells (lowest index) with key == T, ascending.
    // Pass over blocks in order of index with a prefix scan of each block's selected cell count.
    const uint T = prefix;
    uint base = 0, eq_taken = 0;
    for (uint b0 = 0; b0 < nb; b0 += 1024) {
        const uint b = b0 + tid;
        uint w = 0, weq = 0;
        if (b < nb) {
            const uint k = okey(sc[b]);
            const uint cells = b < n_bid ? 4u : (n_kv & 3u);
            if (k > T) w = cells;
            else if (k == T) weq = cells;
        }
        // equal-key cells: the lowest first, `need` of them in total
        const uint eq_before = simd_prefix_exclusive_sum(weq);
        if (lane == 31) pref[sg] = eq_before + weq;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        uint eq_sg = 0;
        for (int i = 0; i < int(sg); ++i) eq_sg += pref[i];
        uint eq_tot = 0;
        for (int i = 0; i < 32; ++i) eq_tot += pref[i];
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const uint eq0 = eq_taken + eq_sg + eq_before;
        uint take_eq = 0;
        if (weq && eq0 < need) take_eq = min(weq, need - eq0);
        const uint mine = w + take_eq;
        const uint before = simd_prefix_exclusive_sum(mine);
        if (lane == 31) pref[sg] = before + mine;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        uint off = base;
        for (int i = 0; i < int(sg); ++i) off += pref[i];
        uint tot = 0;
        for (int i = 0; i < 32; ++i) tot += pref[i];
        threadgroup_barrier(mem_flags::mem_threadgroup);
        off += before;
        for (uint c = 0; c < mine; ++c) out[off + c] = int(4 * b + c);
        base += tot;
        eq_taken += eq_tot;
    }
}

// partials: [n_tok][chunks][QH][QD + 2] (acc, then m, l).  grid (chunks, QKV, n_tok), 256 threads.
kernel void qsa_attn(device const float* qcur [[buffer(0)]], device const char* kq [[buffer(1)]],
                     device const char* vq [[buffer(2)]], device const half* ksc [[buffer(3)]],
                     device const half* vsc [[buffer(4)]], device const int* ids [[buffer(5)]],
                     device float* part [[buffer(6)]], constant QsaArgs& a [[buffer(7)]],
                     uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]],
                     ushort sg [[simdgroup_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
    constexpr int HPG = QH / QKV;   // 12 query heads per kv head
    threadgroup float qs[HPG][QD];
    threadgroup float s[HPG][QCH];
    threadgroup int cid[QCH];
    const int chunk = int(tg.x), g = int(tg.y), t = int(tg.z);
    const uint n_kv = a.pos0 + uint(t) + 1, width = min(n_kv, uint(QBUDGET));
    const int c0 = chunk * QCH;
    const int nchunks = (QBUDGET + QCH - 1) / QCH;
    device float* pp = part + ((uint(t) * nchunks + chunk) * QH) * (QD + 2);
    if (uint(c0) >= width) {
        // an empty chunk: m = -inf, l = 0 (the merge skips it)
        for (int h = 0; h < HPG; ++h)
            if (tid == 0) { pp[(g * HPG + h) * (QD + 2) + QD] = -INFINITY; pp[(g * HPG + h) * (QD + 2) + QD + 1] = 0.0f; }
        return;
    }
    const int nc = min(QCH, int(width) - c0);
    for (int h = 0; h < HPG; ++h) qs[h][tid] = qcur[(t * QH + g * HPG + h) * QD + tid];
    if (tid < QCH) cid[tid] = tid < nc ? ids[t * QBUDGET + c0 + tid] : 0;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    // scores: 12 x 64 dots of 256; simdgroup sg takes cells sg, sg + 8, ...; lanes split the 256 dims (8 each)
    for (int c = sg; c < nc; c += 8) {
        const uint rowi = uint(cid[c]) * QKV + uint(g);
        device const char* kr = kq + rowi * QD;
        float kv[8];
        const float scl = float(ksc[rowi * 4 + (lane >> 3)]);
        for (int i = 0; i < 8; ++i) kv[i] = float(kr[lane * 8 + i]) * scl;
        for (int h = 0; h < HPG; ++h) {
            float d = 0.0f;
            for (int i = 0; i < 8; ++i) d = fma(qs[h][lane * 8 + i], kv[i], d);
            d = simd_sum(d);
            if (lane == 0) s[h][c] = d * 0.0625f;
        }
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    // per head: max, exp, sum (simdgroup h % 8 handles heads h, h + 8)
    threadgroup float mh[HPG], lh[HPG];
    for (int h = sg; h < HPG; h += 8) {
        float m = -INFINITY;
        for (int c = lane; c < nc; c += 32) m = max(m, s[h][c]);
        m = simd_max(m);
        float l = 0.0f;
        for (int c = lane; c < nc; c += 32) {
            const float p = exp(s[h][c] - m);
            s[h][c] = p;
            l += p;
        }
        l = simd_sum(l);
        if (lane == 0) { mh[h] = m; lh[h] = l; }
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    // acc[h][j] = sum_c p[h][c] v[c][j]: thread = dim j
    float acc[HPG];
    for (int h = 0; h < HPG; ++h) acc[h] = 0.0f;
    for (int c = 0; c < nc; ++c) {
        const uint rowi = uint(cid[c]) * QKV + uint(g);
        const float v = float(vq[rowi * QD + tid]) * float(vsc[rowi * 4 + (tid >> 6)]);
        for (int h = 0; h < HPG; ++h) acc[h] = fma(s[h][c], v, acc[h]);
    }
    for (int h = 0; h < HPG; ++h) {
        device float* o = pp + (g * HPG + h) * (QD + 2);
        o[tid] = acc[h];
        if (tid == 0) { o[QD] = mh[h]; o[QD + 1] = lh[h]; }
    }
}

// grid (QH, n_tok), 256 threads: out = merge * sigmoid(gate), as prepared activations of n = QH * QD per token
kernel void qsa_merge(device const float* part [[buffer(0)]], device const float* qfull [[buffer(1)]],
                      device float* xp [[buffer(2)]], constant QsaArgs& a [[buffer(3)]],
                      uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {
    threadgroup float msum[256];
    const int h = int(tg.x), t = int(tg.y);
    const int nchunks = (QBUDGET + QCH - 1) / QCH;
    const uint n_kv = a.pos0 + uint(t) + 1, width = min(n_kv, uint(QBUDGET));
    const int nc = int((width + QCH - 1) / QCH);
    device const float* pp = part + (uint(t) * nchunks * QH + uint(h)) * (QD + 2);
    float M = -INFINITY;
    for (int k = 0; k < nc; ++k) M = max(M, pp[k * QH * (QD + 2) + QD]);
    float L = 0.0f, o = 0.0f;
    for (int k = 0; k < nc; ++k) {
        device const float* q = pp + k * QH * (QD + 2);
        const float e = exp(q[QD] - M);
        L = fma(q[QD + 1], e, L);
        o = fma(q[tid], e, o);
    }
    o = o / L;
    const float gate = qfull[(t * QH + h) * 2 * QD + QD + tid];
    const float y = o * (1.0f / (1.0f + exp(-gate)));
    // prepared planes (xprep layout) of the n = QH * QD vector
    const uint n = QH * QD, plane = a.n_tok * n, d = uint(h) * QD + tid;
    const int j = tid & 15;
    const float q2s[8] = {1.0f, 0.25f, 0.0625f, 0.015625f, 0.00390625f, 0.0009765625f, 0.000244140625f, 0.00006103515625f};
    const float p4s[4] = {1.0f, 0.00390625f, 1.52587890625e-05f, 5.9604644775390625e-08f};
    device float* raw = xp + uint(t) * n + d;
    raw[0] = y;
    raw[plane] = y * q2s[j & 7];
    raw[2 * plane] = y * p4s[j & 3];
    raw[3 * plane] = (j & 1) ? y * 0.00390625f : y;
    msum[tid] = y;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (j == 0) {
        float s0 = 0.0f, s1 = 0.0f;
        for (int k = 0; k < 16; k += 2) { s0 += msum[tid + k]; s1 += msum[tid + k + 1]; }
        xp[4 * plane + uint(t) * (n / 16) + d / 16] = s0 + s1;
    }
}