Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
4.4 KB
Raw Blame
// metal/src/engine/msl/mtp_chain.metal - the two steps that keep a whole MTP draft round on the GPU (one command
// buffer for every draft step; compiled after mac-kern's quant.metal and units.metal):
//
//   mtp_pick     draft j: the pick's token id (through the draft vocabulary) and its probability, the softmax of
//                the draft head's logits at the pick (the host computed it after every step before)
//   mtp_embed_F  the next step's input: the picked token's embedding row dequantized from the model's token_embd
//                (format F) - the host dequantized it before
//
// mtp_embed reads a row with mac-kern's unit decoders (units.metal Dec<F>): a unit's dot with x is
//   d1 * sum_{j < UNIT/2} q[j] xs[j] + d2 * sum_{j >= UNIT/2} q[j] xs[j] - (m1 xsum0 + m2 xsum1)
// with xs the prepared x (raw x times the kind's 2^-k) and xsum the raw sums of 16, so weight j of the unit is
//   w = d * (q[j] * 2^-k) - m        (d = d1 or d2 by half, m = m1 for the first 16 x values, m2 for the rest;
//                                     a 16-wide unit has one sum: m1)
// and its position in the row is what X<F>::load reads when x holds the positions 0 .. n-1.

struct PickArgs {
    uint nv;        // the draft head's rows (the draft vocabulary, or the whole vocabulary)
    uint j;         // the draft step
    uint use_map;   // 1: map the head's row through the draft vocabulary
};

// one threadgroup of 1024
kernel void mtp_pick(device const float* lg [[buffer(0)]], device const int* pick [[buffer(1)]],
                     device const int* vmap [[buffer(2)]], device int* dout [[buffer(3)]], device float* dprob [[buffer(4)]],
                     constant PickArgs& a [[buffer(5)]], ushort tid [[thread_index_in_threadgroup]],
                     ushort sg [[simdgroup_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
    threadgroup float part[32];
    const int di = pick[0];
    const float mx = lg[di];
    float s = 0.0f;
    for (uint i = tid; i < a.nv; i += 1024) s += exp(lg[i] - mx);
    s = simd_sum(s);
    if (lane == 0) part[sg] = s;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float t = 0.0f;
        for (int k = 0; k < 32; ++k) t += part[k];
        dout[a.j] = a.use_map ? vmap[di] : di;
        dprob[a.j] = 1.0f / t;
    }
}

struct EmbArgs {
    uint n;           // the row's width
    uint row_bytes;
    uint j;           // dout[j] is the token
};

template <int F> static inline float kind_scale(int pos) {
    const int k = XKind<F>::K;
    if (k == XK_Q2) {
        const float q2s[8] = {1.0f, 0.25f, 0.0625f, 0.015625f, 0.00390625f, 0.0009765625f, 0.000244140625f, 0.00006103515625f};
        return q2s[pos & 7];
    }
    if (k == XK_P4) {
        const float p4s[4] = {1.0f, 0.00390625f, 1.52587890625e-05f, 5.9604644775390625e-08f};
        return p4s[pos & 3];
    }
    if (k == XK_P2) return (pos & 1) ? 0.00390625f : 1.0f;
    return 1.0f;
}

// one threadgroup of 256; posv = the floats 0 .. n-1
template <int F>
kernel void mtp_embed(device const uchar* W [[buffer(0)]], device const int* dout [[buffer(1)]],
                      device const float* posv [[buffer(2)]], device float* y [[buffer(3)]], constant EmbArgs& a [[buffer(4)]],
                      ushort tid [[thread_index_in_threadgroup]]) {
    constexpr int UNIT = fmt<F>::UNIT, H = UNIT / 2;
    threadgroup uint tab[fmt<F>::TAB];
    load_tab<F>(tab, tid, 256);
    threadgroup_barrier(mem_flags::mem_threadgroup);
    device const uchar* row = W + (ulong) dout[a.j] * a.row_bytes;
    const int U = int(a.n) / UNIT;
    for (int u = tid; u < U; u += 256) {
        float q[UNIT], ps[UNIT], xsum[2];
        float4 sc;
        Dec<F>::run(row, u, q, sc, tab);
        X<F>::load(posv, posv, u, ps, xsum);
        for (int j = 0; j < UNIT; ++j) {
            const int pos = int(ps[j]);
            const float d = j < H ? sc.x : sc.y;
            const float m = (UNIT == 16 || j < 16) ? sc.z : sc.w;
            y[pos] = d * (q[j] * kind_scale<F>(pos)) - m;
        }
    }
}

// host names by the format's GGUF type number (mtp_embed_11 = Q3_K): EMB expands F before EMB_ stringifies it
#define EMB_(F) template [[host_name("mtp_embed_" #F)]] kernel decltype(mtp_embed<F>) mtp_embed<F>;
#define EMB(F) EMB_(F)
EMB(T_BF16) EMB(T_Q8_0) EMB(T_Q4_0) EMB(T_Q5_0) EMB(T_IQ4_NL) EMB(T_IQ4_XS) EMB(T_Q2_0) EMB(T_IQ3_S) EMB(T_Q3_K)
EMB(T_Q4_K) EMB(T_Q5_K) EMB(T_Q6_K) EMB(T_IQ3_XXS)