Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
15.5 KB
Raw Blame
// metal/tests/qsa_parity.cpp - QSA decode (qsa.metal) stage by stage against a CPU reference, and its speed.
//
//   qsa_parity [P]          a context of P cells built by qsa_prep (default 3000: the selection is active above
//                           2051), then a window of 3 tokens: prep, score, top-k, attention, merge.  Each stage is
//                           checked against the CPU fed with the GPU's inputs to that stage; the ids exactly.
//   qsa_parity --bench [P]  GPU time of the window's kernels at context P, 1 and 3 tokens
#include "mtl_rt.h"
#include "ref_quant.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>

namespace {
constexpr int QH = 24, QKV = 2, QD = 256, IH = 4, ID = 128, BUDGET = 2051, CH = 64, NCH = (BUDGET + CH - 1) / CH;
struct QsaArgs { uint32_t pos0, pos_base, max_cells, n_tok; float eps; };
double median(std::vector<double> v) { std::sort(v.begin(), v.end()); return v[v.size() / 2]; }
struct Err {
    double se = 0, sr = 0;
    void add(double g, double r) { se += (g - r) * (g - r); sr += r * r; }
    double l2() const { return sr > 0 ? std::sqrt(se / sr) : std::sqrt(se); }
};
float h2f(uint16_t h) { return ref::f16_to_f32(h); }
}  // namespace

int main(int argc, char** argv) {
    bool bench = false;
    int P = 3000;
    for (int i = 1; i < argc; ++i) {
        std::string a = argv[i];
        if (a == "--bench") bench = true;
        else P = std::atoi(argv[i]);
    }
    const int MAXT = 8, MAXC = std::max(4096, (P + 64) / 4 * 4 + 64);
    mtl::Context cx;
    const int lib = cx.compile({"qsa.metal", "qsa2.metal"}, "#define NO_GGML_COMMON 1\n");
    // QSA_TOPK=1: the v1 top-k; QSA_ATTN=1|2|3 (default 3): which attention (2 has its own chunking and merge)
    const int av = std::getenv("QSA_ATTN") ? std::atoi(std::getenv("QSA_ATTN")) : 1;
    const bool v2 = av == 2;
    mtl::Pipeline pprep = cx.pipeline(lib, "qsa_prep", mtl::Constants().b(24, std::getenv("QSA_MROPE") != nullptr)), pscore = cx.pipeline(lib, "qsa_score"),
                  ptopk = cx.pipeline(lib, std::getenv("QSA_TOPK") ? "qsa_topk" : "qsa_topk2"),
                  pattn = cx.pipeline(lib, av == 1 ? "qsa_attn" : av == 2 ? "qsa_attn2" : "qsa_attn3"),
                  pmerge = cx.pipeline(lib, v2 ? "qsa_merge2" : "qsa_merge");
    const int MB = MAXC / 4 + 2;
    // per-cell inputs for every cell (context + window)
    const int NC = P + MAXT;
    mtl::Buffer qfull = cx.alloc(4 * (size_t)NC * QH * 2 * QD), kc = cx.alloc(4 * (size_t)NC * QKV * QD),
                vc = cx.alloc(4 * (size_t)NC * QKV * QD), iq = cx.alloc(4 * (size_t)NC * IH * ID),
                ik = cx.alloc(4 * (size_t)NC * ID), qn = cx.alloc(4 * QD), kn = cx.alloc(4 * QD), iqn = cx.alloc(4 * ID),
                ikn = cx.alloc(4 * ID), cs = cx.alloc(8 * (size_t)(MAXC + 16) * 32), qcur = cx.alloc(4 * MAXT * QH * QD),
                qidx = cx.alloc(4 * MAXT * IH * ID), kq = cx.alloc((size_t)MAXC * QKV * QD), vq = cx.alloc((size_t)MAXC * QKV * QD),
                ksc = cx.alloc(2 * (size_t)MAXC * QKV * 4), vsc = cx.alloc(2 * (size_t)MAXC * QKV * 4), tail = cx.alloc(4 * 4 * ID),
                dead = cx.alloc(4 * ID), pooled = cx.alloc(4 * (size_t)MB * ID), scores = cx.alloc(4 * (size_t)MAXT * MB),
                ids = cx.alloc(4 * MAXT * BUDGET), part = cx.alloc(4 * (size_t)MAXT * 2 * NCH * QH * (QD + 2)),
                xp = cx.alloc(4 * (4 * MAXT * QH * QD + MAXT * QH * QD / 16));
    ref::Rng r(23);
    for (size_t i = 0; i < qfull.bytes / 4; ++i) qfull.as<float>()[i] = r.sym() * 2.0f;
    for (size_t i = 0; i < kc.bytes / 4; ++i) kc.as<float>()[i] = r.sym() * 2.0f;
    for (size_t i = 0; i < vc.bytes / 4; ++i) vc.as<float>()[i] = r.sym();
    for (size_t i = 0; i < iq.bytes / 4; ++i) iq.as<float>()[i] = r.sym();
    for (size_t i = 0; i < ik.bytes / 4; ++i) ik.as<float>()[i] = r.sym();
    for (int i = 0; i < QD; ++i) { qn.as<float>()[i] = 1.0f + 0.2f * r.sym(); kn.as<float>()[i] = 1.0f + 0.2f * r.sym(); }
    for (int i = 0; i < ID; ++i) { iqn.as<float>()[i] = 1.0f + 0.2f * r.sym(); ikn.as<float>()[i] = 1.0f + 0.2f * r.sym(); }
    const uint32_t pos_base = 0;
    // QSA_MROPE=1: qsa_prep with the M-RoPE table (function constant 24): random (t, h, w) positions per cell, the CPU
    // reference rotating pair i by row tab[cell * 3 + i % 3]; QSA_MROPE=id: the identity table, checked bitwise
    // against the plain kernel
    const char* mre = std::getenv("QSA_MROPE");
    const bool mrope = mre != nullptr, mid = mrope && std::string(mre) == "id";
    mtl::Buffer mtab = cx.alloc(4 * 3 * (size_t)(MAXC + 16));
    {
        ref::Rng rm(77);
        for (int c = 0; c < MAXC + 16; ++c)
            for (int k = 0; k < 3; ++k) mtab.as<int>()[c * 3 + k] = mid ? c : (int)(rm.next() % (uint64_t)(MAXC + 16));
    }
    for (int p = 0; p < MAXC + 16; ++p)
        for (int i = 0; i < 32; ++i) {
            const double inv = std::pow(1e7, -2.0 * i / 64.0), th = (double)p * inv;
            cs.as<float>()[(p * 32 + i) * 2] = (float)std::cos(th);
            cs.as<float>()[(p * 32 + i) * 2 + 1] = (float)std::sin(th);
        }
    auto prep = [&](uint32_t pos0, int nt) {
        QsaArgs a{pos0, pos_base, (uint32_t)MAXC, (uint32_t)nt, 1e-6f};
        mtl::Dispatch d;
        d.p = &pprep;
        d.args = {mtl::Arg::B(qfull, 4 * (size_t)pos0 * QH * 2 * QD), mtl::Arg::B(kc, 4 * (size_t)pos0 * QKV * QD),
                  mtl::Arg::B(vc, 4 * (size_t)pos0 * QKV * QD), mtl::Arg::B(iq, 4 * (size_t)pos0 * IH * ID),
                  mtl::Arg::B(ik, 4 * (size_t)pos0 * ID), mtl::Arg::B(qn), mtl::Arg::B(kn), mtl::Arg::B(iqn),
                  mtl::Arg::B(ikn), mtl::Arg::B(cs), mtl::Arg::B(qcur), mtl::Arg::B(qidx), mtl::Arg::B(kq),
                  mtl::Arg::B(vq), mtl::Arg::B(ksc), mtl::Arg::B(vsc), mtl::Arg::B(tail), mtl::Arg::B(dead),
                  mtl::Arg::B(pooled), mtl::Arg::V(a), mtl::Arg::B(mtab)};
        d.grid[0] = 33;
        d.grid[1] = nt;
        d.tg[0] = 256;
        return d;
    };
    auto window = [&](uint32_t pos0, int nt, std::vector<mtl::Dispatch>& ds) {
        QsaArgs a{pos0, pos_base, (uint32_t)MAXC, (uint32_t)nt, 1e-6f};
        ds.push_back(prep(pos0, nt));
        mtl::Dispatch s;
        s.p = &pscore;
        s.args = {mtl::Arg::B(qidx), mtl::Arg::B(pooled), mtl::Arg::B(dead), mtl::Arg::B(scores), mtl::Arg::V(a)};
        s.grid[0] = ((pos0 + nt) / 4 + 1 + 7) / 8;
        s.grid[1] = nt;
        s.tg[0] = 256;
        ds.push_back(s);
        mtl::Dispatch k;
        k.p = &ptopk;
        k.args = {mtl::Arg::B(scores), mtl::Arg::B(ids), mtl::Arg::V(a)};
        k.grid[0] = nt;
        k.tg[0] = 1024;
        ds.push_back(k);
        mtl::Dispatch at;
        at.p = &pattn;
        at.args = {mtl::Arg::B(qcur), mtl::Arg::B(kq), mtl::Arg::B(vq), mtl::Arg::B(ksc), mtl::Arg::B(vsc),
                   mtl::Arg::B(ids), mtl::Arg::B(part), mtl::Arg::V(a)};
        at.grid[0] = v2 ? (BUDGET + 31) / 32 : NCH;
        at.grid[1] = v2 ? QKV * 2 : QKV;
        at.grid[2] = nt;
        at.tg[0] = 256;
        ds.push_back(at);
        mtl::Dispatch m;
        m.p = &pmerge;
        m.args = {mtl::Arg::B(part), mtl::Arg::B(qfull, 4 * (size_t)pos0 * QH * 2 * QD), mtl::Arg::B(xp), mtl::Arg::V(a)};
        m.grid[0] = QH;
        m.grid[1] = nt;
        m.tg[0] = 256;
        ds.push_back(m);
    };
    // the context, one cell at a time
    {
        std::vector<mtl::Dispatch> ds;
        for (int p = 0; p < P; ++p) ds.push_back(prep(p, 1));
        cx.run(ds);
    }
    if (bench) {
        std::printf("device %s, context %d cells\n", cx.device_name().c_str(), P);
        for (int nt : {1, 3}) {
            std::vector<mtl::Dispatch> ds;
            window(P, nt, ds);
            const char* names[] = {"prep", "score", "topk", "attn", "merge"};
            double total = 0;
            for (int i = 0; i < 5; ++i) {
                std::vector<mtl::Dispatch> one(32, ds[i]);
                cx.warm(one, 0.05);
                std::vector<double> t;
                for (int k = 0; k < 7; ++k) t.push_back(cx.run(one) / 32);
                std::printf("  ntok %d %-6s %7.2f us\n", nt, names[i], median(t) * 1e6);
                total += median(t);
            }
            std::vector<double> t;
            for (int k = 0; k < 7; ++k) t.push_back(cx.run(ds));
            std::printf("  ntok %d window (5 dependent dispatches): %.2f us (sum of parts %.2f)\n", nt, median(t) * 1e6,
                        total * 1e6);
        }
        return 0;
    }
    const int NT = 3;
    std::vector<mtl::Dispatch> ds;
    window(P, NT, ds);
    // stage by stage, re-reading GPU state in between
    cx.run({ds[0]});
    int fails = 0;
    // ---- prep: qcur, qidx, the window cells' KV codes and the pooled keys vs CPU
    {
        Err eq, ei, ekv, ep;
        auto normrope = [&](const float* x, const float* g, int w, int absp, bool rope, std::vector<double>& out) {
            double ss = 0;
            for (int i = 0; i < w; ++i) ss += (double)x[i] * x[i];
            const double rs = 1.0 / std::sqrt(ss / w + 1e-6);
            out.assign(w, 0.0);
            for (int i = 0; i < w; ++i) out[i] = rs * x[i] * g[i];
            if (rope)
                for (int i = 0; i < 32; ++i) {
                    const int pp = mrope ? mtab.as<int>()[absp * 3 + i % 3] : absp;
                    const double c = cs.as<float>()[(pp * 32 + i) * 2], s = cs.as<float>()[(pp * 32 + i) * 2 + 1];
                    const double a = out[i], b = out[i + 32];
                    out[i] = a * c - b * s;
                    out[i + 32] = a * s + b * c;
                }
        };
        std::vector<double> o;
        for (int t = 0; t < NT; ++t) {
            const int cell = P + t;
            for (int h = 0; h < QH; ++h) {
                normrope(qfull.as<float>() + ((size_t)cell * QH + h) * 2 * QD, qn.as<float>(), QD, cell, true, o);
                for (int i = 0; i < QD; ++i) eq.add(qcur.as<float>()[(t * QH + h) * QD + i], o[i]);
            }
            for (int h = 0; h < IH; ++h) {
                normrope(iq.as<float>() + ((size_t)cell * IH + h) * ID, iqn.as<float>(), ID, cell, true, o);
                for (int i = 0; i < ID; ++i) ei.add(qidx.as<float>()[(t * IH + h) * ID + i], o[i]);
            }
            for (int h = 0; h < QKV; ++h) {
                normrope(kc.as<float>() + ((size_t)cell * QKV + h) * QD, kn.as<float>(), QD, cell, true, o);
                for (int i = 0; i < QD; ++i) {
                    const size_t row = (size_t)cell * QKV + h;
                    const double deq = (double)(int8_t)kq.as<int8_t>()[row * QD + i] * h2f(ksc.as<uint16_t>()[row * 4 + i / 64]);
                    ekv.add(deq, o[i]);
                }
            }
        }
        // pooled keys of the last few completed blocks
        const int nb = (P + NT) / 4;
        for (int b = std::max(0, nb - 8); b < nb; ++b) {
            std::vector<float> m(ID);
            for (int i = 0; i < ID; ++i)
                m[i] = (((ik.as<float>()[(4 * b) * ID + i] + ik.as<float>()[(4 * b + 1) * ID + i]) +
                         ik.as<float>()[(4 * b + 2) * ID + i]) + ik.as<float>()[(4 * b + 3) * ID + i]) * 0.25f;
            normrope(m.data(), ikn.as<float>(), ID, 4 * b, true, o);
            for (int i = 0; i < ID; ++i) ep.add(pooled.as<float>()[b * ID + i], o[i]);
        }
        // int8 quantization error is ~1/254 of max|x| per group: the KV check is a quantization tolerance
        const bool ok = eq.l2() < 1e-5 && ei.l2() < 1e-5 && ep.l2() < 1e-5 && ekv.l2() < 6e-3;
        fails += !ok;
        std::printf("prep: q rel L2 %.2e, indexer q %.2e, pooled keys %.2e, K int8 (vs float) %.2e  %s\n", eq.l2(),
                    ei.l2(), ep.l2(), ekv.l2(), ok ? "PASS" : "FAIL");
    }
    // ---- score
    cx.run({ds[1]});
    {
        Err es;
        for (int t = 0; t < NT; ++t) {
            const int n_kv = P + t + 1, n_bid = n_kv / 4;
            for (int b = 0; b <= n_bid; ++b) {
                const float* key = b == n_bid ? dead.as<float>() : pooled.as<float>() + b * ID;
                double s = 0;
                for (int h = 0; h < IH; ++h) {
                    double d = 0;
                    for (int i = 0; i < ID; ++i) d += (double)qidx.as<float>()[(t * IH + h) * ID + i] * key[i];
                    s += std::max(d, 0.0);
                }
                if (b == n_bid && (n_kv & 3)) {
                    // float(s + 1e9) keeps no bits of s: only check that the bias is there
                    if (!(scores.as<float>()[t * MB + b] >= 1e9f)) es.add(1.0, 0.0);
                } else {
                    es.add(scores.as<float>()[t * MB + b], s);
                }
            }
        }
        const bool ok = es.l2() < 1e-5;
        fails += !ok;
        std::printf("score: rel L2 %.2e (tail +1e9 checked separately)  %s\n", es.l2(), ok ? "PASS" : "FAIL");
    }
    // ---- top-k (exact, from the GPU's scores)
    cx.run({ds[2]});
    {
        int bad = 0;
        for (int t = 0; t < NT; ++t) {
            const int n_kv = P + t + 1, n_bid = n_kv / 4, width = std::min(n_kv, BUDGET);
            std::vector<int> cells(n_kv);
            for (int c = 0; c < n_kv; ++c) cells[c] = c;
            if (n_kv > BUDGET) {
                const float* sc = scores.as<float>() + t * MB;
                std::stable_sort(cells.begin(), cells.end(), [&](int x, int y) { return sc[x / 4] > sc[y / 4]; });
                cells.resize(width);
                std::sort(cells.begin(), cells.end());
            }
            (void)n_bid;
            for (int i = 0; i < width; ++i) bad += ids.as<int>()[t * BUDGET + i] != cells[i];
        }
        fails += bad != 0;
        std::printf("top-k: %d of %d ids differ (context %d: selection %s)  %s\n", bad, NT * std::min(P + 1, BUDGET), P,
                    P + 1 > BUDGET ? "active" : "identity", bad ? "FAIL" : "PASS");
    }
    // ---- attention + merge (from the GPU's qcur, ids and KV codes)
    cx.run({ds[3], ds[4]});
    {
        Err eo;
        for (int t = 0; t < NT; ++t) {
            const int n_kv = P + t + 1, width = std::min(n_kv, BUDGET);
            for (int h = 0; h < QH; ++h) {
                const int g = h / (QH / QKV);
                std::vector<double> s(width);
                double mx = -1e300;
                for (int c = 0; c < width; ++c) {
                    const size_t row = (size_t)ids.as<int>()[t * BUDGET + c] * QKV + g;
                    double d = 0;
                    for (int i = 0; i < QD; ++i)
                        d += (double)qcur.as<float>()[(t * QH + h) * QD + i] * (int8_t)kq.as<int8_t>()[row * QD + i] *
                             h2f(ksc.as<uint16_t>()[row * 4 + i / 64]);
                    s[c] = d / 16.0;
                    mx = std::max(mx, s[c]);
                }
                double l = 0;
                for (int c = 0; c < width; ++c) l += (s[c] = std::exp(s[c] - mx));
                for (int i = 0; i < QD; ++i) {
                    double o = 0;
                    for (int c = 0; c < width; ++c) {
                        const size_t row = (size_t)ids.as<int>()[t * BUDGET + c] * QKV + g;
                        o += s[c] * (int8_t)vq.as<int8_t>()[row * QD + i] * h2f(vsc.as<uint16_t>()[row * 4 + i / 64]);
                    }
                    o /= l;
                    const double gate = qfull.as<float>()[((size_t)(P + t) * QH + h) * 2 * QD + QD + i];
                    eo.add(xp.as<float>()[t * QH * QD + h * QD + i], o / (1.0 + std::exp(-gate)));
                }
            }
        }
        const bool ok = eo.l2() < 1e-5;
        fails += !ok;
        std::printf("attention + gate: rel L2 %.2e  %s\n", eo.l2(), ok ? "PASS" : "FAIL");
    }
    std::printf("%s\n", fails ? "SOME FAILED" : "ALL PASS");
    return fails ? 1 : 0;
}