Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
12.7 KB
Raw Blame
// metal/tests/prompt_parity.cpp - the prompt-chunk glue (prompt.metal) against CPU references:
//   route_topk           softmax + top 10 + renormalised weights (moe_route's rule) for 300 tokens, a third of them
//                        with exact ties
//   the HC read as GEMMs gr_norm -> act_f16 -> [w_down; w_inject] GEMM -> gr_epi_down -> w_up GEMM -> gr_epi_up, on
//                        random BF16 weights, vs the exact formula (fp64) - the fp16 operands set the tolerance
//   swiglu_h, moe_out    elementwise
#include "mtl_rt.h"
#include "ref_quant.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <vector>

namespace {
constexpr int N = 2560, HC = 4, LR = 320;
struct GrArgs { uint32_t n_embd, hc, hc_lr, n_tok, apply; float eps; };
struct EpiArgs { uint32_t n_tok, n_embd, hc, hc_lr; };
struct DqArgs { uint32_t n_in, n_out, row_bytes, perm; };
struct GemmArgs { int M, N, K, pad; };
double sig(double v) { return 1.0 / (1.0 + std::exp(-v)); }
}  // namespace

int main() {
    mtl::Context cx;
    const int lib = cx.compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal", "gr.metal", "prompt.metal"});
    std::printf("device %s\n", cx.device_name().c_str());
    int fails = 0;
    // ---- route_topk
    {
        const int T = 300;
        mtl::Pipeline p = cx.pipeline(lib, "route_topk");
        mtl::Buffer lg = cx.alloc(4 * T * 512), ids = cx.alloc(4 * T * 10), w = cx.alloc(4 * T * 10);
        ref::Rng r(9);
        for (int t = 0; t < T; ++t)
            for (int e = 0; e < 512; ++e) {
                float v = r.sym() * 4.0f;
                if (t % 3 == 0) v = std::round(v * 4.0f) / 4.0f;
                lg.as<float>()[t * 512 + e] = v;
            }
        const uint32_t nt = T;
        mtl::Dispatch d;
        d.p = &p;
        d.args = {mtl::Arg::B(lg), mtl::Arg::B(ids), mtl::Arg::B(w), mtl::Arg::V(nt)};
        d.grid[0] = (T + 7) / 8;
        d.tg[0] = 256;
        cx.run({d});
        int bad = 0;
        double worst = 0;
        for (int t = 0; t < T; ++t) {
            const float* l = lg.as<float>() + t * 512;
            float mx = -INFINITY;
            for (int e = 0; e < 512; ++e) mx = std::max(mx, l[e]);
            std::vector<double> pr(512);
            double s = 0;
            for (int e = 0; e < 512; ++e) s += (pr[e] = std::exp((double)l[e] - mx));
            std::vector<int> order(512);
            for (int e = 0; e < 512; ++e) order[e] = e;
            std::stable_sort(order.begin(), order.end(), [&](int x, int y) { return l[x] > l[y]; });
            double ss = 0;
            for (int k = 0; k < 10; ++k) ss += pr[order[k]] / s;
            for (int k = 0; k < 10; ++k) {
                if (ids.as<int>()[t * 10 + k] != order[k]) { ++bad; break; }
                const double ref = pr[order[k]] / s / std::max(ss, 6.103515625e-5);
                worst = std::max(worst, std::fabs(w.as<float>()[t * 10 + k] - ref) / ref);
            }
        }
        const bool ok = bad == 0 && worst < 2e-6;
        fails += !ok;
        std::printf("route_topk, %d tokens: id mismatches %d, worst weight rel err %.2e  %s\n", T, bad, worst, ok ? "PASS" : "FAIL");
    }
    // ---- the HC read as GEMMs
    for (int apply = 0; apply < 2; ++apply) {
        const int T = 70, NIN = HC * N, NR = LR + HC;
        mtl::Buffer R = cx.alloc(4 * (size_t)T * NIN), bo = cx.alloc(4 * (size_t)T * N), inj = cx.alloc(4 * T * HC),
                    wn = cx.alloc(4 * NIN), wdn = cx.alloc(2 * (size_t)LR * NIN), win = cx.alloc(2 * (size_t)HC * NIN),
                    wup = cx.alloc(2 * (size_t)NIN * LR), xn = cx.alloc(4 * (size_t)T * NIN), xnh = cx.alloc(2 * (size_t)T * NIN),
                    WD = cx.alloc(2 * (size_t)NR * NIN), WU = cx.alloc(2 * (size_t)NIN * LR), V = cx.alloc(4 * (size_t)T * NR),
                    loh = cx.alloc(2 * (size_t)T * LR), injo = cx.alloc(4 * T * HC), G = cx.alloc(4 * (size_t)T * NIN),
                    mixed = cx.alloc(4 * (size_t)T * N), mixh = cx.alloc(2 * (size_t)T * N);
        ref::Rng r(11 + apply);
        std::vector<float> R0((size_t)T * NIN);
        for (auto& v : R0) v = r.sym();
        std::copy(R0.begin(), R0.end(), R.as<float>());
        for (size_t i = 0; i < (size_t)T * N; ++i) bo.as<float>()[i] = r.sym();
        for (int i = 0; i < T * HC; ++i) inj.as<float>()[i] = r.sym() * 3.0f;
        for (int i = 0; i < NIN; ++i) wn.as<float>()[i] = 0.5f + r.uni();
        for (size_t i = 0; i < (size_t)LR * NIN; ++i) wdn.as<uint16_t>()[i] = ref::f32_to_bf16(r.sym() * 0.05f);
        for (size_t i = 0; i < (size_t)HC * NIN; ++i) win.as<uint16_t>()[i] = ref::f32_to_bf16(r.sym() * 0.05f);
        for (size_t i = 0; i < (size_t)NIN * LR; ++i) wup.as<uint16_t>()[i] = ref::f32_to_bf16(r.sym() * 0.2f);
        mtl::Pipeline pn = cx.pipeline(lib, "gr_norm"), pa = cx.pipeline(lib, "act_f16"), pdq = cx.pipeline(lib, "dq_f16_30"),
                      pg = cx.pipeline(lib, "gemm_f16_64_64_4"), ped = cx.pipeline(lib, "gr_epi_down"), peu = cx.pipeline(lib, "gr_epi_up");
        std::vector<mtl::Dispatch> ds;
        GrArgs ga{N, HC, LR, (uint32_t)T, (uint32_t)apply, 1e-6f};
        EpiArgs ea{(uint32_t)T, N, HC, LR};
        auto D = [&](const mtl::Pipeline& p, std::vector<mtl::Arg> a, uint32_t gx, uint32_t gy, uint32_t tg) {
            mtl::Dispatch d; d.p = &p; d.args = std::move(a); d.grid[0] = gx; d.grid[1] = gy; d.tg[0] = tg; ds.push_back(d);
        };
        auto dq = [&](const mtl::Buffer& w, const mtl::Buffer& out, size_t ooff, int n_in, int n_out) {
            DqArgs a{(uint32_t)n_in, (uint32_t)n_out, (uint32_t)(2 * n_in)};
            D(pdq, {mtl::Arg::B(w), mtl::Arg::B(out, ooff), mtl::Arg::V(a)}, (n_in / 16 + 63) / 64, n_out, 64);
        };
        auto gemm = [&](const mtl::Buffer& A, const mtl::Buffer& B, const mtl::Buffer& Y, int M, int Nn, int K) {
            GemmArgs g{M, Nn, K, 0};
            D(pg, {mtl::Arg::B(A), mtl::Arg::B(B), mtl::Arg::B(Y), mtl::Arg::V(g)}, (Nn + 63) / 64, (M + 63) / 64, 128);
        };
        const uint32_t nx = T * NIN;
        dq(wdn, WD, 0, NIN, LR);
        dq(win, WD, (size_t)2 * LR * NIN, NIN, HC);
        dq(wup, WU, 0, LR, NIN);
        D(pn, {mtl::Arg::B(R), mtl::Arg::B(bo), mtl::Arg::B(inj), mtl::Arg::B(wn), mtl::Arg::B(xn), mtl::Arg::V(ga)}, HC, T, 256);
        D(pa, {mtl::Arg::B(xn), mtl::Arg::B(xnh), mtl::Arg::V(nx)}, (nx + 255) / 256, 1, 256);
        gemm(xnh, WD, V, T, NR, NIN);
        D(ped, {mtl::Arg::B(V), mtl::Arg::B(loh), mtl::Arg::B(injo), mtl::Arg::V(ea)}, (T * NR + 255) / 256, 1, 256);
        gemm(loh, WU, G, T, NIN, LR);
        D(peu, {mtl::Arg::B(G), mtl::Arg::B(xn), mtl::Arg::B(mixed), mtl::Arg::B(mixh), mtl::Arg::V(ea)}, (T * N + 255) / 256, 1, 256);
        cx.run(ds);
        // the fused read: gr_norm_h (fp16 xn), the down GEMM, gr_epi_down, the up GEMM with the epilogue (w_up rows
        // interleaved by dq_f16's perm)
        mtl::Buffer mixed2 = cx.alloc(4 * (size_t)T * N), mixh2 = cx.alloc(2 * (size_t)T * N), WU2 = cx.alloc(2 * (size_t)NIN * LR),
                    xnh2 = cx.alloc(2 * (size_t)T * NIN), V2 = cx.alloc(4 * (size_t)T * NR), loh2 = cx.alloc(2 * (size_t)T * LR),
                    injo2 = cx.alloc(4 * T * HC);
        std::copy(R0.begin(), R0.end(), R.as<float>());
        {
            mtl::Pipeline pnh = cx.pipeline(lib, "gr_norm_h"), pup = cx.pipeline(lib, "gemm_hc_up_64_64_4");
            ds.clear();
            DqArgs da{(uint32_t)LR, (uint32_t)NIN, (uint32_t)(2 * LR), (uint32_t)HC};
            D(pdq, {mtl::Arg::B(wup), mtl::Arg::B(WU2), mtl::Arg::V(da)}, (LR / 16 + 63) / 64, NIN, 64);
            D(pnh, {mtl::Arg::B(R), mtl::Arg::B(bo), mtl::Arg::B(inj), mtl::Arg::B(wn), mtl::Arg::B(xnh2), mtl::Arg::V(ga)}, HC, T, 256);
            gemm(xnh2, WD, V2, T, NR, NIN);
            D(ped, {mtl::Arg::B(V2), mtl::Arg::B(loh2), mtl::Arg::B(injo2), mtl::Arg::V(ea)}, (T * NR + 255) / 256, 1, 256);
            GemmArgs g{T, NIN, LR, 0};
            D(pup, {mtl::Arg::B(loh2), mtl::Arg::B(WU2), mtl::Arg::B(xnh2), mtl::Arg::B(mixed2), mtl::Arg::B(mixh2), mtl::Arg::V(g),
                    mtl::Arg::V(ea)}, NIN / 64, (T + 63) / 64, 128);
            cx.run(ds);
        }
        // reference
        double se = 0, sr = 0, sei = 0, sri = 0, ser = 0, srr = 0, se2 = 0, sh2 = 0;
        std::vector<double> x(NIN), lo(LR);
        for (int t = 0; t < T; ++t) {
            for (int c = 0; c < HC; ++c) {
                const double g = apply ? 2.0 * sig(inj.as<float>()[t * HC + c] / (double)HC) : 0.0;
                double ss = 0;
                for (int d = 0; d < N; ++d) {
                    const size_t i = (size_t)t * NIN + c * N + d;
                    const double v = R0[i] + (apply ? bo.as<float>()[(size_t)t * N + d] * g : 0.0);
                    x[c * N + d] = v;
                    ss += v * v;
                    ser += (R.as<float>()[i] - v) * (R.as<float>()[i] - v);
                    srr += v * v;
                }
                const double rs = 1.0 / std::sqrt(ss / N + 1e-6);
                for (int d = 0; d < N; ++d) x[c * N + d] *= wn.as<float>()[c * N + d] * rs;
            }
            for (int k = 0; k < NR; ++k) {
                const uint16_t* w = k < LR ? wdn.as<uint16_t>() + (size_t)k * NIN : win.as<uint16_t>() + (size_t)(k - LR) * NIN;
                double s = 0;
                for (int i = 0; i < NIN; ++i) s += ref::bf16_to_f32(w[i]) * x[i];
                if (k < LR) lo[k] = s / HC * sig(s / HC);
                else {
                    const double gv = injo.as<float>()[t * HC + k - LR];
                    sei += (gv - s) * (gv - s);
                    sri += s * s;
                }
            }
            for (int d = 0; d < N; ++d) {
                double m = 0;
                for (int c = 0; c < HC; ++c) {
                    const uint16_t* w = wup.as<uint16_t>() + (size_t)(c * N + d) * LR;
                    double s = 0;
                    for (int k = 0; k < LR; ++k) s += ref::bf16_to_f32(w[k]) * lo[k];
                    m += x[c * N + d] * sig(s);
                }
                m /= HC;
                const double gv = mixed.as<float>()[(size_t)t * N + d], g2 = mixed2.as<float>()[(size_t)t * N + d],
                             h2 = ref::f16_to_f32(mixh2.as<uint16_t>()[(size_t)t * N + d]);
                se += (gv - m) * (gv - m);
                se2 += (g2 - m) * (g2 - m);
                sh2 += (h2 - m) * (h2 - m);
                sr += m * m;
            }
        }
        const double rm = std::sqrt(se / sr), ri = std::sqrt(sei / sri), rr = std::sqrt(ser / srr);
        const double r2 = std::sqrt(se2 / sr), rh2 = std::sqrt(sh2 / sr);
        const bool ok = rm < 2e-3 && ri < 2e-3 && rr < 1e-6 && r2 < 2e-3 && rh2 < 2e-3;
        fails += !ok;
        std::printf("HC read as GEMMs, %d tokens, apply %d: mixed rel err %.2e, inject %.2e, residual %.2e; fused read "
                    "(fp16 xn, up GEMM epilogue): mixed %.2e, fp16 %.2e  %s\n", T, apply, rm, ri, rr, r2, rh2, ok ? "PASS" : "FAIL");
    }
    // ---- swiglu_h, moe_out
    {
        const int n = 5000, T = 7, NE = 640;
        mtl::Pipeline ps = cx.pipeline(lib, "swiglu_h"), pm = cx.pipeline(lib, "moe_out");
        mtl::Buffer g = cx.alloc(4 * n), u = cx.alloc(4 * n), h = cx.alloc(2 * n), yr = cx.alloc(4 * T * NE),
                    ys = cx.alloc(4 * T * NE), sg = cx.alloc(4 * T), bo = cx.alloc(4 * T * NE);
        ref::Rng r(5);
        for (int i = 0; i < n; ++i) { g.as<float>()[i] = r.sym() * 6; u.as<float>()[i] = r.sym(); }
        for (int i = 0; i < T * NE; ++i) { yr.as<float>()[i] = r.sym(); ys.as<float>()[i] = r.sym(); }
        for (int i = 0; i < T; ++i) sg.as<float>()[i] = r.sym() * 4;
        const uint32_t nn = n;
        const uint32_t a2[2] = {(uint32_t)T, (uint32_t)NE};
        mtl::Dispatch d1; d1.p = &ps; d1.args = {mtl::Arg::B(g), mtl::Arg::B(u), mtl::Arg::B(h), mtl::Arg::V(nn)};
        d1.grid[0] = (n + 255) / 256; d1.tg[0] = 256;
        mtl::Dispatch d2; d2.p = &pm; d2.args = {mtl::Arg::B(yr), mtl::Arg::B(ys), mtl::Arg::B(sg), mtl::Arg::B(bo), mtl::Arg::V(a2)};
        d2.grid[0] = (T * NE + 255) / 256; d2.tg[0] = 256;
        cx.run({d1, d2});
        double w1 = 0, w2 = 0;
        for (int i = 0; i < n; ++i) {
            const double gv = g.as<float>()[i], ref = gv * sig(gv) * u.as<float>()[i];
            w1 = std::max(w1, std::fabs(ref::f16_to_f32(h.as<uint16_t>()[i]) - ref) / (std::fabs(ref) + 1e-3));
        }
        for (int i = 0; i < T * NE; ++i) {
            const double ref = yr.as<float>()[i] + sig(sg.as<float>()[i / NE]) * ys.as<float>()[i];
            w2 = std::max(w2, std::fabs(bo.as<float>()[i] - ref));
        }
        const bool ok = w1 < 1e-3 && w2 < 1e-5;
        fails += !ok;
        std::printf("swiglu_h worst rel err %.2e, moe_out worst abs err %.2e  %s\n", w1, w2, ok ? "PASS" : "FAIL");
    }
    std::printf("%s\n", fails ? "SOME FAILED" : "ALL PASS");
    return fails ? 1 : 0;
}