Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
14.9 KB
Raw Blame
// metal/tests/moe_mm_parity.cpp - the prompt-chunk routed experts (moe_mm.metal): plan, gate/up GEMM, down GEMM,
// combine, against a CPU reference that rounds the activations and weights to fp16 as the kernels do (the matrix
// units' operands) - and against the exact float reference for the size of that rounding; then the speed.
//
//   moe_mm_parity [--gu Q2_0] [--dn Q2_0] [--T 256] [--tile 32,64,64,4]          parity (64 experts)
//   moe_mm_parity --bench [--gu ..] [--dn ..] [--T 2048] [--tile ..]            per-layer time, 512 experts
#include "coop_probe.h"
#include "mtl_rt.h"
#include "ref_quant.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <set>
#include <string>
#include <vector>

namespace {
constexpr int N_EMBD = 2560, N_FF = 640, TOPK = 10;
struct MmPlanArgs { uint32_t n_ent, topk, n_expert, tm, n_blocks; };
struct MmArgs { uint64_t gate_off, up_off, down_off, gu_stride, d_stride; uint32_t gu_row, d_row, n_embd, n_ff; };
double median(std::vector<double> v) { std::sort(v.begin(), v.end()); return v[v.size() / 2]; }
int type_of(const std::string& s) {
    for (int t : {ref::TQ2_T, ref::TQK6, ref::TQK7, ref::Q2_0, ref::IQ3_XXS, ref::IQ2_XXS, ref::IQ2_XS, ref::IQ2_S, ref::IQ3_S, ref::IQ4_NL, ref::IQ4_XS, ref::Q4_K, ref::Q5_K, ref::Q6_K, ref::Q8_0, ref::Q5_1})
        if (s == ref::type_name(t)) return t;
    std::exit(2);
}
float h16(float v) { return ref::f16_to_f32(ref::f32_to_f16(v)); }
}  // namespace

int main(int argc, char** argv) {
    int gu = ref::Q2_0, dn = ref::Q2_0, T = 256, TM = 16, TN = 64, TK = 64, SG = 2, DTN = -1, DTK = -1, DSG = -1;
    bool bench = false, v2 = false, v3 = false, v4 = false, v5 = false;
    int MS = 1;
    for (int i = 1; i < argc; ++i) {
        std::string a = argv[i];
        if (a == "--gu") gu = type_of(argv[++i]);
        else if (a == "--dn") dn = type_of(argv[++i]);
        else if (a == "--T") T = std::atoi(argv[++i]);
        else if (a == "--tile") std::sscanf(argv[++i], "%d,%d,%d,%d", &TM, &TN, &TK, &SG);
        else if (a == "--dtile") std::sscanf(argv[++i], "%d,%d,%d", &DTN, &DTK, &DSG);   // the down kernel's TN,TK,SG
        else if (a == "--bench") bench = true;
        else if (a == "--v2") v2 = true;
        else if (a == "--v3") v2 = v3 = true;   // v2's gather + the register-operand GEMMs (TN per simdgroup)
        else if (a == "--v4") { v2 = v3 = v4 = true; MS = std::atoi(argv[++i]); }   // mm_gu4 with MS 16-row subtiles
        else if (a == "--v5") { v2 = v3 = true; v5 = true; }   // mm_gu5 (weight blocks staged in threadgroup memory)
    }
    if (bench && T == 256) T = 2048;
    if (v4) TM = 16 * MS;   // the plan's tile height; mm_gu4's name carries MS instead
    mtl::Context cx;
    const int lib = cx.compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal"});
    const int NE = bench ? 512 : 64, LAYERS = bench ? 4 : 1;
    const size_t gu_row = ref::row_bytes(gu, N_EMBD), d_row = ref::row_bytes(dn, N_FF);
    const size_t up_off = (size_t)N_FF * gu_row, down_off = 2 * up_off, blob = down_off + (size_t)N_EMBD * d_row;
    std::printf("device %s; %s/%s, T %d, %d experts, tile TM %d TN %d TK %d SG %d\n", cx.device_name().c_str(),
                ref::type_name(gu), ref::type_name(dn), T, NE, TM, TN, TK, SG);
    mtl::Buffer W = cx.alloc(blob * NE * LAYERS);
    {
        std::vector<uint8_t> tmp(blob * 16);
        for (int e = 0; e < 16; ++e) {
            ref::random_blocks(gu, tmp.data() + e * blob, 2 * N_FF, N_EMBD, 100 + e, 0.02f);
            ref::random_blocks(dn, tmp.data() + e * blob + down_off, N_EMBD, N_FF, 200 + e, 0.02f);
        }
        for (size_t e = 0; e < (size_t)NE * LAYERS; ++e) std::memcpy(W.as<uint8_t>() + e * blob, tmp.data() + (e % 16) * blob, blob);
    }
    const int nent = T * TOPK, nblk = (nent + 1023) / 1024;
    const int maxrows = nent + NE * TM;
    mtl::Buffer x = cx.alloc(4 * (size_t)T * N_EMBD), ids = cx.alloc(4 * nent), w = cx.alloc(4 * nent),
                bcnt = cx.alloc(4 * (size_t)nblk * NE), boff = cx.alloc(4 * (size_t)nblk * NE), eoff = cx.alloc(4 * NE),
                tiles = cx.alloc(8 * (size_t)(maxrows / TM + 1)), meta = cx.alloc(64), stok = cx.alloc(4 * (size_t)maxrows),
                sw = cx.alloc(4 * (size_t)maxrows), pos = cx.alloc(4 * nent), H = cx.alloc(4 * (size_t)maxrows * N_FF),
                YE = cx.alloc(4 * (size_t)maxrows * N_EMBD), y = cx.alloc(4 * (size_t)T * N_EMBD),
                XS = cx.alloc(2 * (size_t)maxrows * N_EMBD), HS = cx.alloc(2 * (size_t)maxrows * N_FF);
    ref::Rng r(7);
    for (size_t i = 0; i < (size_t)T * N_EMBD; ++i) x.as<float>()[i] = r.sym();
    for (int t = 0; t < T; ++t) {
        std::set<int> used;
        float s = 0;
        for (int k = 0; k < TOPK; ++k) {
            int e;
            do e = (int)(r.next() % NE); while (used.count(e));
            used.insert(e);
            ids.as<int>()[t * TOPK + k] = e;
            s += (w.as<float>()[t * TOPK + k] = 0.2f + r.uni());
        }
        for (int k = 0; k < TOPK; ++k) w.as<float>()[t * TOPK + k] /= s;
    }
    mtl::Pipeline phist = cx.pipeline(lib, "mm_hist"), pscan = cx.pipeline(lib, "mm_scan"), pplace = cx.pipeline(lib, "mm_place"),
                  pclear = cx.pipeline(lib, "mm_clear"), pcomb = cx.pipeline(lib, "mm_combine");
    if (DTN < 0) { DTN = TN; DTK = TK; DSG = SG; }
    if (dn == ref::IQ4_NL && DTK < 32) DTK = 32;
    const std::string tl = std::to_string(TM) + "_" + std::to_string(TN) + "_" + std::to_string(TK) + "_" + std::to_string(SG);
    const std::string dtl = std::to_string(TM) + "_" + std::to_string(DTN) + "_" + std::to_string(DTK) + "_" + std::to_string(DSG);
    mtl::Constants probe;
    probe.i(5, std::getenv("MM_PROBE") ? std::atoi(std::getenv("MM_PROBE")) : 0);
    const bool k4 = coop_k4(cx);
    if (v3) std::printf("right-input layout: %s\n", k4 ? "runs of 4 k" : "other (per-element decode)");
    mtl::Pipeline pgu = v5 ? cx.pipeline(lib, "mm_gu5_" + std::to_string(gu) + "_" + std::to_string(TN) + "_" + std::to_string(SG),
                                         mtl::Constants().b(7, k4))
                      : v4 ? cx.pipeline(lib, "mm_gu4_" + std::to_string(gu) + "_" + std::to_string(MS) + tl.substr(tl.find('_')),
                                         mtl::Constants().b(7, k4).i(5, std::getenv("MM_PROBE") ? std::atoi(std::getenv("MM_PROBE")) : 0))
                      : v3 ? cx.pipeline(lib, "mm_gu3_" + std::to_string(gu) + "_" + tl, mtl::Constants().b(7, k4).i(5, std::getenv("MM_PROBE") ? std::atoi(std::getenv("MM_PROBE")) : 0).b(23, std::getenv("QUANT_CTAB") != nullptr))
                           : cx.pipeline(lib, (v2 ? "mm_gu2_" : "mm_gu_") + std::to_string(gu) + "_" + tl, probe),
                  pdn = v3 ? cx.pipeline(lib, "mm_down3_" + std::to_string(dn) + "_" + dtl, mtl::Constants().b(7, k4))
                           : cx.pipeline(lib, (v2 ? "mm_down2_" : "mm_down_") + std::to_string(dn) + "_" + (v2 ? dtl : tl)),
                  pgat = cx.pipeline(lib, "mm_gather");
    MmPlanArgs pa{(uint32_t)nent, TOPK, (uint32_t)NE, (uint32_t)TM, (uint32_t)nblk};
    auto encode = [&](int layer, std::vector<mtl::Dispatch>& ds, bool plan, bool gemm) {
        if (plan) {
            const uint32_t nclr = maxrows;
            mtl::Dispatch c; c.p = &pclear; c.args = {mtl::Arg::B(stok), mtl::Arg::V(nclr)}; c.grid[0] = (maxrows + 255) / 256; c.tg[0] = 256; ds.push_back(c);
            mtl::Dispatch h; h.p = &phist; h.args = {mtl::Arg::B(ids), mtl::Arg::B(bcnt), mtl::Arg::V(pa)}; h.grid[0] = nblk; h.tg[0] = 1024; ds.push_back(h);
            meta.as<uint32_t>()[2] = N_FF / TN;
            mtl::Dispatch sc; sc.p = &pscan; sc.args = {mtl::Arg::B(bcnt), mtl::Arg::B(boff), mtl::Arg::B(eoff), mtl::Arg::B(tiles), mtl::Arg::B(meta), mtl::Arg::V(pa)}; sc.tg[0] = NE; ds.push_back(sc);
            mtl::Dispatch pl; pl.p = &pplace; pl.args = {mtl::Arg::B(ids), mtl::Arg::B(w), mtl::Arg::B(boff), mtl::Arg::B(eoff), mtl::Arg::B(stok), mtl::Arg::B(sw), mtl::Arg::B(pos), mtl::Arg::V(pa)}; pl.grid[0] = nblk; pl.tg[0] = 1024; ds.push_back(pl);
        }
        if (gemm) {
            MmArgs ma{(uint64_t)layer * NE * blob, (uint64_t)layer * NE * blob + up_off, (uint64_t)layer * NE * blob + down_off,
                      blob, blob, (uint32_t)gu_row, (uint32_t)d_row, N_EMBD, N_FF};
            if (v2 || v3) {   // v3 is v2's gather plus the register-operand GEMMs
                const uint32_t ne = N_EMBD;
                mtl::Dispatch ga; ga.p = &pgat; ga.args = {mtl::Arg::B(x), mtl::Arg::B(stok), mtl::Arg::B(XS), mtl::Arg::V(ne)};
                ga.grid[0] = meta.as<uint32_t>()[1]; ga.tg[0] = 256; ds.push_back(ga);
            }
            mtl::Dispatch g; g.p = &pgu;
            if (v2 || v3) g.args = {mtl::Arg::B(W), mtl::Arg::B(XS), mtl::Arg::B(HS), mtl::Arg::B(tiles), mtl::Arg::V(ma)};
            else g.args = {mtl::Arg::B(W), mtl::Arg::B(x), mtl::Arg::B(H), mtl::Arg::B(stok), mtl::Arg::B(tiles), mtl::Arg::V(ma)};
            g.grid[0] = N_FF / (v3 ? TN * SG : TN); g.grid[1] = meta.as<uint32_t>()[0]; g.tg[0] = 32 * SG; ds.push_back(g);
            mtl::Dispatch d; d.p = &pdn; d.args = {mtl::Arg::B(W), mtl::Arg::B(v2 || v3 ? HS : H), mtl::Arg::B(YE), mtl::Arg::B(tiles), mtl::Arg::V(ma)};
            d.grid[0] = N_EMBD / (v3 ? DTN * DSG : v2 ? DTN : TN); d.grid[1] = meta.as<uint32_t>()[0]; d.tg[0] = 32 * ((v2 || v3) ? DSG : SG); ds.push_back(d);
            const uint32_t ca[4] = {(uint32_t)T, N_EMBD, TOPK, 0};
            mtl::Dispatch cb; cb.p = &pcomb; cb.args = {mtl::Arg::B(YE), mtl::Arg::B(pos), mtl::Arg::B(sw), mtl::Arg::B(y), mtl::Arg::V(ca)};
            cb.grid[0] = (T * N_EMBD + 255) / 256; cb.tg[0] = 256; ds.push_back(cb);
        }
    };
    {
        std::vector<mtl::Dispatch> ds;
        encode(0, ds, true, false);
        cx.run(ds);   // the plan first: the GEMMs' grid is the tile count
    }
    const uint32_t ntiles = meta.as<uint32_t>()[0];
    std::printf("plan: %u tiles of %d, %u padded rows for %d entries\n", ntiles, TM, meta.as<uint32_t>()[1], nent);
    if (!bench) {
        // the plan: every entry placed once, rows of an expert contiguous and in entry order
        bool ok_plan = true;
        std::vector<int> seen(maxrows, 0);
        for (int i = 0; i < nent; ++i) {
            const int row = pos.as<int>()[i];
            if (row < 0 || row >= (int)meta.as<uint32_t>()[1] || stok.as<int>()[row] != i / TOPK) ok_plan = false;
            else seen[row]++;
        }
        for (int i = 0; i < maxrows; ++i) if (seen[i] > 1) ok_plan = false;
        std::vector<mtl::Dispatch> ds;
        encode(0, ds, false, true);
        cx.run(ds);
        // reference: fp16 operands (x, weights, h), fp32+ accumulation; and the exact one
        double se16 = 0, se = 0, sr = 0;
        std::vector<float> wg((size_t)N_FF * N_EMBD), wu(wg.size()), wd((size_t)N_EMBD * N_FF);
        std::vector<double> yr16((size_t)T * N_EMBD, 0.0), yr((size_t)T * N_EMBD, 0.0);
        for (int e = 0; e < NE; ++e) {
            const uint8_t* b = W.as<uint8_t>() + e * blob;
            bool any = false;
            for (int i = 0; i < nent && !any; ++i) any = ids.as<int>()[i] == e;
            if (!any) continue;
            ref::dequant(gu, b, wg.data(), wg.size());
            ref::dequant(gu, b + up_off, wu.data(), wu.size());
            ref::dequant(dn, b + down_off, wd.data(), wd.size());
            for (int i = 0; i < nent; ++i) {
                if (ids.as<int>()[i] != e) continue;
                const int t = i / TOPK;
                const float* xt = x.as<float>() + (size_t)t * N_EMBD;
                std::vector<double> h16v(N_FF), hv(N_FF);
                for (int f = 0; f < N_FF; ++f) {
                    double g = 0, u = 0, g16 = 0, u16 = 0;
                    for (int k = 0; k < N_EMBD; ++k) {
                        const float xk = xt[k], xh = h16(xk);
                        g += (double)wg[(size_t)f * N_EMBD + k] * xk;
                        u += (double)wu[(size_t)f * N_EMBD + k] * xk;
                        g16 += (double)h16(wg[(size_t)f * N_EMBD + k]) * xh;
                        u16 += (double)h16(wu[(size_t)f * N_EMBD + k]) * xh;
                    }
                    hv[f] = g / (1 + std::exp(-g)) * u;
                    h16v[f] = h16((float)(g16 / (1 + std::exp(-g16)) * u16));
                }
                const double wt = w.as<float>()[i];
                for (int d = 0; d < N_EMBD; ++d) {
                    double s = 0, s16 = 0;
                    for (int f = 0; f < N_FF; ++f) {
                        s += (double)wd[(size_t)d * N_FF + f] * hv[f];
                        s16 += (double)h16(wd[(size_t)d * N_FF + f]) * h16v[f];
                    }
                    yr[(size_t)t * N_EMBD + d] += wt * s;
                    yr16[(size_t)t * N_EMBD + d] += wt * s16;
                }
            }
        }
        for (size_t i = 0; i < yr.size(); ++i) {
            const double g = y.as<float>()[i];
            se16 += (g - yr16[i]) * (g - yr16[i]);
            se += (g - yr[i]) * (g - yr[i]);
            sr += yr[i] * yr[i];
        }
        const double rel16 = std::sqrt(se16 / sr), rel = std::sqrt(se / sr);
        const bool ok = ok_plan && rel16 < 2e-4;
        std::printf("parity: plan %s; y vs fp16-operand reference rel L2 %.2e, vs exact float reference %.2e  %s\n",
                    ok_plan ? "ok" : "WRONG", rel16, rel, ok ? "PASS" : "FAIL");
        return ok ? 0 : 1;
    }
    // bench: the plan, the two GEMMs and the combine, LAYERS distinct weight sets rotated
    double flops = 0;
    {
        const double rows = meta.as<uint32_t>()[1];
        flops = 2.0 * nent * (2.0 * N_FF * N_EMBD + (double)N_EMBD * N_FF);
        std::printf("useful GFLOP per layer %.1f (padded rows %.0f = %.2fx entries)\n", flops / 1e9, rows, rows / nent);
    }
    auto timeit = [&](bool plan, bool gemm, int part) {
        std::vector<mtl::Dispatch> ds;
        for (int l = 0; l < 8; ++l) {
            std::vector<mtl::Dispatch> one;
            encode(l % LAYERS, one, plan, gemm);
            if (part >= 0 && gemm) { ds.push_back(one[part]); continue; }
            for (auto& d : one) ds.push_back(d);
        }
        cx.warm(ds, 0.3);
        std::vector<double> t;
        for (int i = 0; i < 5; ++i) t.push_back(cx.run(ds) / 8);
        return median(t);
    };
    const int o = v2 ? 1 : 0;
    const double tp = timeit(true, false, -1), tg = timeit(false, true, o), td = timeit(false, true, o + 1),
                 tc = timeit(false, true, o + 2), ta = timeit(false, true, -1);
    if (v2) std::printf("gather %.1f us\n", timeit(false, true, 0) * 1e6);
    const double fgu = 2.0 * nent * 2.0 * N_FF * N_EMBD, fdn = 2.0 * nent * N_EMBD * N_FF;
    std::printf("per layer: plan %.1f us, gate/up %.1f us (%.1f TFLOP/s useful), down %.1f us (%.1f TFLOP/s), combine %.1f us;"
                " GEMMs + combine %.1f us = %.1f TFLOP/s; %.1f us per token per layer -> 48 layers: %.0f tok/s (experts only)\n",
                tp * 1e6, tg * 1e6, fgu / tg / 1e12, td * 1e6, fdn / td / 1e12, tc * 1e6, ta * 1e6, flops / ta / 1e12,
                (ta + tp) / T * 1e6, T / (48 * (ta + tp)));
    return 0;
}