Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
11.1 KB
Raw Blame
// metal/tests/gr_parity.cpp - the hyper-connection read (gr.metal) against a double-precision CPU reference
// (fused_gr.hpp's formulas), and its speed.
//
//   gr_parity [--ntok N]             parity for N = 1, 3, 8 (or N), with and without the folded write
//   gr_parity --bench [--ntok N]     per-kernel GPU time, 8 weight sets rotated (no cache reuse across calls)
#include "mtl_rt.h"
#include "ref_quant.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>

namespace {
constexpr int N = 2560, HC = 4, LR = 320, NIN = HC * N;
struct GrArgs { uint32_t n_embd, hc, hc_lr, n_tok, apply; float eps; };
double median(std::vector<double> v) { std::sort(v.begin(), v.end()); return v[v.size() / 2]; }
struct Err {
    double se = 0, sr = 0, mx = 0, mr = 0;
    void add(double g, double r) { se += (g - r) * (g - r); sr += r * r; mx = std::max(mx, std::fabs(g - r)); mr = std::max(mr, std::fabs(r)); }
    double l2() const { return sr > 0 ? std::sqrt(se / sr) : std::sqrt(se); }
};
}  // namespace

int main(int argc, char** argv) {
    bool bench = false;
    std::vector<int> ntoks = {1, 3, 8};
    for (int i = 1; i < argc; ++i) {
        std::string a = argv[i];
        if (a == "--bench") bench = true;
        else if (a == "--ntok") ntoks = {std::atoi(argv[++i])};
    }
    mtl::Context cx;
    const int lib = cx.compile({"quant.metal", "units.metal", "gr.metal"});
    std::printf("device %s\n", cx.device_name().c_str());
    const int SETS = bench ? 48 : 1;
    const bool q8 = std::getenv("GR_Q8") != nullptr;   // the Q8_0 form of the HC matrices (gr_down_q8_*, gr_up_q8)
    const int wty = q8 ? ref::Q8_0 : ref::BF16;
    // GR_INJ_F32 (with GR_Q8): the inject rows as F32 (the Unsloth files), read by gr_down_q8f_*
    const bool injf = q8 && std::getenv("GR_INJ_F32") != nullptr;
    const size_t wd_bytes = (size_t)LR * NIN * 2, wi_bytes = (size_t)HC * NIN * 4, wu_bytes = (size_t)NIN * LR * 2;
    const size_t set_bytes = wd_bytes + wi_bytes + wu_bytes + NIN * 4;
    mtl::Buffer W = cx.alloc(set_bytes * SETS);
    ref::Rng rng(3);
    for (int s = 0; s < SETS; ++s) {
        uint8_t* b = W.as<uint8_t>() + s * set_bytes;
        ref::random_blocks(ref::BF16, b, LR, NIN, 11 + s, 0.02f);
        ref::random_blocks(ref::BF16, b + wd_bytes, HC, NIN, 22 + s, 0.02f);
        ref::random_blocks(ref::BF16, b + wd_bytes + wi_bytes, NIN, LR, 33 + s, 0.1f);
        float* wn = (float*)(b + wd_bytes + wi_bytes + wu_bytes);
        for (int i = 0; i < NIN; ++i) wn[i] = 1.0f + 0.1f * rng.sym();
        if (q8) {
            // GR_Q8: the three matrices as Q8_0 (quantized from the BF16 values), each at its BF16 offset (it fits)
            const size_t offs[3] = {0, wd_bytes, wd_bytes + wi_bytes};
            const size_t ns[3] = {(size_t)LR * NIN, (size_t)HC * NIN, (size_t)NIN * LR};
            for (int m = 0; m < 3; ++m) {
                std::vector<float> f(ns[m]);
                ref::dequant(ref::BF16, b + offs[m], f.data(), ns[m]);
                if (m == 1 && injf) std::memcpy(b + offs[m], f.data(), 4 * ns[m]);
                else ref::quantize_q8_0(f.data(), ns[m], b + offs[m]);
            }
        }
    }
    const int MAXT = 8;
    // GR_DOWN=SG_KS selects gr_down's variant (default 8_8: 8 simdgroups split one row)
    const std::string grd = std::getenv("GR_DOWN") ? std::getenv("GR_DOWN") : "8_8";
    int grd_sg = 8, grd_ks = 8;
    std::sscanf(grd.c_str(), "%d_%d", &grd_sg, &grd_ks);
    const int grd_rs = grd_sg / grd_ks;
    mtl::Buffer R = cx.alloc(4 * MAXT * NIN), R0 = cx.alloc(4 * MAXT * NIN), bo = cx.alloc(4 * MAXT * N),
                inj = cx.alloc(4 * MAXT * HC), xn = cx.alloc(4 * MAXT * NIN), lo = cx.alloc(4 * MAXT * LR),
                injo = cx.alloc(4 * MAXT * HC), xp = cx.alloc(4 * (4 * MAXT * N + MAXT * N / 16)),
                xr = cx.alloc(4 * (4 * MAXT * N + MAXT * N / 16));
    for (int i = 0; i < MAXT * NIN; ++i) R0.as<float>()[i] = rng.sym() * 2.0f;
    for (int i = 0; i < MAXT * N; ++i) bo.as<float>()[i] = rng.sym();
    for (int i = 0; i < MAXT * HC; ++i) inj.as<float>()[i] = rng.sym() * 3.0f;
    mtl::Pipeline pprep = cx.pipeline(lib, "xprep");
    int fails = 0;
    for (int nt : ntoks) {
        mtl::Pipeline pn = cx.pipeline(lib, "gr_norm"),
                      pd = cx.pipeline(lib, std::string(injf ? "gr_down_q8f_" : q8 ? "gr_down_q8_" : "gr_down_") + grd, mtl::Constants().i(1, nt)),
                      pu = cx.pipeline(lib, q8 ? "gr_up_q8" : "gr_up", mtl::Constants().i(1, nt));
        for (int apply = 0; apply < 2; ++apply) {
            GrArgs a{N, HC, LR, (uint32_t)nt, (uint32_t)apply, 1e-6f};
            std::memcpy(R.ptr, R0.ptr, R.bytes);
            auto enc = [&](int set, std::vector<mtl::Dispatch>& ds, int which) {
                const size_t o = (size_t)set * set_bytes;
                mtl::Dispatch d;
                if (which & 1) {
                    d.p = &pn;
                    d.args = {mtl::Arg::B(R), mtl::Arg::B(bo), mtl::Arg::B(inj), mtl::Arg::B(W, o + wd_bytes + wi_bytes + wu_bytes),
                              mtl::Arg::B(xn), mtl::Arg::V(a)};
                    d.grid[0] = HC;
                    d.grid[1] = nt;
                    d.tg[0] = 256;
                    ds.push_back(d);
                }
                if (which & 2) {
                    mtl::Dispatch e;
                    e.p = &pd;
                    e.args = {mtl::Arg::B(W, o), mtl::Arg::B(W, o + wd_bytes), mtl::Arg::B(xn), mtl::Arg::B(lo),
                              mtl::Arg::B(injo), mtl::Arg::V(a)};
                    e.grid[0] = (LR + HC + grd_rs - 1) / grd_rs;   // RS = SG / KS rows per threadgroup
                    e.tg[0] = 32 * grd_sg;
                    ds.push_back(e);
                }
                if (which & 4) {
                    mtl::Dispatch f;
                    f.p = &pu;
                    f.args = {mtl::Arg::B(W, o + wd_bytes + wi_bytes), mtl::Arg::B(lo), mtl::Arg::B(xn), mtl::Arg::B(xp),
                              mtl::Arg::V(a)};
                    f.grid[0] = N / 16;
                    f.tg[0] = 256;
                    ds.push_back(f);
                }
            };
            if (!bench) {
                std::vector<mtl::Dispatch> ds;
                enc(0, ds, 7);
                cx.run(ds);
                // reference
                const uint8_t* b = W.as<uint8_t>();
                std::vector<float> wd((size_t)LR * NIN), wi((size_t)HC * NIN), wu((size_t)NIN * LR);
                ref::dequant(wty, b, wd.data(), wd.size());
                ref::dequant(injf ? ref::F32 : wty, b + wd_bytes, wi.data(), wi.size());
                ref::dequant(wty, b + wd_bytes + wi_bytes, wu.data(), wu.size());
                const float* wn = (const float*)(b + wd_bytes + wi_bytes + wu_bytes);
                Err eR, elo, einj, emix;
                for (int t = 0; t < nt; ++t) {
                    std::vector<double> r(NIN), x(NIN), l(LR);
                    for (int c = 0; c < HC; ++c) {
                        const double g = apply ? 2.0 / (1.0 + std::exp(-(double)inj.as<float>()[t * HC + c] / HC)) : 0.0;
                        double ss = 0;
                        for (int d = 0; d < N; ++d) {
                            double v = R0.as<float>()[t * NIN + c * N + d];
                            if (apply) v += bo.as<float>()[t * N + d] * g;
                            r[c * N + d] = v;
                            ss += v * v;
                        }
                        const double rs = 1.0 / std::sqrt(ss / N + 1e-6);
                        for (int d = 0; d < N; ++d) x[c * N + d] = r[c * N + d] * wn[c * N + d] * rs;
                    }
                    for (int i = 0; i < NIN; ++i) eR.add(R.as<float>()[t * NIN + i], r[i]);
                    for (int k = 0; k < LR; ++k) {
                        double s = 0;
                        for (int i = 0; i < NIN; ++i) s += (double)wd[(size_t)k * NIN + i] * x[i];
                        const double z = s / HC;
                        l[k] = z / (1.0 + std::exp(-z));
                        elo.add(lo.as<float>()[t * LR + k], l[k]);
                    }
                    for (int c = 0; c < HC; ++c) {
                        double s = 0;
                        for (int i = 0; i < NIN; ++i) s += (double)wi[(size_t)c * NIN + i] * x[i];
                        einj.add(injo.as<float>()[t * HC + c], s);
                    }
                    for (int d = 0; d < N; ++d) {
                        double m = 0;
                        for (int c = 0; c < HC; ++c) {
                            double s = 0;
                            // the up projection reads the GPU's lo, so the check isolates gr_up
                            for (int k = 0; k < LR; ++k) s += (double)wu[(size_t)(c * N + d) * LR + k] * lo.as<float>()[t * LR + k];
                            m += x[c * N + d] / (1.0 + std::exp(-s));
                        }
                        emix.add(xp.as<float>()[t * N + d], m / HC);
                    }
                }
                // the prepared planes must be bitwise xprep's of the raw plane
                {
                    mtl::Dispatch d;
                    d.p = &pprep;
                    const uint32_t n = N, ntt = nt;
                    d.args = {mtl::Arg::B(xp), mtl::Arg::B(xr), mtl::Arg::V(n), mtl::Arg::V(ntt)};
                    d.tg[0] = 256;
                    d.grid[0] = (N / 16 * nt + 255) / 256;
                    cx.run({d});
                }
                const size_t elems = (size_t)4 * nt * N + (size_t)nt * N / 16;
                const bool planes = std::memcmp(xp.ptr, xr.ptr, 4 * elems) == 0;
                const bool ok = eR.l2() < 1e-6 && elo.l2() < 1e-5 && einj.l2() < 1e-5 && emix.l2() < 1e-5 && planes;
                fails += !ok;
                std::printf("parity ntok %d apply %d: R rel L2 %.2e, lo %.2e, inject %.2e, mixed %.2e, prepared planes "
                            "== xprep: %s  %s\n", nt, apply, eR.l2(), elo.l2(), einj.l2(), emix.l2(),
                            planes ? "bitwise" : "NO", ok ? "PASS" : "FAIL");
            } else {
                for (int which : {1, 2, 4, 7}) {
                    std::vector<mtl::Dispatch> ds;
                    for (int r = 0; r < 96; ++r) enc(r % SETS, ds, which);
                    cx.warm(ds, 0.1);
                    std::vector<double> ts;
                    for (int i = 0; i < 7; ++i) ts.push_back(cx.run(ds) / 96);
                    const double m = median(ts);
                    const double bytes = which == 1 ? 2.0 * nt * NIN * 4 + NIN * 4 + nt * NIN * 4
                                         : which == 2 ? wd_bytes + wi_bytes
                                         : which == 4 ? wu_bytes
                                                      : wd_bytes + wi_bytes + wu_bytes;
                    std::printf("bench ntok %d apply %d %-8s %7.2f us  %6.1f GB/s (%.2f MB)\n", nt, apply,
                                which == 1 ? "norm" : which == 2 ? "down" : which == 4 ? "up" : "all 3", m * 1e6,
                                bytes / m / 1e9, bytes / 1e6);
                }
            }
        }
    }
    if (!bench) std::printf("%s\n", fails ? "SOME FAILED" : "ALL PASS");
    return fails ? 1 : 0;
}