Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
14.2 KB
Raw Blame
// prefill_ws_parity_test - the weight-stationary expert products of PR #3752 (STRATA_MMQ_WS, GP100) against the per-group
// MMQ they replace, byte for byte, on random experts (GPU, synthetic data, no model).
//
// A layer of E resident experts (random gate/up [1280 x 2560] and down [2560 x 640] blocks with sane fp16 scales, laid out
// as the expert cache holds them: gate rows then up rows, the down matrix at down_off) gets random activations, quantized
// as the prompt path does (mmq::quantize), and a routing of random row counts per expert (1 .. 300 rows, so u spans the
// measured range, some experts with one row).  The reference is mmq::Context::run per expert (gate/up, then
// swiglu + quantize, then down), i.e. what Prefill does per 16-expert group; the candidate is ws::gate_up, ONE
// mmq::swiglu_quant over all rows and ws::down.  Compared: GU (floats), Hq (the q8_1 bytes) and Dm (floats) of every row.
// The down is also run alone on the reference Hq, so a difference points at the product. For the IQ4_NL down the run
// repeats with STRATA_WS_FAST on (ws::set_fast(1)); the same Dm is required.
//
// The kernels were written for GP100 (the integer sums are exact, the float epilogue follows MMQ's sm_60 SASS), so the
// verdict is only given on a compute capability 6.0 card; on another card the kernels are run too (ws::set_mode(1))
// and the counts printed for information, with exit 77.  Needs the CUDA build with STRATA_EXPERIMENTAL_SM60.
//
//   prefill_ws_parity_test [--device N] [--any]    exit 0: all identical (cc 6.0); 77: other card; 1: a difference
#include "strata/prefill/moe_mmq.hpp"
#include "strata/prefill/moe_ws.hpp"

#include "ggml.h"

#include <cuda_runtime.h>

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <numeric>
#include <random>
#include <stdexcept>
#include <string>
#include <vector>

namespace {
namespace mmq = strata::prefill::mmq;
namespace ws = strata::prefill::ws;

void ck(cudaError_t e, const char* what) {
    if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
}

struct Dev {
    void* p = nullptr;
    size_t n = 0;
    explicit Dev(size_t bytes) : n(bytes) {
        ck(cudaMalloc(&p, bytes + 256), "cudaMalloc");
        ck(cudaMemset(p, 0, bytes + 256), "cudaMemset");
    }
    ~Dev() { cudaFree(p); }
    Dev(const Dev&) = delete;
    Dev& operator=(const Dev&) = delete;
    template <class T> T* as() const { return (T*) p; }
    template <class T> void up(const std::vector<T>& h) { ck(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload"); }
    template <class T> std::vector<T> down(size_t count) const {
        std::vector<T> h(count);
        ck(cudaMemcpy(h.data(), p, count * sizeof(T), cudaMemcpyDeviceToHost), "download");
        return h;
    }
};

constexpr int64_t kNEmbd = 2560, kNFF = 640, kGuRows = 1280;

// random blocks of `type` for a [rows x cols] matrix; every block starts with its fp16 scale d: a finite value in
// [2^-6, 2^-5) (all the formats used here put d first)
std::vector<uint8_t> random_matrix(ggml_type type, int64_t rows, int64_t cols, std::mt19937& rng) {
    const size_t bytes = mmq::matrix_bytes((int) type, rows, cols);
    const size_t bs = ggml_type_size(type);
    std::vector<uint8_t> w(bytes);
    for (auto& b : w) b = (uint8_t) rng();
    for (size_t o = 0; o + bs <= bytes; o += bs) {
        const uint16_t d = (uint16_t) ((9u << 10) | (rng() & 0x3FFu));
        std::memcpy(&w[o], &d, 2);
    }
    return w;
}

size_t count_diff(const std::vector<uint8_t>& a, const std::vector<uint8_t>& b, size_t n) {
    size_t d = 0;
    for (size_t i = 0; i < n; ++i) d += a[i] != b[i];
    return d;
}

// the largest |a - b| over the largest |a|, both float arrays in byte form (how far apart two results are in rounding terms)
double max_norm_err(const std::vector<uint8_t>& a, const std::vector<uint8_t>& b, size_t bytes) {
    const float* x = (const float*) a.data();
    const float* y = (const float*) b.data();
    double mx = 0.0, err = 0.0;
    for (size_t i = 0; i < bytes / 4; ++i) {
        mx = std::max(mx, (double) std::fabs(x[i]));
        err = std::max(err, (double) std::fabs(x[i] - y[i]));
    }
    return mx > 0.0 ? err / mx : err;
}

struct Result {
    size_t gu = 0, hq = 0, dm = 0, dm_ref_hq = 0, dm_fast = 0, fast_vs_default = 0;
    double e_gu = 0, e_dm = 0, e_dm_rh = 0;   // normalized max errors against the reference
    bool ran = true;
};

Result run_case(ggml_type gt, ggml_type dt, const std::vector<int32_t>& cnt, uint32_t seed, bool test_fast) {
    std::mt19937 rng(seed);
    const int E = (int) cnt.size();
    std::vector<int32_t> bounds(E + 1, 0);
    for (int j = 0; j < E; ++j) bounds[j + 1] = bounds[j] + cnt[j];
    const int64_t rows = bounds[E];
    const size_t gub = mmq::matrix_bytes((int) gt, kGuRows, kNEmbd), db = mmq::matrix_bytes((int) dt, kNEmbd, kNFF);
    const size_t stride = (gub + db + 255) / 256 * 256;
    // the layer's experts, "resident" in one buffer at 256-byte aligned offsets (the expert cache's slots)
    std::vector<uint8_t> blob((size_t) E * stride, 0);
    std::vector<int64_t> xoff(E);
    for (int j = 0; j < E; ++j) {
        xoff[j] = (int64_t) j * (int64_t) stride;
        const auto g = random_matrix(gt, kGuRows, kNEmbd, rng), d = random_matrix(dt, kNEmbd, kNFF, rng);
        std::memcpy(&blob[(size_t) xoff[j]], g.data(), gub);
        std::memcpy(&blob[(size_t) xoff[j] + gub], d.data(), db);
    }
    std::normal_distribution<float> nd(0.0f, 1.0f);
    std::vector<float> x((size_t) (rows * kNEmbd));
    for (auto& v : x) v = nd(rng) * 0.7f;

    const size_t xqb = mmq::q8_bytes(rows, kNEmbd), hqb = mmq::q8_bytes(rows, kNFF);
    Dev dblob(blob.size()), dx(x.size() * 4), dxq(xqb), dbnd((size_t) (E + 1) * 4), dxoff((size_t) E * 8);
    Dev dident((size_t) rows * 4), dgu_ref((size_t) rows * kGuRows * 4), dgu((size_t) rows * kGuRows * 4);
    Dev dh((size_t) rows * kNFF * 4), dhq_ref(hqb), dhq(hqb), ddm_ref((size_t) rows * kNEmbd * 4), ddm((size_t) rows * kNEmbd * 4),
        ddm_rh((size_t) rows * kNEmbd * 4), ddm_fast((size_t) rows * kNEmbd * 4);
    dblob.up(blob); dx.up(x); dbnd.up(bounds); dxoff.up(xoff);
    std::vector<int32_t> ident((size_t) rows);
    for (int64_t i = 0; i < rows; ++i) ident[(size_t) i] = (int32_t) i;
    dident.up(ident);
    mmq::quantize(dx.as<float>(), nullptr, dxq.p, (int) gt, kNEmbd, kNEmbd, rows, nullptr);

    // the reference: per expert, as a one-expert group (absolute rows)
    mmq::Context ctx;
    for (int j = 0; j < E; ++j) {
        mmq::Product gu;
        gu.w = dblob.as<uint8_t>() + xoff[j]; gu.type = (int) gt; gu.w_rows = kGuRows; gu.w_cols = kNEmbd; gu.expert_bytes = gub;
        gu.n = 1; gu.xq = dxq.p; gu.bounds = dbnd.as<int32_t>() + j; gu.ids = dident.as<int32_t>();
        gu.total_rows = rows; gu.max_rows = cnt[j]; gu.dst = dgu_ref.as<float>(); gu.ld_dst = kGuRows;
        ctx.run(gu, nullptr);
    }
    mmq::swiglu(dgu_ref.as<float>(), dh.as<float>(), rows, kNFF, false, nullptr);
    mmq::quantize(dh.as<float>(), nullptr, dhq_ref.p, (int) dt, kNFF, kNFF, rows, nullptr);
    for (int j = 0; j < E; ++j) {
        mmq::Product dn;
        dn.w = dblob.as<uint8_t>() + xoff[j] + gub; dn.type = (int) dt; dn.w_rows = kNEmbd; dn.w_cols = kNFF; dn.expert_bytes = db;
        dn.n = 1; dn.xq = dhq_ref.p; dn.bounds = dbnd.as<int32_t>() + j; dn.ids = dident.as<int32_t>();
        dn.total_rows = rows; dn.max_rows = cnt[j]; dn.dst = ddm_ref.as<float>(); dn.ld_dst = kNEmbd;
        ctx.run(dn, nullptr);
    }
    ck(cudaDeviceSynchronize(), "reference");

    // the candidate
    std::vector<int32_t> order(E);
    for (int j = 0; j < E; ++j) order[j] = j;
    std::vector<ws::Chunk> chunks(ws::max_chunks(rows, E));
    const size_t nch = ws::chunks(order.data(), (size_t) E, cnt.data(), bounds.data(), chunks.data());
    chunks.resize(nch);
    Dev dch(std::max<size_t>(1, nch) * sizeof(ws::Chunk));
    dch.up(chunks);
    ws::Layer L;
    L.gu_type = (int) gt; L.d_type = (int) dt; L.w = dblob.p; L.xoff = dxoff.as<int64_t>(); L.down_off = (int64_t) gub;
    L.chunks = dch.as<ws::Chunk>(); L.nchunks = (int64_t) nch; L.rows = rows; L.n = E; L.xq = dxq.p; L.gu = dgu.as<float>();
    L.hq = dhq.p; L.dm = ddm.as<float>();
    Result r;
    if (!ws::gate_up(L, nullptr)) { r.ran = false; return r; }
    mmq::swiglu_quant(dgu.as<float>(), dhq.p, rows, false, nullptr);
    ws::down(L, nullptr);
    ck(cudaDeviceSynchronize(), "candidate");
    // the down alone on the reference H (a difference above this one is the gate/up's)
    ws::Layer L2 = L;
    L2.hq = dhq_ref.p; L2.dm = ddm_rh.as<float>();
    ws::down(L2, nullptr);
    if (test_fast) {
        ws::set_fast(1);
        ws::Layer L3 = L;
        L3.dm = ddm_fast.as<float>();
        ws::down(L3, nullptr);
        ws::set_fast(-1);
    }
    ck(cudaDeviceSynchronize(), "candidate (down alone, fast)");

    const size_t gun = (size_t) rows * kGuRows * 4, dmn = (size_t) rows * kNEmbd * 4;
    const auto gur = dgu_ref.down<uint8_t>(gun), gun_c = dgu.down<uint8_t>(gun);
    r.gu = count_diff(gur, gun_c, gun);
    r.e_gu = max_norm_err(gur, gun_c, gun);
    r.hq = count_diff(dhq_ref.down<uint8_t>(hqb), dhq.down<uint8_t>(hqb), hqb);
    const auto dmr = ddm_ref.down<uint8_t>(dmn);
    const auto dmc = ddm.down<uint8_t>(dmn), dmh = ddm_rh.down<uint8_t>(dmn);
    r.dm = count_diff(dmr, dmc, dmn);
    r.dm_ref_hq = count_diff(dmr, dmh, dmn);
    r.e_dm = max_norm_err(dmr, dmc, dmn);
    r.e_dm_rh = max_norm_err(dmr, dmh, dmn);
    if (test_fast) {
        const auto dmf = ddm_fast.down<uint8_t>(dmn);
        r.dm_fast = count_diff(dmr, dmf, dmn);
        r.fast_vs_default = count_diff(dmc, dmf, dmn);   // STRATA_WS_FAST: the same bits as the default kernel, on any card
    }
    return r;
}

std::vector<int32_t> counts(int E, int lo, int hi, std::mt19937& rng) {
    std::vector<int32_t> c(E);
    for (auto& v : c) v = lo + (int32_t) (rng() % (uint32_t) (hi - lo + 1));
    return c;
}
}  // namespace

int main(int argc, char** argv) {
    int device = 0;
    bool any = false;
    for (int i = 1; i < argc; ++i) {
        if (std::strcmp(argv[i], "--device") == 0 && i + 1 < argc) device = std::atoi(argv[++i]);
        else if (std::strcmp(argv[i], "--any") == 0) any = true;
        else {
            std::fprintf(stderr, "usage: prefill_ws_parity_test [--device N] [--any]\n");
            return 2;
        }
    }
    int ndev = 0;
    if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
        std::puts("prefill_ws_parity_test: no CUDA device (skipped)");
        return 77;
    }
    try {
        ck(cudaSetDevice(device), "cudaSetDevice");
        cudaDeviceProp prop{};
        ck(cudaGetDeviceProperties(&prop, device), "cudaGetDeviceProperties");
        const bool gp100 = prop.major == 6 && prop.minor == 0;
        std::printf("prefill_ws_parity_test on %s (%d.%d)%s\n", prop.name, prop.major, prop.minor,
                    gp100 ? "" : ": not a GP100, so the counts are for information only");
        if (!gp100 && !any) {
            std::puts("prefill_ws_parity_test: not a compute capability 6.0 card (skipped; --any runs it for information)");
            return 77;
        }
        ws::set_mode(1);
        std::mt19937 rng(20261026u);
        const ggml_type gus[] = {GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S};
        const ggml_type dns[] = {GGML_TYPE_Q2_0, GGML_TYPE_IQ4_NL};
        size_t total_diff = 0, total_fast = 0;
        double worst_err = 0.0;
        int cases = 0, declined = 0;
        for (ggml_type gt : gus)
            for (ggml_type dt : dns) {
                // a spread of rows per expert (u from ~2 to ~150), and a tiny layer with one-row experts
                const std::vector<std::vector<int32_t>> shapes = {counts(24, 1, 12, rng), counts(24, 5, 40, rng),
                                                                  counts(20, 60, 300, rng), {1, 2, 65, 1, 64, 63}};
                for (const auto& cnt : shapes) {
                    const Result r = run_case(gt, dt, cnt, 1000u + (uint32_t) cases, dt == GGML_TYPE_IQ4_NL);
                    ++cases;
                    if (!r.ran) { ++declined; std::printf("  %-8s -> %-6s %zu experts: ws declined (not faster here)\n", ggml_type_name(gt), ggml_type_name(dt), cnt.size()); continue; }
                    const size_t bad = r.gu + r.hq + r.dm + r.dm_ref_hq + r.dm_fast;
                    total_diff += gp100 ? bad : 0;
                    total_fast += r.fast_vs_default;
                    worst_err = std::max(worst_err, std::max(r.e_gu, r.e_dm_rh));   // Dm after the full chain also carries int8 flips of H
                    std::printf("  %-8s -> %-6s %2zu experts %5d rows: bytes differ GU %zu  Hq %zu  Dm %zu  Dm(down alone) %zu  Dm(fast) %zu"
                                "  fast vs default %zu | max error / max value: GU %.2e  Dm %.2e  Dm(down alone) %.2e%s\n",
                                ggml_type_name(gt), ggml_type_name(dt), cnt.size(),
                                (int) std::accumulate(cnt.begin(), cnt.end(), 0), r.gu, r.hq, r.dm, r.dm_ref_hq, r.dm_fast,
                                r.fast_vs_default, r.e_gu, r.e_dm, r.e_dm_rh, gp100 && bad ? "  <-- FAIL" : "");
                }
            }
        if (!gp100) {   // no verdict on this card: the float epilogue of its MMQ is not the one the kernels follow
            std::printf("prefill_ws_parity_test (information, %s): %d cases (%d declined), STRATA_WS_FAST vs default: %zu differing bytes, "
                        "largest GU / down-alone error against this card's MMQ %.2e of the largest value -> %s\n", prop.name, cases, declined,
                        total_fast, worst_err, total_fast == 0 && worst_err < 1e-5 && declined < cases ? "consistent" : "INCONSISTENT");
            return total_fast == 0 && worst_err < 1e-5 && declined < cases ? 77 : 1;
        }
        std::printf("prefill_ws_parity_test: %d cases (%d declined), %zu differing bytes, STRATA_WS_FAST vs default %zu -> %s\n", cases,
                    declined, total_diff, total_fast, total_diff == 0 && total_fast == 0 && declined < cases ? "PASS" : "FAIL");
        return total_diff == 0 && total_fast == 0 && declined < cases ? 0 : 1;
    } catch (const std::exception& e) {
        std::fprintf(stderr, "prefill_ws_parity_test: %s\n", e.what());
        return 1;
    }
}