Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
16.5 KB
Raw Blame
// src/kernels/fused_gr_bench.cpp - fused_gr_read_multi: the AMD fast kernels (STRATA_GR_FAST) against the old
// ones, bit for bit on every output (lo, rs, inject, mixed, the in-place R), and each one's time.
//
//     build/fused_gr_bench [iters] [T min] [T max] [cold sets]
// With "cold sets" > 1 it also times the read with the weights rotating over that many copies (a layer's weights are
// cold in the engine) and prints the per-stage split (norm / down / up) from the GPU stamps.
#include "strata/kernels/fused_gr.hpp"
#include "strata/kernels/verify_kernels.hpp"

#include <cuda_runtime.h>

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <vector>

namespace K = strata::kernels;

static uint16_t bf16(float x) {
    uint32_t u;
    std::memcpy(&u, &x, 4);
    return (uint16_t) ((u + 0x7fff + ((u >> 16) & 1)) >> 16);
}

// Q8_0 (ggml reference rounding: d = amax / 127 as fp16, q = round(x / d)) of a row-major [rows][cols] bf16 matrix
static std::vector<uint8_t> q8_0_of_bf16(const std::vector<uint16_t>& w, size_t cols) {
    std::vector<uint8_t> out(w.size() / 32 * 34);
    for (size_t b = 0; b < w.size() / 32; ++b) {
        float v[32], amax = 0.f;
        for (int i = 0; i < 32; ++i) {
            uint32_t u = (uint32_t) w[b * 32 + i] << 16;
            std::memcpy(&v[i], &u, 4);
            amax = std::fmax(amax, std::fabs(v[i]));
        }
        const float d = amax / 127.f, id = d != 0.f ? 1.f / d : 0.f;
        // fp16 of d
        uint32_t x; std::memcpy(&x, &d, 4);
        const uint32_t sign = (x >> 16) & 0x8000u;
        int32_t e = (int32_t) ((x >> 23) & 0xff) - 127 + 15;
        uint32_t m = x & 0x7fffffu;
        uint16_t h;
        if (e <= 0) h = (uint16_t) sign;   // flush denormals: weights here are far from them
        else if (e >= 31) h = (uint16_t) (sign | 0x7c00u);
        else h = (uint16_t) (sign | ((uint32_t) e << 10) | (m >> 13));
        uint8_t* o = &out[b * 34];
        std::memcpy(o, &h, 2);
        for (int i = 0; i < 32; ++i) o[2 + i] = (uint8_t) (int8_t) std::lround(v[i] * id);
    }
    (void) cols;
    return out;
}

int main(int argc, char** argv) {
    // the bench's contract is the default kernels' bitwise hashes; the split read (a default on a 12.0 card since 0.2.7) is another summation order
#if defined(_WIN32)
    if (std::getenv("STRATA_GR_V3") == nullptr) _putenv_s("STRATA_GR_V3", "0");
#else
    if (std::getenv("STRATA_GR_V3") == nullptr) setenv("STRATA_GR_V3", "0", 1);
#endif
    setvbuf(stdout, nullptr, _IONBF, 0);
    const int iters = argc > 1 ? std::atoi(argv[1]) : 500;
    const int t_lo = argc > 2 ? std::atoi(argv[2]) : 1, t_hi = argc > 3 ? std::atoi(argv[3]) : 6;
    const int cold_sets = argc > 4 ? std::atoi(argv[4]) : 0;
    const int N = 2560, HC = 4, D = N * HC, LR = 320, TM = K::kFusedGrMaxT;
    std::mt19937 rng(7);
    std::normal_distribution<float> nd(0.f, 1.f);
    auto fill = [&](std::vector<float>& v, float sc) { for (auto& x : v) x = sc * nd(rng); };
    std::vector<float> R((size_t) TM * D), bo((size_t) TM * N), inj((size_t) TM * HC), wn(D);
    fill(R, 1.0f); fill(bo, 0.5f); fill(inj, 1.0f); fill(wn, 0.3f);
    for (auto& x : wn) x += 1.0f;
    std::vector<uint16_t> wd((size_t) LR * D), wu((size_t) D * LR), wi((size_t) HC * D);
    for (auto& x : wd) x = bf16(0.02f * nd(rng));
    for (auto& x : wu) x = bf16(0.05f * nd(rng));
    for (auto& x : wi) x = bf16(0.02f * nd(rng));
    float *dR, *dbo, *dinj, *dwn, *dxn, *dlo, *drs, *dio, *dmix;
    uint16_t *dwd, *dwu, *dwi;
    cudaMalloc((void**) &dR, R.size() * 4);
    cudaMalloc((void**) &dbo, bo.size() * 4);
    cudaMalloc((void**) &dinj, inj.size() * 4);
    cudaMalloc((void**) &dwn, wn.size() * 4);
    cudaMalloc((void**) &dxn, (size_t) TM * D * 4);
    cudaMalloc((void**) &dlo, (size_t) TM * LR * 4);
    cudaMalloc((void**) &drs, (size_t) TM * HC * 4);
    cudaMalloc((void**) &dio, (size_t) TM * HC * 4);
    cudaMalloc((void**) &dmix, (size_t) TM * N * 4);
    cudaMalloc((void**) &dwd, wd.size() * 2);
    cudaMalloc((void**) &dwu, wu.size() * 2);
    cudaMalloc((void**) &dwi, wi.size() * 2);
    cudaMemcpy(dbo, bo.data(), bo.size() * 4, cudaMemcpyHostToDevice);
    cudaMemcpy(dinj, inj.data(), inj.size() * 4, cudaMemcpyHostToDevice);
    cudaMemcpy(dwn, wn.data(), wn.size() * 4, cudaMemcpyHostToDevice);
    cudaMemcpy(dwd, wd.data(), wd.size() * 2, cudaMemcpyHostToDevice);
    cudaMemcpy(dwu, wu.data(), wu.size() * 2, cudaMemcpyHostToDevice);
    cudaMemcpy(dwi, wi.data(), wi.size() * 2, cudaMemcpyHostToDevice);
    cudaStream_t s;
    cudaStreamCreate(&s);
    cudaEvent_t e0, e1;
    cudaEventCreate(&e0);
    cudaEventCreate(&e1);
    int failures = 0;
    for (int T = t_lo; T <= t_hi; ++T)
        for (int apply = 0; apply < 2; ++apply)
            for (int inject = 0; inject < 2; ++inject) {
                K::FusedGrArgs a[TM];
                for (int t = 0; t < T; ++t) {
                    a[t].R = dR + (size_t) t * D; a[t].R_out = dR + (size_t) t * D; a[t].apply = apply;
                    a[t].bo_prev = dbo + (size_t) t * N; a[t].inj_prev = dinj + (size_t) t * HC;
                    a[t].w_norm = dwn; a[t].w_down = dwd; a[t].w_up = dwu; a[t].w_inject = inject ? dwi : nullptr;
                    a[t].lo = dlo + (size_t) t * LR; a[t].rs = drs + (size_t) t * HC;
                    a[t].inject_out = dio + (size_t) t * HC; a[t].mixed = dmix + (size_t) t * N;
                }
                std::vector<float> out[2];
                for (int f = 0; f < 2; ++f) {
                    cudaMemcpy(dR, R.data(), R.size() * 4, cudaMemcpyHostToDevice);
                    cudaMemset(dlo, 0xff, (size_t) TM * LR * 4);
                    cudaMemset(drs, 0xff, (size_t) TM * HC * 4);
                    cudaMemset(dio, 0xff, (size_t) TM * HC * 4);
                    cudaMemset(dmix, 0xff, (size_t) TM * N * 4);
                    K::fused_gr_set_fast(f);
                    K::fused_gr_read_multi(a, T, dxn, s);
                    cudaStreamSynchronize(s);
                    auto& o = out[f];
                    o.resize((size_t) TM * (D + LR + HC + HC + N));
                    float* p = o.data();
                    cudaMemcpy(p, dR, (size_t) TM * D * 4, cudaMemcpyDeviceToHost); p += (size_t) TM * D;
                    cudaMemcpy(p, dlo, (size_t) TM * LR * 4, cudaMemcpyDeviceToHost); p += (size_t) TM * LR;
                    cudaMemcpy(p, drs, (size_t) TM * HC * 4, cudaMemcpyDeviceToHost); p += (size_t) TM * HC;
                    cudaMemcpy(p, dio, (size_t) TM * HC * 4, cudaMemcpyDeviceToHost); p += (size_t) TM * HC;
                    cudaMemcpy(p, dmix, (size_t) TM * N * 4, cudaMemcpyDeviceToHost);
                }
                const bool same = std::memcmp(out[0].data(), out[1].data(), out[0].size() * 4) == 0;
                double us[2] = {1e30, 1e30};
                for (int round = 0; round < 3; ++round)
                    for (int f = 0; f < 2; ++f) {
                        K::fused_gr_set_fast(f);
                        for (int i = 0; i < 20; ++i) K::fused_gr_read_multi(a, T, dxn, s);
                        cudaEventRecord(e0, s);
                        for (int i = 0; i < iters; ++i) K::fused_gr_read_multi(a, T, dxn, s);
                        cudaEventRecord(e1, s);
                        cudaEventSynchronize(e1);
                        float ms = 0;
                        cudaEventElapsedTime(&ms, e0, e1);
                        us[f] = std::fmin(us[f], 1e3 * ms / iters);
                    }
                unsigned long long h = 1469598103934665603ull;   // FNV-1a of the outputs: compare across runs (env switches)
                for (size_t b = 0; b < out[0].size() * 4; ++b) h = (h ^ ((const unsigned char*) out[0].data())[b]) * 1099511628211ull;
                std::printf("T %d apply %d inject %d | old %6.1f us  fast %6.1f us | %s | hash %016llx\n", T, apply, inject,
                            us[0], us[1], same ? "bitwise equal" : "DIFFERS", h);
                if (!same) ++failures;
            }
    {   // the Q8_0 read against the bf16 read: relative RMS error of lo / mixed / R / inject, and the q8_1 image against mixed
        const std::vector<uint8_t> qd = q8_0_of_bf16(wd, D), qu = q8_0_of_bf16(wu, LR);
        uint8_t *dqd, *dqu, *dqm; unsigned* dcnt;
        cudaMalloc((void**) &dqd, qd.size()); cudaMemcpy(dqd, qd.data(), qd.size(), cudaMemcpyHostToDevice);
        cudaMalloc((void**) &dqu, qu.size()); cudaMemcpy(dqu, qu.data(), qu.size(), cudaMemcpyHostToDevice);
        cudaMalloc((void**) &dqm, (size_t) TM * (N / 32) * 36); cudaMalloc((void**) &dcnt, (N / 32) * 4);
        cudaMemset(dcnt, 0, (N / 32) * 4);
        for (int T = t_lo; T <= t_hi; ++T) for (int apply = 0; apply < 2; ++apply) {
            K::FusedGrArgs a[TM];
            for (int t = 0; t < T; ++t) {
                a[t].R = dR + (size_t) t * D; a[t].R_out = dR + (size_t) t * D; a[t].apply = apply;
                a[t].bo_prev = dbo + (size_t) t * N; a[t].inj_prev = dinj + (size_t) t * HC;
                a[t].w_norm = dwn; a[t].w_down = dwd; a[t].w_up = dwu; a[t].w_inject = dwi;
                a[t].lo = dlo + (size_t) t * LR; a[t].rs = drs + (size_t) t * HC;
                a[t].inject_out = dio + (size_t) t * HC; a[t].mixed = dmix + (size_t) t * N;
            }
            std::vector<float> o[2];
            std::vector<uint8_t> img;
            for (int q = 0; q < 2; ++q) {
                for (int t = 0; t < T; ++t) {
                    a[t].q8_down = q ? dqd : nullptr; a[t].q8_up = q ? dqu : nullptr;
                    a[t].q8_mixed = (uint8_t*) dqm + (size_t) t * (N / 32) * 36; a[t].q8_cnt = dcnt;
                }
                cudaMemcpy(dR, R.data(), R.size() * 4, cudaMemcpyHostToDevice);
                cudaMemset(dqm, 0, (size_t) TM * (N / 32) * 36);
                K::fused_gr_read_multi(a, T, dxn, s);
                cudaStreamSynchronize(s);
                o[q].resize((size_t) T * (LR + N + D + HC));
                float* p = o[q].data();
                cudaMemcpy(p, dlo, (size_t) T * LR * 4, cudaMemcpyDeviceToHost); p += (size_t) T * LR;
                cudaMemcpy(p, dmix, (size_t) T * N * 4, cudaMemcpyDeviceToHost); p += (size_t) T * N;
                cudaMemcpy(p, dR, (size_t) T * D * 4, cudaMemcpyDeviceToHost); p += (size_t) T * D;
                cudaMemcpy(p, dio, (size_t) T * HC * 4, cudaMemcpyDeviceToHost);
                if (q) { img.resize((size_t) T * (N / 32) * 36); cudaMemcpy(img.data(), dqm, img.size(), cudaMemcpyDeviceToHost); }
            }
            unsigned long long hq = 1469598103934665603ull;   // FNV-1a of the Q8 read's outputs and q8_1 image
            for (size_t b = 0; b < o[1].size() * 4; ++b) hq = (hq ^ ((const unsigned char*) o[1].data())[b]) * 1099511628211ull;
            for (size_t b = 0; b < img.size(); ++b) hq = (hq ^ img[b]) * 1099511628211ull;
            double num = 0, den = 0, nm = 0, dm = 0;
            for (size_t i = 0; i < (size_t) T * LR; ++i) { num += std::pow(o[1][i] - o[0][i], 2); den += std::pow(o[0][i], 2); }
            for (size_t i = (size_t) T * LR; i < (size_t) T * (LR + N); ++i) { nm += std::pow(o[1][i] - o[0][i], 2); dm += std::pow(o[0][i], 2); }
            double rn = 0;   // R (apply) and inject must be identical or close
            for (size_t i = (size_t) T * (LR + N); i < o[0].size(); ++i) rn = std::fmax(rn, std::fabs(o[1][i] - o[0][i]));
            // q8_1 image: 36 bytes = half2 (d, sum) + 32 int8; dequantized against mixed
            double qn = 0, qd2 = 0;
            for (int t = 0; t < T; ++t) for (int g = 0; g < N / 32; ++g) {
                const uint8_t* b = &img[((size_t) t * (N / 32) + g) * 36];
                uint16_t h; std::memcpy(&h, b, 2);
                const uint32_t sg = (h >> 15) & 1, ex = (h >> 10) & 31, mn = h & 1023;
                const float d = (ex ? std::ldexp(1.0f + mn / 1024.0f, (int) ex - 15) : std::ldexp(mn / 1024.0f, -14)) * (sg ? -1.f : 1.f);
                for (int j = 0; j < 32; ++j) {
                    const float ref = o[1][(size_t) T * LR + (size_t) t * N + g * 32 + j];
                    qn += std::pow(d * (int8_t) b[4 + j] - ref, 2); qd2 += ref * ref;
                }
            }
            std::printf("Q8 vs bf16 T %d apply %d | rel rms lo %.4f mixed %.4f | max |dR,dinject| %.3g | q8_1 image vs mixed rel rms %.4f | hash %016llx\n", T, apply,
                        std::sqrt(num / den), std::sqrt(nm / dm), rn, std::sqrt(qn / qd2), hq);
        }
    }
    if (cold_sets > 1) {
        // the engine reads a different layer's weights every time: rotate over `cold_sets` copies (13 MB each)
        std::vector<uint16_t*> cwd(cold_sets), cwu(cold_sets), cwi(cold_sets);
        const std::vector<uint8_t> qd = q8_0_of_bf16(wd, D), qu = q8_0_of_bf16(wu, LR);
        std::vector<uint8_t*> cqd(cold_sets), cqu(cold_sets);
        std::vector<float*> cwn(cold_sets);
        for (int c = 0; c < cold_sets; ++c) {
            cudaMalloc((void**) &cwd[c], wd.size() * 2); cudaMemcpy(cwd[c], wd.data(), wd.size() * 2, cudaMemcpyHostToDevice);
            cudaMalloc((void**) &cwu[c], wu.size() * 2); cudaMemcpy(cwu[c], wu.data(), wu.size() * 2, cudaMemcpyHostToDevice);
            cudaMalloc((void**) &cwi[c], wi.size() * 2); cudaMemcpy(cwi[c], wi.data(), wi.size() * 2, cudaMemcpyHostToDevice);
            cudaMalloc((void**) &cqd[c], qd.size()); cudaMemcpy(cqd[c], qd.data(), qd.size(), cudaMemcpyHostToDevice);
            cudaMalloc((void**) &cqu[c], qu.size()); cudaMemcpy(cqu[c], qu.data(), qu.size(), cudaMemcpyHostToDevice);
            cudaMalloc((void**) &cwn[c], wn.size() * 4); cudaMemcpy(cwn[c], wn.data(), wn.size() * 4, cudaMemcpyHostToDevice);
        }
        uint8_t* dqm; unsigned* dcnt;   // the q8_1 image of mixed (STRATA_QFUSE_HC) as the engine asks for it
        cudaMalloc((void**) &dqm, (size_t) TM * (N / 32) * 36); cudaMalloc((void**) &dcnt, (N / 32) * 4);
        cudaMemset(dcnt, 0, (N / 32) * 4);
        const bool with_q = std::getenv("STRATA_BENCH_QMIXED") != nullptr;
        unsigned long long* dst;
        cudaMalloc((void**) &dst, (size_t) iters * 4 * 8);
        std::vector<unsigned long long> hst((size_t) iters * 4);
        std::vector<K::FusedGrArgs> as((size_t) cold_sets * 8);
        for (int T = t_lo; T <= t_hi; ++T) for (int f = 0; f < 3; ++f) {
            K::fused_gr_set_fast(f == 1);
            for (int c = 0; c < cold_sets; ++c)
                for (int t = 0; t < T; ++t) {
                    K::FusedGrArgs& a = as[(size_t) c * 8 + t];
                    a = K::FusedGrArgs();
                    a.R = dR + (size_t) t * D; a.R_out = dR + (size_t) t * D; a.apply = 1;
                    a.bo_prev = dbo + (size_t) t * N; a.inj_prev = dinj + (size_t) t * HC;
                    a.w_norm = cwn[c]; a.w_down = cwd[c]; a.w_up = cwu[c]; a.w_inject = cwi[c];
                    if (with_q) { a.q8_mixed = dqm + (size_t) t * (N / 32) * 36; a.q8_cnt = dcnt; }
                    if (f == 2) { a.q8_down = cqd[c]; a.q8_up = cqu[c]; }
                    a.lo = dlo + (size_t) t * LR; a.rs = drs + (size_t) t * HC;
                    a.inject_out = dio + (size_t) t * HC; a.mixed = dmix + (size_t) t * N;
                }
            for (int rep = 0; rep < 2; ++rep) {
                for (int i = 0; i < iters; ++i) {
                    K::gpu_stamp(dst, i * 4, s);
                    K::fused_gr_read_multi(&as[(size_t) (i % cold_sets) * 8], T, dxn, s, dst, i * 4 + 1);
                    K::gpu_stamp(dst, i * 4 + 3, s);
                }
                cudaStreamSynchronize(s);
            }
            cudaEventRecord(e0, s);
            for (int i = 0; i < iters; ++i) K::fused_gr_read_multi(&as[(size_t) (i % cold_sets) * 8], T, dxn, s);
            cudaEventRecord(e1, s);
            cudaEventSynchronize(e1);
            float ms = 0;
            cudaEventElapsedTime(&ms, e0, e1);
            cudaMemcpy(hst.data(), dst, hst.size() * 8, cudaMemcpyDeviceToHost);
            // stamps: 0 start, 1 after norm, 2 after down, 3 after up (a ~1-2 us stamp kernel between each)
            std::vector<double> n_, d_, u_;
            for (int i = 1; i < iters; ++i) {
                const unsigned long long* p = &hst[(size_t) i * 4];
                n_.push_back((double) (p[1] - p[0])); d_.push_back((double) (p[2] - p[1])); u_.push_back((double) (p[3] - p[2]));
            }
            auto med = [](std::vector<double>& v) { std::sort(v.begin(), v.end()); return v[v.size() / 2] / 1e3; };
            std::printf("COLD(%d sets) fast %d T %d | read %6.1f us (events) | stamps median: norm %5.1f down %5.1f up %5.1f us (incl. a stamp kernel each)\n",
                        cold_sets, f, T, 1e3 * ms / iters, med(n_), med(d_), med(u_));
        }
    }
    std::printf("fused_gr_bench: %d differing\n", failures);
    return failures ? 1 : 0;
}