Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
10.3 KB
Raw Blame
// src/prefill/gr_upmix_parity.cu - the hyper-connection read's fused up projection + mix (gr_upmix, STRATA_HC_UPMIX=1)
// against the default pair (Gemm::bf16 - cuBLAS - writing `gated`, then gr_mix_r), on synthetic inputs of the model's
// shapes (N = 2560, 4 streams, K = 320).  Where cuBLAS's kernel sums K in the fused kernel's order the outputs are the
// same bits (CUDA 13.3 on an RTX 5090 from 33 tokens - the prompt path's default there); elsewhere FP32 rounding: the
// check bounds max |diff| / max |value| of `mixed`, counts the BF16 / FP16 images that differ, and measures both
// against an FP64 reference on sampled tokens (the fused one must be as close to it as cuBLAS).
//   gr_upmix_parity            the checks (T = 1 .. 32768)
//   gr_upmix_parity --bench    and the timings of the pair and the fused kernel at T = 2048 / 8192 / 32768
#include "strata/prefill/gemm.hpp"
#include "strata/prefill/kernels.hpp"

#include <cuda_runtime.h>

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <random>
#include <string>
#include <vector>

namespace {

constexpr int N = 2560, HC = 4, D = N * HC, LR = 320;

uint16_t to_bf16(float f) {
    uint32_t u;
    std::memcpy(&u, &f, 4);
    u += 0x7fffu + ((u >> 16) & 1u);
    return (uint16_t) (u >> 16);
}
float from_bf16(uint16_t h) {
    const uint32_t u = (uint32_t) h << 16;
    float f;
    std::memcpy(&f, &u, 4);
    return f;
}
bool ck(cudaError_t e, const char* what) {
    if (e != cudaSuccess) std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e));
    return e == cudaSuccess;
}
template <typename T> T* dalloc(size_t n) {
    T* p = nullptr;
    if (!ck(cudaMalloc(&p, n * sizeof(T)), "cudaMalloc")) std::exit(2);
    return p;
}

struct Inputs {
    int64_t T;
    std::vector<uint16_t> lo16, wu;
    std::vector<float> R, rs, w;
    uint16_t *d_lo16, *d_wu;
    float *d_R, *d_rs, *d_w;
    Inputs(int64_t t, uint32_t seed) : T(t) {
        std::mt19937 g(seed);
        std::normal_distribution<float> nd(0.0f, 1.0f);
        lo16.resize((size_t) T * LR);
        for (auto& x : lo16) x = to_bf16(nd(g) * 0.5f);            // silu outputs: O(1) and below
        wu.resize((size_t) D * LR);
        for (auto& x : wu) x = to_bf16(nd(g) * 0.12f);              // gated ~ N(0, ~1): sigmoid in its live range
        R.resize((size_t) T * D);
        for (auto& x : R) x = nd(g) * 3.0f;
        rs.resize((size_t) T * HC);
        for (auto& x : rs) x = 0.2f + 0.3f * std::fabs(nd(g));
        w.resize(D);
        for (auto& x : w) x = 1.0f + 0.1f * nd(g);
        d_lo16 = dalloc<uint16_t>(lo16.size());
        d_wu = dalloc<uint16_t>(wu.size());
        d_R = dalloc<float>(R.size());
        d_rs = dalloc<float>(rs.size());
        d_w = dalloc<float>(w.size());
        cudaMemcpy(d_lo16, lo16.data(), lo16.size() * 2, cudaMemcpyHostToDevice);
        cudaMemcpy(d_wu, wu.data(), wu.size() * 2, cudaMemcpyHostToDevice);
        cudaMemcpy(d_R, R.data(), R.size() * 4, cudaMemcpyHostToDevice);
        cudaMemcpy(d_rs, rs.data(), rs.size() * 4, cudaMemcpyHostToDevice);
        cudaMemcpy(d_w, w.data(), w.size() * 4, cudaMemcpyHostToDevice);
    }
    ~Inputs() { cudaFree(d_lo16); cudaFree(d_wu); cudaFree(d_R); cudaFree(d_rs); cudaFree(d_w); }
};

struct Outputs {
    float* mixed;
    uint16_t *bf, *h;
    explicit Outputs(int64_t T) {
        mixed = dalloc<float>((size_t) T * N);
        bf = dalloc<uint16_t>((size_t) T * N);
        h = dalloc<uint16_t>((size_t) T * N);
    }
    ~Outputs() { cudaFree(mixed); cudaFree(bf); cudaFree(h); }
};

// mixed[t][d] in FP64 from the same BF16 inputs (the exact result both GPU paths round)
double exact_mixed(const Inputs& in, int64_t t, int d) {
    double s = 0.0;
    for (int c = 0; c < HC; ++c) {
        double g = 0.0;
        const int j = c * N + d;
        for (int k = 0; k < LR; ++k)
            g += (double) from_bf16(in.lo16[(size_t) t * LR + k]) * (double) from_bf16(in.wu[(size_t) j * LR + k]);
        const double x = (double) in.R[(size_t) t * D + j] * in.rs[(size_t) t * HC + c] * in.w[j];
        s += x / (1.0 + std::exp(-g));
    }
    return s / HC;
}

int check(strata::prefill::Gemm& gm, cudaStream_t st, float* gated, int64_t T, uint32_t seed) {
    Inputs in(T, seed);
    Outputs ref(T), got(T);
    gm.bf16(in.d_lo16, in.d_wu, gated, T, D, LR);
    strata::prefill::gr_mix_r(in.d_R, in.d_rs, in.d_w, gated, ref.mixed, ref.bf, T, st, ref.h);
    if (!strata::prefill::gr_upmix(in.d_lo16, in.d_wu, in.d_R, in.d_rs, in.d_w, got.mixed, got.bf, got.h, T, st)) {
        std::printf("FAIL T=%lld: gr_upmix refused (needs CUDA sm_80+)\n", (long long) T);
        return 1;
    }
    if (!ck(cudaStreamSynchronize(st), "run")) return 1;
    const size_t n = (size_t) T * N;
    std::vector<float> a(n), b(n);
    std::vector<uint16_t> abf(n), bbf(n), ah(n), bh(n);
    cudaMemcpy(a.data(), ref.mixed, n * 4, cudaMemcpyDeviceToHost);
    cudaMemcpy(b.data(), got.mixed, n * 4, cudaMemcpyDeviceToHost);
    cudaMemcpy(abf.data(), ref.bf, n * 2, cudaMemcpyDeviceToHost);
    cudaMemcpy(bbf.data(), got.bf, n * 2, cudaMemcpyDeviceToHost);
    cudaMemcpy(ah.data(), ref.h, n * 2, cudaMemcpyDeviceToHost);
    cudaMemcpy(bh.data(), got.h, n * 2, cudaMemcpyDeviceToHost);
    double md = 0, mv = 0;
    size_t bf_diff = 0, h_diff = 0, bit_diff = 0, nonfinite = 0;
    for (size_t i = 0; i < n; ++i) {
        if (!std::isfinite(b[i])) ++nonfinite;
        md = std::max(md, (double) std::fabs(a[i] - b[i]));
        mv = std::max(mv, (double) std::fabs(a[i]));
        bf_diff += abf[i] != bbf[i];
        h_diff += ah[i] != bh[i];
        bit_diff += std::memcmp(&a[i], &b[i], 4) != 0;
    }
    // FP64 on sampled tokens: each path's max |error| / max |value|
    std::mt19937 g(seed ^ 0x5a5a);
    const int samples = (int) std::min<int64_t>(T, 24);
    double ea = 0, eb = 0, ev = 0;
    for (int i = 0; i < samples; ++i) {
        const int64_t t = T <= 24 ? i : (int64_t) (g() % (uint32_t) T);
        for (int d = 0; d < N; d += 37) {
            const double x = exact_mixed(in, t, d);
            ea = std::max(ea, std::fabs(a[(size_t) t * N + d] - x));
            eb = std::max(eb, std::fabs(b[(size_t) t * N + d] - x));
            ev = std::max(ev, std::fabs(x));
        }
    }
    const double rel = md / std::max(mv, 1e-30);
    const bool bad = nonfinite != 0 || rel > 1e-5 || eb > 2.0 * ea + 1e-6 * ev;
    std::printf("%s T=%-6lld %s mixed max |diff| / max |value| %.2e, BF16 / FP16 images differing %.4f%% / %.4f%%; "
                "vs FP64 (sampled): cuBLAS + mix %.2e, fused %.2e\n", bad ? "FAIL" : "ok  ", (long long) T,
                bit_diff == 0 && bf_diff == 0 && h_diff == 0 ? "bitwise " : "        ", rel,
                100.0 * (double) bf_diff / (double) n, 100.0 * (double) h_diff / (double) n, ea / ev, eb / ev);
    return bad ? 1 : 0;
}

float median(std::vector<float> v) {
    std::sort(v.begin(), v.end());
    return v[v.size() / 2];
}

void bench(strata::prefill::Gemm& gm, cudaStream_t st, float* gated, int64_t T) {
    Inputs in(T, 7);
    Outputs o(T);
    cudaEvent_t e0, e1;
    cudaEventCreate(&e0);
    cudaEventCreate(&e1);
    std::vector<float> tp, tg, tm, tf;
    for (int i = 0; i < 23; ++i) {
        const bool fused_first = i & 1;   // alternate which runs first
        for (int k = 0; k < 2; ++k) {
            if ((k == 0) == fused_first) {
                cudaEventRecord(e0, st);
                strata::prefill::gr_upmix(in.d_lo16, in.d_wu, in.d_R, in.d_rs, in.d_w, o.mixed, o.bf, o.h, T, st);
                cudaEventRecord(e1, st);
                cudaEventSynchronize(e1);
                float ms;
                cudaEventElapsedTime(&ms, e0, e1);
                if (i >= 3) tf.push_back(ms);
            } else {
                cudaEvent_t em;
                cudaEventCreate(&em);
                cudaEventRecord(e0, st);
                gm.bf16(in.d_lo16, in.d_wu, gated, T, D, LR);
                cudaEventRecord(em, st);
                strata::prefill::gr_mix_r(in.d_R, in.d_rs, in.d_w, gated, o.mixed, o.bf, T, st, o.h);
                cudaEventRecord(e1, st);
                cudaEventSynchronize(e1);
                float ms, msg;
                cudaEventElapsedTime(&ms, e0, e1);
                cudaEventElapsedTime(&msg, e0, em);
                if (i >= 3) { tp.push_back(ms); tg.push_back(msg); tm.push_back(ms - msg); }
                cudaEventDestroy(em);
            }
        }
    }
    const float p = median(tp), f = median(tf);
    // the fused kernel's bytes: R and the outputs (FP32 + 3 x 16-bit), lo16 and w_up once
    const double bytes = (double) T * D * 4 + (double) T * N * (4 + 6) + (double) T * LR * 2 + (double) D * LR * 2;
    std::printf("  T=%-6lld default pair %.3f ms (up GEMM %.3f + gr_mix_r %.3f) | fused %.3f ms: %.2fx  "
                "(fused: %.0f GB/s of its bytes, %.0f TFLOPS of its GEMM)\n", (long long) T, p, median(tg), median(tm),
                f, p / f, bytes / f / 1e6, 2.0 * T * D * LR / f / 1e9);
    cudaEventDestroy(e0);
    cudaEventDestroy(e1);
}

}  // namespace

int main(int argc, char** argv) {
    {   // the fused read is sm_80+ (cp.async, BF16 mma): an older card has nothing to compare
        int dev = 0, major = 0;
        if (cudaGetDevice(&dev) != cudaSuccess || cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess) {
            std::printf("gr_upmix_parity: no CUDA device, skipped\n");
            return 77;
        }
        if (major < 8) { std::printf("gr_upmix_parity: sm_%d0, the fused read needs sm_80+: skipped\n", major); return 77; }
    }
    const bool do_bench = argc > 1 && std::string(argv[1]) == "--bench";
    cudaStream_t st = nullptr;
    if (!ck(cudaStreamCreate(&st), "stream")) return 1;
    strata::prefill::Gemm gm;
    std::string err;
    if (!gm.init(st, 0, err)) { std::printf("FAIL: %s\n", err.c_str()); return 1; }
    const int64_t Tmax = 32768;
    float* gated = dalloc<float>((size_t) Tmax * D);
    int fails = 0;
    uint32_t seed = 101;
    for (const int64_t T : {1, 7, 16, 31, 32, 33, 48, 63, 64, 65, 96, 126, 127, 128, 129, 600, 1000, 2048, 4099, 8192, 32768}) fails += check(gm, st, gated, T, seed++);
    if (do_bench) {
        std::printf("--bench: the hyper-connection read's up projection + mix (medians of 20 alternating runs)\n");
        for (const int64_t T : {2048, 8192, 32768}) bench(gm, st, gated, T);
    }
    cudaFree(gated);
    std::printf("gr_upmix_parity: %d failures\n", fails);
    return fails ? 1 : 0;
}