Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
6.7 KB
Raw Blame
// src/prefill/gemm_f32_parity.cu - Pascal: Gemm::f16's opt-in FP32 route (cublasSgemm on FP32 copies of the FP16
// operands, STRATA_DENSE_F32=1) against the cuBLAS FP16-operand call it replaces, bitwise, over the dense projections'
// shapes and a sweep of chunk sizes T (GPU, synthetic, no model).  Whether the two agree is a property of the cuBLAS
// build (measured: 12.4.5.8 on a Tesla P40, 5,201 sizes): run this before setting STRATA_DENSE_F32=1.
//
//     build/gemm_f32_parity [--quick | --full] [--bench]
// default: every 7th T from the route's first to 8192, every T of 7000-7100 and around the powers of two, 8192;
// --quick: every 251st T; --full: every T.  --bench prints both times for a few T.  Exit 0: every output bitwise
// equal (or the route is not built here: not the experimental CUDA 12 engine).
#include "strata/prefill/gemm.hpp"

#include <cuda_fp16.h>
#include <cuda_runtime.h>

#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <set>
#include <string>
#include <vector>

namespace {
__global__ void fill_f16(__half* p, int64_t n, unsigned seed, float scale) {
    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
    if (i >= n) return;
    unsigned x = (unsigned) i * 2654435761u + seed;
    x ^= x >> 16; x *= 2246822519u; x ^= x >> 13; x *= 3266489917u; x ^= x >> 16;
    p[i] = __float2half(((float) (x & 0xffff) / 32768.0f - 1.0f) * scale);
}
__global__ void count_diff(const unsigned* a, const unsigned* b, int64_t n, unsigned* out) {
    const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
    if (i < n && a[i] != b[i]) atomicAdd(out, 1u);
}
bool ok(cudaError_t e, const char* what) {
    if (e != cudaSuccess) std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e));
    return e == cudaSuccess;
}
struct Shape { const char* what; int64_t N, K, t_min; };
}  // namespace

int main(int argc, char** argv) {
    setvbuf(stdout, nullptr, _IONBF, 0);
    bool quick = false, full = false, bench = false;
    for (int i = 1; i < argc; ++i) {
        const std::string a = argv[i];
        quick |= a == "--quick"; full |= a == "--full"; bench |= a == "--bench";
    }
    int dev = 0, maj = 0, min = 0;
    cudaGetDevice(&dev);
    cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev);
    cudaDeviceGetAttribute(&min, cudaDevAttrComputeCapabilityMinor, dev);
    std::printf("gemm_f32_parity: compute capability %d.%d\n", maj, min);
    cudaStream_t st = nullptr;
    if (!ok(cudaStreamCreate(&st), "stream")) return 1;
    strata::prefill::Gemm gemm;
    std::string err;
    if (!gemm.init(st, 0, err)) { std::printf("FAIL: %s\n", err.c_str()); return 1; }
    // the shapes the route takes and its first T for each (gemm.cu, dense_f32_min_t)
    const Shape shapes[] = {{"gdn qkv", 10240, 2560, 256}, {"qsa query", 12288, 2560, 256}, {"gdn gate", 6144, 2560, 512},
                            {"gdn out", 2560, 6144, 4096}, {"qsa out", 2560, 4096, 4096}};
    const int64_t TMAX = 8192;
    unsigned* cnt = nullptr;
    if (!ok(cudaMalloc((void**) &cnt, 4), "counter")) return 1;
    cudaEvent_t e0, e1;
    cudaEventCreate(&e0);
    cudaEventCreate(&e1);
    long long cases = 0, bad_cases = 0;
    for (const Shape& s : shapes) {
        const int64_t N = s.N, K = s.K;
        uint16_t *W = nullptr, *X = nullptr;
        float *Ya = nullptr, *Yb = nullptr;
        if (!ok(cudaMalloc((void**) &W, (size_t) N * K * 2), "W") || !ok(cudaMalloc((void**) &X, (size_t) TMAX * K * 2), "X") ||
            !ok(cudaMalloc((void**) &Ya, (size_t) TMAX * N * 4), "Ya") || !ok(cudaMalloc((void**) &Yb, (size_t) TMAX * N * 4), "Yb")) return 1;
        fill_f16<<<(unsigned) ((N * K + 255) / 256), 256, 0, st>>>((__half*) W, N * K, 1u + (unsigned) N, 0.05f);
        fill_f16<<<(unsigned) ((TMAX * K + 255) / 256), 256, 0, st>>>((__half*) X, TMAX * K, 7u + (unsigned) K, 2.0f);
        std::set<int64_t> ts;
        const int64_t step = full ? 1 : quick ? 251 : 7;
        for (int64_t t = s.t_min; t <= TMAX; t += step) ts.insert(t);
        ts.insert(s.t_min); ts.insert(TMAX);
        if (!quick) {
            for (int64_t t = 7000; t <= 7100; ++t) if (t >= s.t_min) ts.insert(t);
            for (int64_t p = 256; p <= TMAX; p *= 2) for (int64_t d = -3; d <= 3; ++d) if (p + d >= s.t_min && p + d <= TMAX) ts.insert(p + d);
            for (int64_t t = 6540; t <= 6570; ++t) if (t >= s.t_min) ts.insert(t);   // around the activation slices' first split (K 2560)
        }
        long long bad = 0, bad_outputs = 0;
        int64_t first_bad = 0;
        for (int64_t T : ts) {
            strata::prefill::gemm_set_dense_f32(0);
            gemm.f16(X, W, Ya, T, N, K);
            strata::prefill::gemm_set_dense_f32(1);
            gemm.f16(X, W, Yb, T, N, K);
            cudaMemsetAsync(cnt, 0, 4, st);
            count_diff<<<(unsigned) ((T * N + 255) / 256), 256, 0, st>>>((const unsigned*) Ya, (const unsigned*) Yb, T * N, cnt);
            unsigned nd = 0;
            cudaMemcpyAsync(&nd, cnt, 4, cudaMemcpyDeviceToHost, st);
            if (!ok(cudaStreamSynchronize(st), "sync")) return 1;
            ++cases;
            if (nd) { if (!bad) first_bad = T; ++bad; bad_outputs += nd; }
        }
        std::printf("%-10s %5lld x %-5lld: %zu sizes from T %lld to %lld, %lld with different bits%s\n", s.what, (long long) N, (long long) K, ts.size(),
                    (long long) *ts.begin(), (long long) *ts.rbegin(), bad, bad ? "" : " (bitwise equal)");
        if (bad) std::printf("FAIL %s: first at T %lld, %lld outputs in all\n", s.what, (long long) first_bad, bad_outputs);
        bad_cases += bad;
        if (bench) {
            for (int64_t T : {(int64_t) 560, (int64_t) 2048, (int64_t) 4096, (int64_t) 7040, (int64_t) 7041}) {
                if (T < s.t_min) continue;
                double ms[2] = {0, 0};
                for (int on = 0; on < 2; ++on) {
                    strata::prefill::gemm_set_dense_f32(on);
                    gemm.f16(X, W, Ya, T, N, K);
                    cudaEventRecord(e0, st);
                    for (int i = 0; i < 3; ++i) gemm.f16(X, W, Ya, T, N, K);
                    cudaEventRecord(e1, st);
                    cudaEventSynchronize(e1);
                    float t = 0;
                    cudaEventElapsedTime(&t, e0, e1);
                    ms[on] = t / 3;
                }
                std::printf("BENCH %-10s T %5lld | cuBLAS FP16 %7.2f ms | FP32 copies %7.2f ms (the conversions included) | %.2fx\n", s.what, (long long) T, ms[0],
                            ms[1], ms[0] / ms[1]);
            }
        }
        cudaFree(W); cudaFree(X); cudaFree(Ya); cudaFree(Yb);
    }
    strata::prefill::gemm_set_dense_f32(-1);
    std::printf("gemm_f32_parity: %lld sizes, %s\n", cases, bad_cases ? "FAILED" : "OK (bitwise)");
    return bad_cases ? 1 : 0;
}