Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
8.0 KB
Raw Blame
// src/kernels/nvfp4_decode_bench.cpp - the decode path's grouped GPU experts (native_expert_grouped) timed on random
// blobs of a format pair, as weight bytes per second against the card's DRAM bandwidth, plus a hash of the output so
// a kernel change can be checked bit for bit.
//
//     nvfp4_decode_bench [gu_type d_type] [groups] [tokens per group] [iters]
//
// Defaults: 40 40 (NVFP4), 64 groups (distinct ~2.7 MB blobs: far past L2), 2 tokens each, 50 iterations.  UD-IQ4_XS's
// experts: 21 20 (IQ3_S gate/up, IQ4_NL down).  Random bytes: the values are meaningless, the timing is not (the
// kernels have no data-dependent branches but the NVFP4 scale decode), and the hash is stable for a given build.
#include "strata/kernels/iq_kernels.hpp"

#include <cuda_runtime.h>

#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <vector>

namespace k = strata::kernels;

namespace {
void ck(cudaError_t e, const char* what) {
    if (e != cudaSuccess) { std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e)); std::exit(1); }
}
}  // namespace

int main(int argc, char** argv) {
    {   // the UE4M3 decode built from bits (iq_kernels.cu) against the ldexpf form it replaced, all 256 bytes
        int bad = 0;
        for (int x = 0; x < 256; ++x) {
            const int e = (x >> 3) & 0xF, m = x & 7;
            const float old = (x == 0 || x == 0x7F) ? 0.0f
                              : (e == 0 ? std::ldexp((float) m, -9) : std::ldexp(1.0f + (float) m / 8.0f, e - 7)) * 0.5f;
            uint32_t bits = ((uint32_t) (e + 119) << 23) | ((uint32_t) m << 20);
            float nw;
            std::memcpy(&nw, &bits, 4);
            if (x == 0 || x == 0x7F) nw = 0.0f;
            else if (e == 0) nw = (float) m * 0.0009765625f;
            bad += std::memcmp(&old, &nw, 4) != 0;
        }
        std::printf("ue4m3 decode: %d of 256 bytes differ from the ldexpf form\n", bad);
        if (bad) return 1;
    }
    const int gu = argc > 2 ? std::atoi(argv[1]) : 40, dt = argc > 2 ? std::atoi(argv[2]) : 40;
    const int G = argc > 3 ? std::atoi(argv[3]) : 64, TPG = argc > 4 ? std::atoi(argv[4]) : 2;
    const int IT = argc > 5 ? std::atoi(argv[5]) : 50;
    constexpr int64_t N = 2560, FF = 640;
    if (!k::native_expert_supported(gu, dt, N, FF)) { std::fprintf(stderr, "types %d/%d not supported\n", gu, dt); return 2; }
    const k::NativeExpertLayout L = k::native_expert_layout(gu, dt, N, FF);
    const int E = G * TPG, T = 8;                      // entries; tokens of the window (activations)
    std::mt19937 rng(20261012);
    std::vector<uint8_t> blobs((size_t) G * L.bytes);
    for (auto& b : blobs) b = (uint8_t) (rng() & 0xFF);
    if (gu == 40 || dt == 40) {                        // NVFP4: UE4M3 scales in the checkpoint's range, sane tails
        for (int g = 0; g < G; ++g) {
            uint8_t* b = blobs.data() + (size_t) g * L.bytes;
            auto fix = [&](size_t off, size_t bytes) {
                for (size_t o = off; o + 36 <= off + bytes; o += 36)
                    for (int s = 0; s < 4; ++s) b[o + s] = (uint8_t) (0x20 + rng() % 0x40);
            };
            if (gu == 40) fix(0, L.down_off);
            if (dt == 40) fix(L.down_off, (size_t) N * L.d_row);
            if (L.tail_off) {
                const float t[4] = {8e-5f, 8e-5f, 1e-4f, 0.f};
                std::memcpy(b + L.tail_off, t, sizeof t);
            }
        }
    } else {                                           // the other formats: fp16 block scales kept finite (|d| < 2)
        for (size_t i = 1; i < blobs.size(); i += 2) blobs[i] &= 0x3B;
    }
    // NVFP4_BENCH_FILE=<experts.bin or its head>: the blobs are read from it (the first G whole blobs, cycled when it
    // holds fewer), real GPTQ / ModelOpt weights instead of random ones
    if (const char* fp = std::getenv("NVFP4_BENCH_FILE")) {
        FILE* f = std::fopen(fp, "rb");
        if (!f) { std::fprintf(stderr, "cannot open %s\n", fp); return 2; }
        int have = 0;
        while (have < G && std::fread(blobs.data() + (size_t) have * L.bytes, 1, L.bytes, f) == L.bytes) ++have;
        std::fclose(f);
        if (have == 0) { std::fprintf(stderr, "%s holds no whole blob\n", fp); return 2; }
        for (int g = have; g < G; ++g)
            std::memcpy(blobs.data() + (size_t) g * L.bytes, blobs.data() + (size_t) (g % have) * L.bytes, L.bytes);
        std::printf("blobs: %d real from %s (cycled to %d)\n", have, fp, G);
    }
    std::normal_distribution<float> nd(0.f, 1.f);
    std::vector<float> x((size_t) T * N);
    for (float& v : x) v = nd(rng);
    std::vector<unsigned long long> ptr(G);
    std::vector<int32_t> start(G + 1), dst(E), tok(E);
    uint8_t* db;
    ck(cudaMalloc(&db, blobs.size()), "blobs");
    ck(cudaMemcpy(db, blobs.data(), blobs.size(), cudaMemcpyHostToDevice), "blobs");
    for (int g = 0; g < G; ++g) {
        ptr[g] = (unsigned long long) (db + (size_t) g * L.bytes);
        start[g] = g * TPG;
        for (int j = 0; j < TPG; ++j) { dst[g * TPG + j] = (g * TPG + j) % T + T * ((g * TPG + j) / T); tok[g * TPG + j] = (g + j) % T; }
    }
    start[G] = E;
    for (int e = 0; e < E; ++e) dst[e] = e;            // one output row per entry (no summation races in the hash)
    float *dx, *dout; void *dxq, *scr; unsigned long long* dptr; int32_t *dstart, *dng, *ddst, *dtok;
    ck(cudaMalloc(&dx, x.size() * 4), "x");
    ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");
    ck(cudaMalloc(&dxq, (size_t) T * N / 32 * 36), "xq");
    ck(cudaMalloc(&dout, (size_t) E * N * 4), "out");
    ck(cudaMalloc(&scr, k::native_expert_scratch_bytes(E, FF)), "scratch");
    ck(cudaMalloc(&dptr, 8 * G), "p");
    ck(cudaMemcpy(dptr, ptr.data(), 8 * G, cudaMemcpyHostToDevice), "p");
    ck(cudaMalloc(&dstart, 4 * (G + 1)), "s");
    ck(cudaMemcpy(dstart, start.data(), 4 * (G + 1), cudaMemcpyHostToDevice), "s");
    ck(cudaMalloc(&dng, 4), "n");
    ck(cudaMemcpy(dng, &G, 4, cudaMemcpyHostToDevice), "n");
    ck(cudaMalloc(&ddst, 4 * E), "d");
    ck(cudaMemcpy(ddst, dst.data(), 4 * E, cudaMemcpyHostToDevice), "d");
    ck(cudaMalloc(&dtok, 4 * E), "t");
    ck(cudaMemcpy(dtok, tok.data(), 4 * E, cudaMemcpyHostToDevice), "t");
    k::quantize_q8_1_rows(dx, T, N, dxq, nullptr);
    // NVFP4_BENCH_PHASE=1: gate/up + SwiGLU + quantize only, 2: down only (native_expert_set_mode), 0/unset: all
    if (const char* ph = std::getenv("NVFP4_BENCH_PHASE")) k::native_expert_set_mode(-1, std::atoi(ph));
    auto run = [&] { k::native_expert_grouped(L, dptr, dstart, dng, ddst, dtok, G, E, dxq, scr, dout, nullptr); };
    run();
    ck(cudaDeviceSynchronize(), "warm");
    std::vector<float> out((size_t) E * N);
    ck(cudaMemcpy(out.data(), dout, out.size() * 4, cudaMemcpyDeviceToHost), "out");
    uint64_t h = 1469598103934665603ull;               // FNV-1a over the output bytes
    for (const float& v : out) {
        const auto* p = reinterpret_cast<const uint8_t*>(&v);
        for (int i = 0; i < 4; ++i) h = (h ^ p[i]) * 1099511628211ull;
    }
    cudaEvent_t a, b;
    cudaEventCreate(&a);
    cudaEventCreate(&b);
    float best = 1e30f, sum = 0.f;
    for (int i = 0; i < IT; ++i) {
        cudaEventRecord(a);
        run();
        cudaEventRecord(b);
        cudaEventSynchronize(b);
        float ms = 0.f;
        cudaEventElapsedTime(&ms, a, b);
        best = ms < best ? ms : best;
        sum += ms;
    }
    ck(cudaGetLastError(), "run");
    int dev = 0, bus = 0, clk = 0;
    cudaGetDevice(&dev);
    cudaDeviceGetAttribute(&bus, cudaDevAttrGlobalMemoryBusWidth, dev);
    cudaDeviceGetAttribute(&clk, cudaDevAttrMemoryClockRate, dev);   // kHz
    const double peak = 2.0 * clk * 1e3 * (bus / 8) / 1e9;            // GB/s, DDR
    const double bytes = (double) G * (double) L.bytes;
    std::printf("types %d/%d  blob %zu B  groups %d x %d tokens: best %.3f ms  mean %.3f ms  -> %.1f GB/s of weights "
                "(%.0f%% of the %.0f GB/s DRAM peak)  out hash %016llx\n", gu, dt, L.bytes, G, TPG, best, sum / IT,
                bytes / (best * 1e6), 100.0 * bytes / (best * 1e6) / peak, peak, (unsigned long long) h);
    return 0;
}