Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
10.5 KB
Raw Blame
// src/kernels/native_expert_bench.cpp - the grouped native experts on real GGUF rows: one AMD layout against
// another, bit for bit, and each one's time.
//
//     build/native_expert_bench <shard1.gguf> <layer[,layer...]> <groups> <tokens per group> <ref mode> <mode> [iters]
//
// Modes are STRATA_EXP_MODE values.  A group is one expert (distinct blobs, ~2 MB each, so G >= 8 does not fit
// in L2); its entries read distinct tokens of an 8-token window.  The output must be bitwise equal to the
// reference mode's: the verify window's text depends on it.
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/artifact/gguf_reader.hpp"
#include "strata/kernels/cpu/native_expert.hpp"
#include "strata/kernels/iq_kernels.hpp"

#include "ggml.h"

#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <sstream>
#include <string>
#include <vector>

namespace cpu = strata::kernels::cpu;
namespace K = strata::kernels;

int main(int argc, char** argv) {
    setvbuf(stdout, nullptr, _IONBF, 0);
    if (argc < 7) {
        std::fprintf(stderr, "usage: native_expert_bench <shard1.gguf> <layer[,layer]> <groups> <tokens> <ref mode> <mode> [iters]\n");
        return 2;
    }
    strata::GgufFile gguf(argv[1]);
    std::vector<int> layers;
    {
        std::stringstream ss(argv[2]);
        std::string t;
        while (std::getline(ss, t, ',')) layers.push_back(std::atoi(t.c_str()));
    }
    const int G = std::atoi(argv[3]), T = std::atoi(argv[4]), mref = std::atoi(argv[5]), mode = std::atoi(argv[6]);
    const int iters = argc > 7 ? std::atoi(argv[7]) : 200;
    const int NTOK = 8;
    if (T < 1 || T > NTOK || G < 1) { std::fprintf(stderr, "tokens 1..8, groups >= 1\n"); return 2; }
    const int64_t H = 2560, FF_FULL = 640, FF = FF_FULL;   // Flash-Next's expert geometry
    int failures = 0;
    dpct::queue_ptr s;
    s = dpct::get_current_device().create_queue(true);
    // SYCL port: the queue carries an exception handler, which would swallow a failing launch and leave the timing
    // at "the kernel did nothing" (0.3 us for a group of experts).  Every wait goes through this, so a device-side
    // error is printed instead of silently producing an empty measurement.
    auto sync = [&](const char* what) {
        try {
            s->wait_and_throw();
            return true;
        } catch (std::exception const& e) {
            std::printf("%s: %s\n", what, e.what());
            return false;
        }
    };
    for (int l : layers) {
        const strata::TensorInfo* t[3] = {};
        const char* roles[3] = {"gate", "up", "down"};
        for (const auto& ti : gguf.tensors())
            for (int r = 0; r < 3; ++r)
                if (ti.name == "blk." + std::to_string(l) + ".ffn_" + roles[r] + "_exps.weight") t[r] = &ti;
        if (!t[0] || !t[1] || !t[2]) { std::printf("layer %d: no expert tensors\n", l); ++failures; continue; }
        cpu::NativeFmt f;
        std::string err;
        if (!cpu::native_fmt((int) t[0]->type, (int) t[2]->type, H, FF, f, err)) {
            std::printf("layer %d: %s\n", l, err.c_str()); ++failures; continue;
        }
        cpu::NativeFmt ff;    // the full expert's geometry, for strides into the GGUF
        if (!cpu::native_fmt((int) t[0]->type, (int) t[2]->type, H, FF_FULL, ff, err)) {
            std::printf("layer %d: %s\n", l, err.c_str()); ++failures; continue;
        }
        const size_t dsz = f.bytes - f.down_off, dsz_full = ff.bytes - ff.down_off;
        const size_t drow = dsz / H, drow_full = dsz_full / H;
        std::vector<uint8_t> blobs((size_t) G * f.bytes);
        for (int g = 0; g < G; ++g) {
            const size_t E = (size_t) ((g * 37 + 5) % 256);
            uint8_t* b = blobs.data() + (size_t) g * f.bytes;
            std::memcpy(b, gguf.tensor_data(*t[0]) + E * ff.up_off, f.up_off);
            std::memcpy(b + f.up_off, gguf.tensor_data(*t[1]) + E * ff.up_off, f.up_off);
            for (int64_t r = 0; r < H; ++r)
                std::memcpy(b + f.down_off + r * drow, gguf.tensor_data(*t[2]) + E * dsz_full + r * drow_full, drow);
        }
        std::mt19937 rng(17 + l);
        std::normal_distribution<float> nd(0.f, 1.f);
        std::vector<float> x((size_t) NTOK * H);
        for (auto& v : x) v = nd(rng);
        const int NE = G * T;
        std::vector<unsigned long long> ptr(G);
        std::vector<int32_t> start(G + 1), dst(NE), tok(NE);
        const auto L = K::native_expert_layout(f.gu_type, f.d_type, H, FF);
        // SYCL port: this bench was never a build target, so its CUDA calls were never migrated.  Device buffers are
        // USM allocations on the bench's own in-order queue, and every copy is ordered on that queue.
        uint8_t* dblob = (uint8_t*) sycl::malloc_device(blobs.size(), *s);
        void *dx = sycl::malloc_device((size_t) NTOK * H * 4, *s);
        void *dxq = sycl::malloc_device((size_t) NTOK * H / 32 * 36, *s);
        void *dscr = sycl::malloc_device(K::native_expert_scratch_bytes(NE, FF), *s);
        float* dout = (float*) sycl::malloc_device((size_t) NE * H * 4, *s);
        unsigned long long* dptr = (unsigned long long*) sycl::malloc_device((size_t) G * 8, *s);
        int32_t* dstart = (int32_t*) sycl::malloc_device((size_t) (G + 1) * 4, *s);
        int32_t* dn = (int32_t*) sycl::malloc_device(4, *s);
        int32_t* ddst = (int32_t*) sycl::malloc_device((size_t) NE * 4, *s);
        int32_t* dtok = (int32_t*) sycl::malloc_device((size_t) NE * 4, *s);
        for (int g = 0; g < G; ++g) {
            ptr[g] = (unsigned long long) (dblob + (size_t) g * f.bytes);
            start[g] = g * T;
            for (int j = 0; j < T; ++j) {
                dst[g * T + j] = g * T + j;
                tok[g * T + j] = (g * 3 + j) % NTOK;
            }
        }
        start[G] = NE;
        s->memcpy(dblob, blobs.data(), blobs.size()).wait();
        s->memcpy(dx, x.data(), x.size() * 4).wait();
        s->memcpy(dptr, ptr.data(), (size_t) G * 8).wait();
        s->memcpy(dstart, start.data(), (size_t) (G + 1) * 4).wait();
        s->memcpy(dn, &G, 4).wait();
        s->memcpy(ddst, dst.data(), (size_t) NE * 4).wait();
        s->memcpy(dtok, tok.data(), (size_t) NE * 4).wait();
        K::quantize_q8_1_rows((const float*) dx, NTOK, H, dxq, s);
        auto run = [&](int m, int phase) {
            K::native_expert_set_mode(m, phase);
            K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, G, NE, dxq, dscr, dout, s);
        };
        std::vector<float> ref((size_t) NE * H), got((size_t) NE * H);
        s->memset(dout, 0xff, got.size() * 4).wait();
        run(mref, 0);
        if (!sync("reference window")) return 1;

        s->memcpy(ref.data(), dout, ref.size() * 4).wait();
        // SYCL port: a launch that returns without enqueueing leaves the 0xff fill intact, and then the mode and the
        // reference agree bitwise on an untouched buffer - a "0 differing" that measures nothing.  The fill is NaN as
        // a float, so a reference window with no finite value anywhere did no work: say so instead of timing it.
        {
            size_t written = 0;
            for (size_t i = 0; i < ref.size(); ++i) {
                float v;
                std::memcpy(&v, &ref[i], 4);
                if (std::isfinite(v)) ++written;
            }
            if (written == 0) {
                std::printf("layer %d: the expert launch wrote nothing (%zu outputs, none finite) - refusing to time it\n",
                            (int) l, ref.size());
                ++failures;
                continue;
            }
        }
        s->memset(dout, 0xff, got.size() * 4).wait();
        run(mode, 0);
        if (!sync("mode window")) return 1;
        s->memcpy(got.data(), dout, got.size() * 4).wait();
        size_t ndiff = 0;
        double worst = 0;
        for (size_t i = 0; i < ref.size(); ++i)
            if (std::memcmp(&ref[i], &got[i], 4) != 0) {
                ++ndiff;
                worst = std::fmax(worst, std::fabs((double) ref[i] - got[i]) / (std::fabs((double) ref[i]) + 1e-6));
            }
        auto time = [&](int m, int phase) {
            for (int i = 0; i < 10; ++i) run(m, phase);
            s->wait();
            const auto t0 = std::chrono::steady_clock::now();
            for (int i = 0; i < iters; ++i) run(m, phase);
            sync("timing loop");
            const double ms =
                std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
            // SYCL port: the printout below and the GB/s it derives both assume MICROseconds ("all %7.1f us",
            // bytes / us / 1e3), but this returned milliseconds - so every published number was 1000x too small in
            // time and 1000x too large in bandwidth (0.1 "us" and 148,713 GB/s for a launch that takes ~100 us and
            // moves 19.7 MB at ~198 GB/s).  The wall clock always agreed with the real work: 200 iterations 618 ms,
            // 2,000 iterations 3,566 ms.  Return microseconds.
            return ms * 1e3 / iters;
        };
        const double gu_b = (double) G * 2 * f.up_off, d_b = (double) G * dsz;
        double us[2][3] = {{1e30, 1e30, 1e30}, {1e30, 1e30, 1e30}};
        for (int round = 0; round < 3; ++round)   // interleaved, the minimum: the clocks wander between runs
            for (int k = 0; k < 2; ++k)
                for (int p = 0; p < 3; ++p) us[k][p] = std::fmin(us[k][p], time(k ? mode : mref, p));
        std::printf("layer %2d %-7s/%-6s G %d T %d | mode %d: all %7.1f us  gu %7.1f us (%3.0f GB/s)  down %6.1f us (%3.0f GB/s)"
                    " | mode %d: all %7.1f us  gu %7.1f us (%3.0f GB/s)  down %6.1f us (%3.0f GB/s) | %s\n",
                    l, ggml_type_name((ggml_type) f.gu_type), ggml_type_name((ggml_type) f.d_type), G, T,
                    mref, us[0][0], us[0][1], gu_b / us[0][1] / 1e3, us[0][2], d_b / us[0][2] / 1e3,
                    mode, us[1][0], us[1][1], gu_b / us[1][1] / 1e3, us[1][2], d_b / us[1][2] / 1e3,
                    ndiff ? "DIFFERS" : "bitwise equal");
        if (ndiff) {
            std::printf("          %zu of %zu outputs differ, worst rel %.2e\n", ndiff, ref.size(), worst);
            ++failures;
        }
        K::native_expert_set_mode(-1, 0);
        for (void* p : {(void*) dblob, (void*) dx, (void*) dxq, (void*) dscr, (void*) dout, (void*) dptr,
                        (void*) dstart, (void*) dn, (void*) ddst, (void*) dtok})
            sycl::free(p, *s);
    }
    std::printf("native_expert_bench: %d differing\n", failures);
    return failures ? 1 : 0;
}