Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
7.9 KB
Raw Blame
// route_window_parity - the verify window's batched routing against the per-token calls, bit for bit.
//
// `moe_route_window` replaces n per-token (projection + top-k) pairs by one multi-column BF16 projection and one
// n-token top-k.  Speculative decoding is exact only if a token's route does not depend on how many tokens share
// its window, so both batched kernels must equal their one-token forms exactly - this checks that on random data
// shaped like the router (2560 -> 256, top-10) and like the shared-expert gate (2560 -> 1).
#define DPCT_PROFILING_ENABLED
#include <sycl/sycl.hpp>
#include <dpct/dpct.hpp>
#include "strata/kernels/bf16_gemv.hpp"
#include "strata/kernels/router_top10.hpp"

#include <cstdint>
#include <cstdio>
#include <cstring>
#include <random>
#include <vector>

namespace {
void ck(dpct::err0 e, const char *what) {
    /*
    DPCT1009: SYCL reports errors using exceptions and does not use error
    codes. Please replace the "get_error_string_dummy(...)" with a real
    error-handling function.
    */
}
uint16_t bf16(float f) { uint32_t i; std::memcpy(&i, &f, 4); return (uint16_t) ((i + ((i >> 16) & 1u) + 0x7FFFu) >> 16); }
}  // namespace

int main() {
    const int N = 2560, E = 256, K = 10;
    int bad = 0;
    std::mt19937 rng(7);
    std::normal_distribution<float> nd(0.0f, 1.0f);
    for (int n_out : {E, 1}) {
        for (int T : {2, 3, 4, 6}) {
            std::vector<uint16_t> w((size_t) n_out * N);
            std::vector<float> x((size_t) T * N);
            for (auto& v : w) v = bf16(nd(rng) * 0.05f);
            for (auto& v : x) v = nd(rng);
            uint16_t* dw; float *dx, *dy1, *dyn;
            ck(DPCT_CHECK_ERROR(dw = (uint16_t *)sycl::malloc_device(
                                    w.size() * 2, dpct::get_in_order_queue())),
               "w");
                ck(DPCT_CHECK_ERROR(
                       dx = (float *)sycl::malloc_device(
                           x.size() * 4, dpct::get_in_order_queue())),
                   "x");
            ck(DPCT_CHECK_ERROR(
                   dy1 = (float *)sycl::malloc_device(
                       (size_t)T * n_out * 4, dpct::get_in_order_queue())),
               "y1");
                ck(DPCT_CHECK_ERROR(
                       dyn = (float *)sycl::malloc_device(
                           (size_t)T * n_out * 4, dpct::get_in_order_queue())),
                   "yn");
            /*
            DPCT1114: cudaMemcpy is migrated to asynchronization memcpy,
            assuming in the original code the source host memory is pageable
            memory. If the memory is not pageable, call wait() on event return
            by memcpy API to ensure synchronization behavior.
            */
            ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue()).memcpy(
                   dw, w.data(), w.size() * 2).wait()),
               "up w");
            /*
            DPCT1114: cudaMemcpy is migrated to asynchronization memcpy,
            assuming in the original code the source host memory is pageable
            memory. If the memory is not pageable, call wait() on event return
            by memcpy API to ensure synchronization behavior.
            */
            ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue()).memcpy(
                   dx, x.data(), x.size() * 4).wait()),
               "up x");
            for (int t = 0; t < T; ++t)
                strata::kernels::bf16_gemv_fp32_mmvf(dx + (size_t) t * N, dw, dy1 + (size_t) t * n_out, N, n_out, nullptr);
            strata::kernels::bf16_gemv_fp32_mmvf_cols(dx, dw, dyn, N, n_out, T, nullptr);
            ck(DPCT_CHECK_ERROR(
                   dpct::get_current_device().queues_wait_and_throw()),
               "sync");
            std::vector<float> y1((size_t) T * n_out), yn((size_t) T * n_out);
            ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                    .memcpy(y1.data(), dy1, y1.size() * 4)
                                    .wait()),
               "down 1");
            ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                    .memcpy(yn.data(), dyn, yn.size() * 4)
                                    .wait()),
               "down n");
            const bool same = std::memcmp(y1.data(), yn.data(), y1.size() * 4) == 0;
            std::printf("  mmvf cols  n_out %3d  T %d  %s\n", n_out, T, same ? "bit-identical" : "*** DIFFERS ***");
            if (!same) ++bad;
            if (n_out == E) {
                int *i1, *in_; float *w1, *wn;
                ck(DPCT_CHECK_ERROR(
                       i1 = (int *)sycl::malloc_device(
                           (size_t)T * K * 4, dpct::get_in_order_queue())),
                   "i1");
                    ck(DPCT_CHECK_ERROR(
                           in_ = (int *)sycl::malloc_device(
                               (size_t)T * K * 4, dpct::get_in_order_queue())),
                       "in");
                ck(DPCT_CHECK_ERROR(
                       w1 = (float *)sycl::malloc_device(
                           (size_t)T * K * 4, dpct::get_in_order_queue())),
                   "w1");
                    ck(DPCT_CHECK_ERROR(
                           wn = (float *)sycl::malloc_device(
                               (size_t)T * K * 4, dpct::get_in_order_queue())),
                       "wn");
                for (int t = 0; t < T; ++t)
                    strata::kernels::router_top10(dy1 + (size_t) t * E, 1, E, K, i1 + t * K, w1 + t * K, nullptr);
                strata::kernels::router_top10(dy1, T, E, K, in_, wn, nullptr);
                ck(DPCT_CHECK_ERROR(
                       dpct::get_current_device().queues_wait_and_throw()),
                   "sync top");
                std::vector<int> a((size_t) T * K), b((size_t) T * K);
                std::vector<float> fa((size_t) T * K), fb((size_t) T * K);
                ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                        .memcpy(a.data(), i1, a.size() * 4)
                                        .wait()),
                   "ids1");
                ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                        .memcpy(b.data(), in_, b.size() * 4)
                                        .wait()),
                   "idsn");
                ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                        .memcpy(fa.data(), w1, fa.size() * 4)
                                        .wait()),
                   "w1");
                ck(DPCT_CHECK_ERROR((dpct::get_current_device().queues_wait_and_throw(), dpct::get_in_order_queue())
                                        .memcpy(fb.data(), wn, fb.size() * 4)
                                        .wait()),
                   "wn");
                const bool s2 = a == b && std::memcmp(fa.data(), fb.data(), fa.size() * 4) == 0;
                std::printf("  top10 n    T %d            %s\n", T, s2 ? "bit-identical" : "*** DIFFERS ***");
                if (!s2) ++bad;
                sycl::free(i1, dpct::get_in_order_queue());
                    sycl::free(in_, dpct::get_in_order_queue());
                    sycl::free(w1, dpct::get_in_order_queue());
                    sycl::free(wn, dpct::get_in_order_queue());
            }
            sycl::free(dw, dpct::get_in_order_queue());
                sycl::free(dx, dpct::get_in_order_queue());
                sycl::free(dy1, dpct::get_in_order_queue());
                sycl::free(dyn, dpct::get_in_order_queue());
        }
    }
    std::printf(bad ? "route_window_parity: %d FAILURES\n" : "route_window_parity OK\n", bad);
    return bad ? 1 : 0;
}