Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
9.2 KB
Raw Blame
// esimd_iq3s_decode_bench (151-b70): IQ3_S rows decoded to DPAS B fragments (int8) by an ESIMD kernel, lanes = 16 rows.
//   esimd_iq3s_decode_bench [N=1280] [K=2560] [experts=16] [write=1]
// Output per expert: Wq [N/16][K/32][8][16] dwords (the B tile of a DPAS s8 8x16x32: dword r of row n at r*16+n), sw [N/16][K/32][16] floats
// (d * (1 + 2 s) per row and 32-block).  Checked against a host decode with ggml's own grid.  Reports us per expert and GB/s of
// compressed weights consumed.  write=0 keeps the fragments in registers (an xor checksum) to separate decode from the plane write.
#include <sycl/sycl.hpp>
#include <sycl/ext/intel/esimd.hpp>
#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
#include <sycl/ext/intel/experimental/esimd/memory.hpp>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <vector>
#include <utility>
#include <type_traits>

#define GGML_COMMON_DECL_CPP
#define GGML_COMMON_IMPL_CPP
#include "ggml-common.h"

using namespace sycl::ext::intel::esimd;

constexpr int BLK = 110;   // sizeof(block_iq3_s)

template <int N, class F>
SYCL_ESIMD_FUNCTION __attribute__((always_inline)) inline void sfor(F&& f) {
    [&]<int... I>(std::integer_sequence<int, I...>) SYCL_ESIMD_FUNCTION {
        (f(std::integral_constant<int, I>{}), ...);
    }(std::make_integer_sequence<int, N>{});
}

// unaligned 32-bit read at a 2-byte aligned byte offset (per lane) through two dword gathers
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> ld32(const uint8_t* base, simd<uint32_t, 16> off) {
    simd<uint32_t, 16> a = off & ~3u;
    simd<uint32_t, 16> lo = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a);
    simd<uint32_t, 16> hi = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a + 4u);
    simd<uint32_t, 16> sh = (off & 2u) << 3;   // 0 or 16
    return (lo >> sh) | ((hi << 16) << (16u - sh));
}
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> ld16(const uint8_t* base, simd<uint32_t, 16> off) {
    simd<uint32_t, 16> a = off & ~3u;
    simd<uint32_t, 16> lo = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a);
    return (lo >> ((off & 2u) << 3)) & 0xffffu;
}

template <int WRITE>
void run(sycl::queue& q, const uint8_t* W, const uint32_t* grid, uint32_t* Wq, float* sw, uint32_t* sink, int N, int K, int E) {
    const int nsb = K / 256, ntiles = N / 16, rowb = nsb * BLK;
    const size_t nthr = (size_t) E * ntiles;
    auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<128>};
    q.submit([&](sycl::handler& h) {
        h.parallel_for(sycl::range<1>(nthr), props, [=](sycl::id<1> id) SYCL_ESIMD_KERNEL {
            const int t = (int) id[0], nt = t % ntiles, e = t / ntiles;
            const uint8_t* We = W + (size_t) e * N * rowb;
            simd<uint32_t, 16> rowoff = (simd<uint32_t, 16>(0, 1) + (uint32_t) (nt * 16)) * (uint32_t) rowb;
            simd<uint32_t, 16> chk = 0;
            for (int sb = 0; sb < nsb; ++sb) {
                const uint32_t P = sb * BLK;
                simd<uint32_t, 16> d16 = ld16(We, rowoff + P);
                simd<uint16_t, 16> dlo = d16.template bit_cast_view<uint16_t>().template select<16, 2>(0);
                simd<sycl::half, 16> dh16 = dlo.template bit_cast_view<sycl::half>();
                simd<float, 16> d = dh16;
                simd<uint32_t, 16> sc4 = ld32(We, rowoff + (P + 106));
                simd<uint32_t, 16> qh4 = ld32(We, rowoff + (P + 66));
                simd<uint32_t, 16> qh8 = ld32(We, rowoff + (P + 70));
#pragma unroll
                for (int ib = 0; ib < 8; ++ib) {
                    simd<uint32_t, 16> q0 = ld32(We, rowoff + (P + 2 + 8 * ib));
                    simd<uint32_t, 16> q1 = ld32(We, rowoff + (P + 6 + 8 * ib));
                    simd<uint32_t, 16> sg = ld32(We, rowoff + (P + 74 + 4 * ib));
                    simd<uint32_t, 16> qhv = ((ib < 4 ? qh4 : qh8) >> (8 * (ib & 3))) & 0xffu;
                    simd<uint32_t, 16> scn = ((sc4 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;
                    simd<float, 16> swv = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));
                    simd<uint32_t, 128> frag;
#pragma unroll
                    for (int r = 0; r < 8; ++r) {
                        simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
                        simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);
                        simd<uint32_t, 16> g = gather<uint32_t, 16>(grid, idx << 2);
                        simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;
                        simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);
                        simd<uint32_t, 16> mask = m * 255u;
                        frag.template select<16, 1>(16 * r) = (g ^ mask) + m;
                    }
                    if constexpr (WRITE) {
                        const size_t fi = ((size_t) e * ntiles + nt) * (K / 32) + sb * 8 + ib;
                        block_store<uint32_t, 128>(Wq + fi * 128, frag);
                        block_store<float, 16>(sw + fi * 16, swv);
                    } else {
                        simd<uint32_t, 16> x = swv.template bit_cast_view<uint32_t>();
#pragma unroll
                        for (int r = 0; r < 8; ++r) x ^= frag.template select<16, 1>(16 * r);
                        chk ^= x;
                    }
                }
            }
            if constexpr (!WRITE) block_store<uint32_t, 16>(sink + (size_t) t * 16, chk);
        });
    });
}

int main(int argc, char** argv) {
    const int N = argc > 1 ? std::atoi(argv[1]) : 1280, K = argc > 2 ? std::atoi(argv[2]) : 2560;
    const int E = argc > 3 ? std::atoi(argv[3]) : 16, WR = argc > 4 ? std::atoi(argv[4]) : 1;
    sycl::queue q{sycl::gpu_selector_v, sycl::property::queue::in_order()};
    std::printf("device %s, N %d K %d, %d experts, write %d\n", q.get_device().get_info<sycl::info::device::name>().c_str(), N, K, E, WR);
    const int nsb = K / 256, ntiles = N / 16, rowb = nsb * BLK;
    std::mt19937 rng(11);
    std::vector<uint8_t> hW((size_t) E * N * rowb);
    for (size_t i = 0; i < hW.size(); ++i) hW[i] = (uint8_t) rng();
    for (size_t b = 0; b < hW.size() / BLK; ++b) {   // sane fp16 d
        const sycl::half dh = sycl::half(0.002f + 0.0001f * (rng() % 50));
        std::memcpy(&hW[b * BLK], &dh, 2);
    }
    uint8_t* dW = sycl::malloc_device<uint8_t>(hW.size() + 4096, q);
    uint32_t* dgrid = sycl::malloc_device<uint32_t>(512, q);
    uint32_t* dWq = sycl::malloc_device<uint32_t>((size_t) E * ntiles * (K / 32) * 128, q);
    float* dsw = sycl::malloc_device<float>((size_t) E * ntiles * (K / 32) * 16, q);
    uint32_t* dsink = sycl::malloc_device<uint32_t>((size_t) E * ntiles * 16, q);
    q.memcpy(dW, hW.data(), hW.size()); q.memcpy(dgrid, iq3s_grid, 512 * 4); q.wait();
    auto go = [&] { if (WR) run<1>(q, dW, dgrid, dWq, dsw, dsink, N, K, E); else run<0>(q, dW, dgrid, dWq, dsw, dsink, N, K, E); q.wait(); };
    go();
    double best = 1e30;
    for (int r = 0; r < 10; ++r) {
        const auto t0 = std::chrono::steady_clock::now();
        go();
        best = std::min(best, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());
    }
    std::printf("decode: %.1f us per expert (%.0f GB/s of compressed weights)\n", best / E, (double) N * rowb / (best / E) / 1e3);
    if (WR) {   // host check of expert 1
        const int e = E > 1 ? 1 : 0;
        std::vector<uint32_t> hQ((size_t) ntiles * (K / 32) * 128);
        std::vector<float> hS((size_t) ntiles * (K / 32) * 16);
        q.memcpy(hQ.data(), dWq + (size_t) e * hQ.size(), hQ.size() * 4).wait();
        q.memcpy(hS.data(), dsw + (size_t) e * hS.size(), hS.size() * 4).wait();
        long bad = 0, badS = 0;
        for (int n = 0; n < N; n += 3) {
            for (int sb = 0; sb < nsb; ++sb) {
                const uint8_t* blk = &hW[((size_t) e * N + n) * rowb + (size_t) sb * BLK];
                sycl::half dh; std::memcpy(&dh, blk, 2);
                const uint8_t *qs = blk + 2, *qh = blk + 66, *sgn = blk + 74, *scl = blk + 106;
                for (int ib = 0; ib < 8; ++ib) {
                    const int sc = (scl[ib / 2] >> (4 * (ib % 2))) & 0xf;
                    const float swr = (float) dh * (1 + 2 * sc);
                    const size_t fi = ((size_t) (n / 16) * (K / 32) + sb * 8 + ib);
                    if (std::fabs(hS[fi * 16 + n % 16] - swr) > 1e-6f * std::fabs(swr)) ++badS;
                    for (int r = 0; r < 8; ++r) {
                        const int idx = qs[8 * ib + r] | (((qh[ib] >> r) & 1) << 8);
                        const uint8_t* g = (const uint8_t*) &iq3s_grid[idx];
                        const uint8_t sbyte = sgn[4 * ib + r / 2];
                        uint32_t want = 0;
                        for (int j = 0; j < 4; ++j) {
                            const int kk = 4 * (r & 1) + j;
                            const int v = (sbyte >> kk & 1) ? -(int) g[j] : (int) g[j];
                            want |= (uint32_t) (uint8_t) (int8_t) v << (8 * j);
                        }
                        if (hQ[fi * 128 + r * 16 + n % 16] != want) ++bad;
                    }
                }
            }
        }
        std::printf("check expert %d: %ld bad dwords, %ld bad scales\n", e, bad, badS);
    }
    return 0;
}