Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// 151-b70: fused int8 DPAS expert products (ESIMD, Battlemage) - see include/strata/prefill/esimd_moe.hpp and
// docs/INTEL_INT8_EXPERTS.md (the behaviour note).
//
// * Tile. One hardware thread owns 16 output rows (a DPAS N tile; for gate/up the 16 virtual rows alternate gate/up) and up to
// MT x 8 activation rows (MT = 2, 4 or 8, picked per group from the experts' mean row count). It walks K in steps of 32:
// decode the 16 weight rows' 32 values into the B operand (registers only), then for each of its row tiles one s8 DPAS 8x16x32
// into a zeroed int32 tile, converted to float and scaled by (weight scale of the row and 32-block) x (activation scale of the
// row and 32-block) into a float accumulator. There is no shared memory and no barrier.
// * Decode once per tile: the B operand is decoded once per thread and applied to all its MT row tiles. An expert with more rows
// than 8 x MT is split over row-block threads, each decoding again. A thread's accumulators are MT tiles of 512 bytes in an
// 8 KB register file (256 GRF of 32 bytes): MT 12, or two output tiles per thread, spilled (3-15 KB) and ran 2-5x slower.
// * Scales. Activations: q8_1 as llama.cpp (d = amax / 127, q = round half away from zero), one float per row and 32 of K.
// Weights: IQ3_S d (1 + 2 s), IQ3_XXS d (2 s + 1) / 4, IQ2_S d (2 s + 1) / 8 per 16 (two DPAS per 32 with the other half
// zeroed), IQ4_NL d per 32, Q2_0 d per 64; the int8 values are the codebook entries with their signs, never rounded. The int32
// dot product of a 32-block is exact and exactly convertible to float. The float accumulation order is fixed (K ascending).
// * Edge cases. Rows are padded to tiles of 8 with zeros (stores are masked to the real rows); a group of up to 16 experts per
// launch; K a multiple of 256 (IQ3_S / IQ3_XXS / IQ2_S) / 64 (Q2_0) / 32 (IQ4_NL); output rows a multiple of 16. Blob reads
// never go past a matrix end.
// * Fallback. available() is false without the s8 8x16x32 DPAS (a non-Battlemage device) and the caller keeps the dequantize +
// oneMKL path; gate_up()/down() return false for a type / shape they do not handle.
// * ESIMD on this compiler (oneAPI 2026.1, bmg-g31): a dynamic index into the kernel-argument arrays, and `/` or `%` by a run-time
// value, gave wrong values / garbage pointers (the first weight read faulted the GPU); the kernels use constant-index reads
// (pick_expert) and 2-D grids. gather<T, N, VS> with VS > 1 is not available on Battlemage.
#include <sycl/sycl.hpp>
#include <sycl/ext/intel/esimd.hpp>
#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
#include <sycl/ext/oneapi/matrix/matrix.hpp>
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <mutex>
#include <type_traits>
#include <string>
#include <vector>
#include <unordered_map>
#include "strata/sycl_queue.hpp"
#include "strata/prefill/esimd_moe.hpp"
#ifdef STRATA_I8_NO_ESIMD
// an AOT build for a device without the s8 8x16x32 DPAS (Alchemist, Xe-HPC): no ESIMD kernels, available() is false
namespace strata::prefill::i8 {
void make_layout(Layout& lo, const int32_t* cnt, int n) {
lo.n = n;
int r = 0, t = 0;
for (int i = 0; i < n; ++i) { lo.cnt[i] = cnt[i]; lo.row0[i] = r; lo.tile0[i] = t; r += cnt[i]; t += (cnt[i] + 7) / 8; }
lo.row0[n] = r;
lo.tile0[n] = t;
}
bool available(void*) { return false; }
bool battlemage(void*) { return false; }
bool any_battlemage() { return false; }
size_t xq_bytes(const Layout& lo, int K) { return (size_t) lo.tile0[lo.n] * 8 * K; }
size_t dx_bytes(const Layout& lo, int K) { return (size_t) lo.tile0[lo.n] * (K / 32) * 8 * sizeof(float); }
bool workspace(const Layout&, int, int, Workspace&, void*) { return false; }
void quantize_rows(const uint16_t*, const Layout&, int, int8_t*, float*, void*) {}
bool gate_up(int, const uint8_t* const*, size_t, int, const Layout&, int, const int8_t*, const float*, float*, void*) { return false; }
void swiglu_quantize(const float*, const Layout&, int, int8_t*, float*, void*) {}
bool down(int, const uint8_t* const*, size_t, int, const Layout&, int, const int8_t*, const float*, float*, void*) { return false; }
} // namespace strata::prefill::i8
#else
namespace strata::prefill::i8 {
namespace {
#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {
#define GGML_TABLE_END() };
#include "iq_tables.inc"
#undef GGML_TABLE_BEGIN
#undef GGML_TABLE_END
using namespace sycl::ext::intel::esimd;
namespace xmx = sycl::ext::intel::esimd::xmx;
using half = sycl::half;
// ggml types handled
constexpr int T_IQ3XXS = 18, T_IQ4NL = 20, T_IQ3S = 21, T_IQ2S = 22, T_Q2_0 = 42;
template <int F> constexpr int blk_bytes() { return F == T_IQ3S ? 110 : F == T_IQ3XXS ? 98 : F == T_IQ2S ? 82 : F == T_IQ4NL ? 18 : 18; }
constexpr int block_bytes_of(int f) { return f == T_IQ3S ? 110 : f == T_IQ3XXS ? 98 : f == T_IQ2S ? 82 : 18; }
constexpr int block_vals_of(int f) { return (f == T_IQ3S || f == T_IQ3XXS || f == T_IQ2S) ? 256 : f == T_Q2_0 ? 64 : 32; }
// One device table buffer (uint32 units): the codebooks the decoders gather from.
constexpr int O_IQ3S = 0; // iq3s_grid[512]
constexpr int O_IQ3XXS = 512; // iq3xxs_grid[256]
constexpr int O_IQ2S = 768; // iq2s_grid[1024] as 2048 dwords (low dword of an entry first)
constexpr int O_KSIGNS = 2816; // ksigns_iq2xs[128], one byte in the low byte of each dword
constexpr int O_IQ4 = 2944; // [256]: byte b -> kvalues_iq4nl[b & 15] in byte 0, kvalues_iq4nl[b >> 4] in byte 1
constexpr int O_Q2 = 3200; // [256]: byte b -> its four 2-bit codes minus 1, as four int8 (Q2_0 values)
constexpr int N_TABLE = 3456;
uint32_t* tables(sycl::queue& q) {
static std::mutex mu;
static std::unordered_map<sycl::device, uint32_t*> per_dev; // a layer split has one prompt path per card
std::lock_guard<std::mutex> lk(mu);
uint32_t*& t = per_dev[q.get_device()];
if (!t) {
std::vector<uint32_t> h(N_TABLE, 0);
for (int i = 0; i < 512; ++i) h[O_IQ3S + i] = iq3s_grid[i];
for (int i = 0; i < 256; ++i) h[O_IQ3XXS + i] = iq3xxs_grid[i];
for (int i = 0; i < 1024; ++i) { h[O_IQ2S + 2 * i] = (uint32_t) iq2s_grid[i]; h[O_IQ2S + 2 * i + 1] = (uint32_t) (iq2s_grid[i] >> 32); }
for (int i = 0; i < 128; ++i) h[O_KSIGNS + i] = ksigns_iq2xs[i];
for (int b = 0; b < 256; ++b) {
h[O_IQ4 + b] = (uint32_t) (uint8_t) kvalues_iq4nl[b & 15] | ((uint32_t) (uint8_t) kvalues_iq4nl[b >> 4] << 8);
uint32_t q2 = 0;
for (int k = 0; k < 4; ++k) q2 |= (uint32_t) (uint8_t) (int8_t) (((b >> (2 * k)) & 3) - 1) << (8 * k);
h[O_Q2 + b] = q2;
}
uint32_t* d = sycl::malloc_device<uint32_t>(N_TABLE, q);
q.memcpy(d, h.data(), N_TABLE * 4);
q.wait();
t = d;
}
return t;
}
// unaligned 32-bit read at a 2-byte aligned byte offset (per lane) through two dword gathers
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> ld32(const uint8_t* base, simd<uint32_t, 16> off) {
simd<uint32_t, 16> a = off & ~3u;
simd<uint32_t, 16> lo = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a);
simd<uint32_t, 16> hi = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a + 4u);
simd<uint32_t, 16> sh = (off & 2u) << 3; // 0 or 16
return (lo >> sh) | ((hi << 16) << (16u - sh));
}
// the same with the second dword clamped to byte offset `lim` (the last dword of the matrix): the down matrix ends the expert blob, and a block
// whose last field is not dword aligned would otherwise read up to 3 bytes past it
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> ld32c(const uint8_t* base, simd<uint32_t, 16> off, uint32_t lim) {
simd<uint32_t, 16> a = off & ~3u;
simd<uint32_t, 16> a2 = a + 4u;
a2.merge(simd<uint32_t, 16>(lim), a2 > lim);
simd<uint32_t, 16> lo = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a);
simd<uint32_t, 16> hi = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a2);
simd<uint32_t, 16> sh = (off & 2u) << 3;
return (lo >> sh) | ((hi << 16) << (16u - sh));
}
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> ld16(const uint8_t* base, simd<uint32_t, 16> off) {
simd<uint32_t, 16> a = off & ~3u;
simd<uint32_t, 16> lo = gather<uint32_t, 16>(reinterpret_cast<const uint32_t*>(base), a);
return (lo >> ((off & 2u) << 3)) & 0xffffu;
}
SYCL_ESIMD_FUNCTION inline simd<float, 16> half_of(simd<uint32_t, 16> h16) {
simd<uint16_t, 16> lo = h16.template bit_cast_view<uint16_t>().template select<16, 2>(0);
simd<half, 16> hh = lo.template bit_cast_view<half>();
simd<float, 16> f = hh;
return f;
}
SYCL_ESIMD_FUNCTION inline simd<int, 128> dpas_s8(simd<uint32_t, 128> b, simd<int, 64> a) {
simd<int, 128> bi = b.template bit_cast_view<int>();
return xmx::dpas<8, 8, int, int, int, int, xmx::dpas_argument_type::s8, xmx::dpas_argument_type::s8>(simd<int, 128>(0), bi, a);
}
// sign nibble -> four negation bytes: (g ^ mask) + m negates the bytes whose sign bit is set (g is never 0xFF per byte)
SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> apply_signs(simd<uint32_t, 16> g, simd<uint32_t, 16> nib) {
simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);
simd<uint32_t, 16> mask = m * 255u;
return (g ^ mask) + m;
}
struct GArgs {
const uint8_t* blob[kMaxGroup];
uint32_t sec_off; // gate/up: byte offset of the up rows; down: byte offset of the down rows
int n;
int row0[kMaxGroup + 1];
int tile0[kMaxGroup + 1];
int rb0[kMaxGroup + 1]; // first row block of expert i
const int8_t* Xq;
const float* dx;
float* Y;
const uint32_t* tab;
int ntiles; // output tiles of 16
int nkb; // K / 32
int rowb; // bytes per weight row
int ldy;
uint32_t lim; // byte offset of the last dword of the matrix the clamped reads (ld32c) stay inside
int dbg; // bring-up switches (STRATA_I8_DBG): 1 skip the decode, 2 skip the activation loads, 4 skip the stores, 8 skip the K loop
};
// The expert a work item belongs to, with constant-index reads of the kernel argument arrays.
struct Pick { int e, row0, cnt, tile0, rb0; const uint8_t* blob; };
SYCL_ESIMD_FUNCTION inline Pick pick_expert(const GArgs& a, int rbg) {
Pick p{0, a.row0[0], a.row0[1] - a.row0[0], a.tile0[0], a.rb0[0], a.blob[0]};
#pragma unroll
for (int i = 1; i < kMaxGroup; ++i)
if (i < a.n && rbg >= a.rb0[i]) p = Pick{i, a.row0[i], a.row0[i + 1] - a.row0[i], a.tile0[i], a.rb0[i], a.blob[i]};
return p;
}
int fill_args_mt(GArgs& a, const Layout& lo, int MT) {
int rb = 0;
for (int i = 0; i < lo.n; ++i) {
a.row0[i] = lo.row0[i];
a.tile0[i] = lo.tile0[i];
a.rb0[i] = rb;
const int ntl = (lo.cnt[i] + 7) / 8;
rb += (ntl + MT - 1) / MT;
}
a.row0[lo.n] = lo.row0[lo.n];
a.tile0[lo.n] = lo.tile0[lo.n];
a.rb0[lo.n] = rb;
a.n = lo.n;
return rb;
}
template <int F, int MT, int G> struct TagGu;
template <int F, int MT, int G> struct TagDn;
// ------------------------------------------------------------------------------------------------ gate/up
template <int F, int MT, int G>
void launch_gu(sycl::queue& q, const GArgs a, int nrowblocks) {
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<G>};
constexpr int NS = (F == T_IQ2S) ? 2 : 1; // IQ2_S: two scales per 32 values -> two DPAS with the other half zeroed
q.submit([&](sycl::handler& h) {
h.parallel_for<TagGu<F, MT, G>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int rbg = (int) id[0], nt = (int) id[1];
const Pick pk = pick_expert(a, rbg);
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
const uint32_t* T = a.tab;
simd<uint32_t, 16> lane(0, 1);
simd<uint32_t, 16> ro = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;
simd<float, 128 * MT> acc = 0.f;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
for (int sb = 0; sb < ((a.dbg & 8) ? 0 : (a.nkb >> 3)); ++sb) {
const uint32_t P = (uint32_t) sb * blk_bytes<F>();
const simd<float, 16> d = half_of(ld16(We, ro + P));
simd<uint32_t, 16> w0 = 0, w1 = 0, w2 = 0, w3 = 0; // sb-level fields
if constexpr (F == T_IQ3S) {
w0 = ld32(We, ro + (P + 106)); // scales
w1 = ld32(We, ro + (P + 66)); // qh 0..3
w2 = ld32(We, ro + (P + 70)); // qh 4..7
} else if constexpr (F == T_IQ2S) {
w0 = ld32(We, ro + (P + 74)); // scales 0..3
w1 = ld32(We, ro + (P + 78)); // scales 4..7
w2 = ld32(We, ro + (P + 66)); // qh 0..3
w3 = ld32(We, ro + (P + 70)); // qh 4..7
}
#pragma unroll 1
for (int ib = 0; ib < 8; ++ib) {
const int kb = sb * 8 + ib;
simd<uint32_t, 128> fr0 = 0x01010101u, fr1 = 0u;
simd<float, 16> sw0 = 1.f, sw1 = 1.f;
if (!(a.dbg & 1)) {
if constexpr (F == T_IQ3S) {
simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
simd<uint32_t, 16> sg = ld32(We, ro + (P + 74 + 4 * ib));
simd<uint32_t, 16> qhv = (((ib < 4) ? w1 : w2) >> (8 * (ib & 3))) & 0xffu;
simd<uint32_t, 16> scn = ((w0 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;
sw0 = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));
#pragma unroll
for (int r = 0; r < 8; ++r) {
simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);
simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3S, idx << 2);
simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;
fr0.template select<16, 1>(16 * r) = apply_signs(g, nib);
}
} else if constexpr (F == T_IQ3XXS) {
simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
simd<uint32_t, 16> aux = ld32(We, ro + (P + 66 + 4 * ib));
sw0 = d * simd<float, 16>(simd<int, 16>((aux >> 28) * 2u + 1u)) * 0.25f;
#pragma unroll
for (int il = 0; il < 4; ++il) {
simd<uint32_t, 16> sbyte = gather<uint32_t, 16>(T + O_KSIGNS, ((aux >> (7 * il)) & 127u) << 2);
#pragma unroll
for (int hh = 0; hh < 2; ++hh) {
const int r = 2 * il + hh;
simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
simd<uint32_t, 16> idx = (qsw >> (8 * (r & 3))) & 0xffu;
simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3XXS, idx << 2);
fr0.template select<16, 1>(16 * r) = apply_signs(g, (sbyte >> (4 * hh)) & 0xfu);
}
}
} else if constexpr (F == T_IQ2S) {
simd<uint32_t, 16> idw = ld32(We, ro + (P + 2 + 4 * ib));
simd<uint32_t, 16> sgw = ld32(We, ro + (P + 34 + 4 * ib));
simd<uint32_t, 16> qhb = (((ib < 4) ? w2 : w3) >> (8 * (ib & 3))) & 0xffu;
simd<uint32_t, 16> scb = (((ib < 4) ? w0 : w1) >> (8 * (ib & 3))) & 0xffu;
sw0 = d * simd<float, 16>(simd<int, 16>((scb & 0xfu) * 2u + 1u)) * 0.125f;
sw1 = d * simd<float, 16>(simd<int, 16>((scb >> 4) * 2u + 1u)) * 0.125f;
#pragma unroll
for (int il = 0; il < 4; ++il) {
simd<uint32_t, 16> idx = ((idw >> (8 * il)) & 0xffu) | ((qhb << (8 - 2 * il)) & 0x300u);
simd<uint32_t, 16> gl = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3));
simd<uint32_t, 16> gh = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3) + 4u);
simd<uint32_t, 16> sbyte = (sgw >> (8 * il)) & 0xffu;
simd<uint32_t, 16> vl = apply_signs(gl, sbyte & 0xfu);
simd<uint32_t, 16> vh = apply_signs(gh, sbyte >> 4);
if (il < 2) {
fr0.template select<16, 1>(16 * (2 * il)) = vl;
fr0.template select<16, 1>(16 * (2 * il + 1)) = vh;
} else {
fr1.template select<16, 1>(16 * (2 * il)) = vl;
fr1.template select<16, 1>(16 * (2 * il + 1)) = vh;
}
}
// fr0 holds values 0..15 (dwords 0..3, the rest zero), fr1 values 16..31 (dwords 4..7)
fr0.template select<64, 1>(64) = 0u;
fr1.template select<64, 1>(0) = 0u;
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
simd<int, 64> av = 0x01010101;
simd<float, 8> dxv = 1.f;
if (!(a.dbg & 2)) {
av = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);
dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);
}
simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
}
if constexpr (NS == 2) {
simd<float, 128> tg = dpas_s8(fr1, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tg.template select<16, 1>(16 * r) * (sw1 * dxr);
}
}
}
}
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
});
});
}
// ------------------------------------------------------------------------------------------------ down
template <int F, int MT, int G>
void launch_dn(sycl::queue& q, const GArgs a, int nrowblocks) {
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<G>};
q.submit([&](sycl::handler& h) {
h.parallel_for<TagDn<F, MT, G>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int rbg = (int) id[0], nt = (int) id[1];
const Pick pk = pick_expert(a, rbg);
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
const uint32_t* T = a.tab;
simd<uint32_t, 16> lane(0, 1);
simd<uint32_t, 16> ro = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;
simd<float, 128 * MT> acc = 0.f;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
for (int kb = 0; kb < ((a.dbg & 8) ? 0 : a.nkb); ++kb) {
simd<float, 16> sw0 = 1.f;
simd<uint32_t, 128> fr0 = 0x01010101u;
if (!(a.dbg & 1)) {
if constexpr (F == T_IQ4NL) {
const uint32_t P = (uint32_t) kb * 18;
sw0 = half_of(ld16(We, ro + P));
#pragma unroll
for (int j = 0; j < 4; ++j) {
simd<uint32_t, 16> qd = ld32c(We, ro + (P + 2 + 4 * j), a.lim);
// four byte-pair lookups (lo value in byte 0, hi value in byte 1), repacked into the two dwords
simd<uint32_t, 16> t0 = gather<uint32_t, 16>(T + O_IQ4, (qd & 0xffu) << 2);
simd<uint32_t, 16> t1 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 8) & 0xffu) << 2);
simd<uint32_t, 16> t2 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 16) & 0xffu) << 2);
simd<uint32_t, 16> t3 = gather<uint32_t, 16>(T + O_IQ4, (qd >> 24) << 2);
fr0.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);
fr0.template select<16, 1>(16 * (j + 4)) =
((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);
}
} else { // T_Q2_0: 64 values per block (2 + 16 bytes), two 32-blocks per block, one table lookup per byte
const uint32_t P = (uint32_t) (kb >> 1) * 18;
const int hb = kb & 1;
sw0 = half_of(ld16(We, ro + P));
simd<uint32_t, 16> q0 = ld32c(We, ro + (P + 2 + 8 * hb), a.lim);
simd<uint32_t, 16> q1 = ld32c(We, ro + (P + 6 + 8 * hb), a.lim);
#pragma unroll
for (int r = 0; r < 8; ++r) {
simd<uint32_t, 16> qb = ((r < 4 ? q0 : q1) >> (8 * (r & 3))) & 0xffu;
fr0.template select<16, 1>(16 * r) = gather<uint32_t, 16>(T + O_Q2, qb << 2);
}
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
simd<int, 64> av = 0x01010101;
simd<float, 8> dxv = 1.f;
if (!(a.dbg & 2)) {
av = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);
dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);
}
simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
}
}
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
});
});
}
// ------------------------------------------------------------------------------------------------ work-group versions
// A work group is WGT = 8 threads side by side on 8 consecutive output tiles (128 outputs) of the same row block. The
// activation tiles of the row block are read from global memory ONCE per group, in chunks of 4 steps of 32 (128 of K), into local
// memory (double buffered, one barrier per chunk) and every thread reads them from there, instead of every thread reading all of
// them from L2 (80 threads per expert re-read the same ~200 KB: the activation loads were 8 of 19 us of gate/up). The codebook
// the decoders gather from is copied into local memory by the group once (its gathers then cost ~nothing).
constexpr int WGT = 8;
constexpr auto kAl16 = sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::alignment<16>};
// bytes of the tables a format gathers from (local memory), and the segments copied from the global table buffer
template <int F> constexpr int tab_bytes() {
return F == T_IQ3S ? 2048 : F == T_IQ3XXS ? 1536 : F == T_IQ2S ? 8192 : 1024; // IQ3XXS: grid 1024 + ksigns 512
}
template <int F, int MT>
struct WgLayout {
static constexpr int TB = tab_bytes<F>();
static constexpr int ASZ = MT * 1024; // activation tiles of one stage: MT tiles x 4 steps x 256 bytes
static constexpr int DSZ = MT * 128; // their scales: MT tiles x 4 steps x 32 bytes
static constexpr int STAGE = ASZ + DSZ;
static constexpr int TOTAL = TB + 2 * STAGE;
static constexpr int SPLIT = 8 / MT; // threads per tile in the staging load
static constexpr int PART = 1024 / SPLIT; // bytes of one tile's activations a thread stages
};
// Staging of one chunk's activation tiles: global -> registers (LOAD_CHUNK) -> local memory stage SS (STORE_CHUNK). Macros, not
// lambdas: a lambda capturing the staging registers by reference put them in private memory.
#define LOAD_CHUNK(CC) \
if (s_tile < nvalid) { \
const int* src_ = Xi + ((size_t) (tbase + s_tile) * a.nkb + (CC) * 4) * 64 + s_part * (L::PART / 4); \
_Pragma("unroll") \
for (int k = 0; k < L::PART / 256; ++k) stg.template select<64, 1>(64 * k) = block_load<int, 64>(src_ + 64 * k, kAl16); \
if (s_part == 0) stg_dx = block_load<float, 32>(a.dx + ((size_t) (tbase + s_tile) * a.nkb + (CC) * 4) * 8, kAl16); \
}
#define STORE_CHUNK(SS) \
if (s_tile < nvalid) { \
const uint32_t base_ = L::TB + (SS) * L::STAGE; \
_Pragma("unroll") \
for (int k = 0; k < L::PART / 256; ++k) \
slm_block_store<int, 64>(base_ + s_tile * 1024 + s_part * L::PART + 256 * k, stg.template select<64, 1>(64 * k), kAl16); \
if (s_part == 0) slm_block_store<float, 32>(base_ + L::ASZ + s_tile * 128, stg_dx, kAl16); \
}
template <int F, int MT, int G> struct TagGuW;
template <int F, int MT, int G> struct TagDnW;
template <int F, int MT, int G>
void launch_gu_wg(sycl::queue& q, const GArgs a, int nrowblocks) {
using L = WgLayout<F, MT>;
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<G>};
constexpr int NS = (F == T_IQ2S) ? 2 : 1;
q.submit([&](sycl::handler& h) {
h.parallel_for<TagGuW<F, MT, G>>(sycl::nd_range<2>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), sycl::range<2>(1, WGT)), props,
[=](sycl::nd_item<2> it) SYCL_ESIMD_KERNEL {
slm_init<L::TOTAL>();
const int rbg = (int) it.get_group(0), lt = (int) it.get_local_id(1), nt = (int) it.get_global_id(1);
const Pick pk = pick_expert(a, rbg);
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
const uint32_t* T = a.tab;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
// the codebook into local memory: thread lt copies its eighth of each segment
if constexpr (F == T_IQ3S) {
#pragma unroll
for (int k = 0; k < tab_bytes<F>() / WGT / 64; ++k)
slm_block_store<uint32_t, 16>(lt * (tab_bytes<F>() / WGT) + 64 * k, block_load<uint32_t, 16>(T + O_IQ3S + (lt * (tab_bytes<F>() / WGT) + 64 * k) / 4, kAl16), kAl16);
} else if constexpr (F == T_IQ3XXS) {
#pragma unroll
for (int k = 0; k < 1024 / WGT / 64; ++k)
slm_block_store<uint32_t, 16>(lt * (1024 / WGT) + 64 * k, block_load<uint32_t, 16>(T + O_IQ3XXS + (lt * (1024 / WGT) + 64 * k) / 4, kAl16), kAl16);
slm_block_store<uint32_t, 16>(1024 + lt * (512 / WGT), block_load<uint32_t, 16>(T + O_KSIGNS + lt * (512 / WGT) / 4, kAl16), kAl16);
} else { // IQ2_S
#pragma unroll
for (int k = 0; k < tab_bytes<F>() / WGT / 64; ++k)
slm_block_store<uint32_t, 16>(lt * (tab_bytes<F>() / WGT) + 64 * k, block_load<uint32_t, 16>(T + O_IQ2S + (lt * (tab_bytes<F>() / WGT) + 64 * k) / 4, kAl16), kAl16);
}
// staging: this thread's piece of the chunk c (tile, part) into registers, then into stage s
const int s_tile = lt / L::SPLIT, s_part = lt % L::SPLIT;
simd<int, L::PART / 4> stg;
simd<float, 32> stg_dx;
simd<uint32_t, 16> lane(0, 1);
simd<uint32_t, 16> ro = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;
simd<float, 128 * MT> acc = 0.f;
const int nchunks = (a.dbg & 8) ? 0 : (a.nkb >> 2);
if (nchunks > 0) { LOAD_CHUNK(0); STORE_CHUNK(0); }
barrier(); // the codebook and chunk 0 are in local memory
simd<float, 16> d = 0.f;
simd<uint32_t, 16> w0 = 0, w1 = 0, w2 = 0, w3 = 0;
for (int c = 0; c < nchunks; ++c) {
const int sb = c >> 1;
const int stage_off = L::TB + (c & 1) * L::STAGE;
if (c + 1 < nchunks) { LOAD_CHUNK(c + 1); }
if ((c & 1) == 0) { // new superblock: its scale and the fields shared by its 8 steps
const uint32_t P = (uint32_t) sb * blk_bytes<F>();
d = half_of(ld16(We, ro + P));
if constexpr (F == T_IQ3S) {
w0 = ld32(We, ro + (P + 106)); w1 = ld32(We, ro + (P + 66)); w2 = ld32(We, ro + (P + 70));
} else if constexpr (F == T_IQ2S) {
w0 = ld32(We, ro + (P + 74)); w1 = ld32(We, ro + (P + 78)); w2 = ld32(We, ro + (P + 66)); w3 = ld32(We, ro + (P + 70));
}
}
const uint32_t P = (uint32_t) sb * blk_bytes<F>();
#pragma unroll 1
for (int ib4 = 0; ib4 < 4; ++ib4) {
const int ib = ((c & 1) << 2) + ib4;
simd<uint32_t, 128> fr0 = 0x01010101u, fr1 = 0u;
simd<float, 16> sw0 = 1.f, sw1 = 1.f;
if (!(a.dbg & 1)) {
if constexpr (F == T_IQ3S) {
simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
simd<uint32_t, 16> sg = ld32(We, ro + (P + 74 + 4 * ib));
simd<uint32_t, 16> qhv = (((ib < 4) ? w1 : w2) >> (8 * (ib & 3))) & 0xffu;
simd<uint32_t, 16> scn = ((w0 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;
sw0 = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));
#pragma unroll
for (int r = 0; r < 8; ++r) {
simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);
simd<uint32_t, 16> g = slm_gather<uint32_t, 16>(idx << 2);
simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;
fr0.template select<16, 1>(16 * r) = apply_signs(g, nib);
}
} else if constexpr (F == T_IQ3XXS) {
simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
simd<uint32_t, 16> aux = ld32(We, ro + (P + 66 + 4 * ib));
sw0 = d * simd<float, 16>(simd<int, 16>((aux >> 28) * 2u + 1u)) * 0.25f;
#pragma unroll
for (int il = 0; il < 4; ++il) {
simd<uint32_t, 16> sbyte = slm_gather<uint32_t, 16>(1024u + (((aux >> (7 * il)) & 127u) << 2));
#pragma unroll
for (int hh = 0; hh < 2; ++hh) {
const int r = 2 * il + hh;
simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
simd<uint32_t, 16> idx = (qsw >> (8 * (r & 3))) & 0xffu;
simd<uint32_t, 16> g = slm_gather<uint32_t, 16>(idx << 2);
fr0.template select<16, 1>(16 * r) = apply_signs(g, (sbyte >> (4 * hh)) & 0xfu);
}
}
} else if constexpr (F == T_IQ2S) {
simd<uint32_t, 16> idw = ld32(We, ro + (P + 2 + 4 * ib));
simd<uint32_t, 16> sgw = ld32(We, ro + (P + 34 + 4 * ib));
simd<uint32_t, 16> qhb = (((ib < 4) ? w2 : w3) >> (8 * (ib & 3))) & 0xffu;
simd<uint32_t, 16> scb = (((ib < 4) ? w0 : w1) >> (8 * (ib & 3))) & 0xffu;
sw0 = d * simd<float, 16>(simd<int, 16>((scb & 0xfu) * 2u + 1u)) * 0.125f;
sw1 = d * simd<float, 16>(simd<int, 16>((scb >> 4) * 2u + 1u)) * 0.125f;
#pragma unroll
for (int il = 0; il < 4; ++il) {
simd<uint32_t, 16> idx = ((idw >> (8 * il)) & 0xffu) | ((qhb << (8 - 2 * il)) & 0x300u);
simd<uint32_t, 16> gl = slm_gather<uint32_t, 16>(idx << 3);
simd<uint32_t, 16> gh = slm_gather<uint32_t, 16>((idx << 3) + 4u);
simd<uint32_t, 16> sbyte = (sgw >> (8 * il)) & 0xffu;
simd<uint32_t, 16> vl = apply_signs(gl, sbyte & 0xfu);
simd<uint32_t, 16> vh = apply_signs(gh, sbyte >> 4);
if (il < 2) {
fr0.template select<16, 1>(16 * (2 * il)) = vl;
fr0.template select<16, 1>(16 * (2 * il + 1)) = vh;
} else {
fr1.template select<16, 1>(16 * (2 * il)) = vl;
fr1.template select<16, 1>(16 * (2 * il + 1)) = vh;
}
}
fr0.template select<64, 1>(64) = 0u;
fr1.template select<64, 1>(0) = 0u;
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
simd<int, 64> av = 0x01010101;
simd<float, 8> dxv = 1.f;
if (!(a.dbg & 2)) {
av = slm_block_load<int, 64>(stage_off + i * 1024 + ib4 * 256, kAl16);
dxv = slm_block_load<float, 8>(stage_off + L::ASZ + i * 128 + ib4 * 32, kAl16);
}
simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
}
if constexpr (NS == 2) {
simd<float, 128> tg = dpas_s8(fr1, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tg.template select<16, 1>(16 * r) * (sw1 * dxr);
}
}
}
}
}
if (c + 1 < nchunks) { STORE_CHUNK((c + 1) & 1); }
barrier();
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
});
});
}
template <int F, int MT, int G>
void launch_dn_wg(sycl::queue& q, const GArgs a, int nrowblocks) {
using L = WgLayout<F, MT>;
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<G>};
q.submit([&](sycl::handler& h) {
h.parallel_for<TagDnW<F, MT, G>>(sycl::nd_range<2>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), sycl::range<2>(1, WGT)), props,
[=](sycl::nd_item<2> it) SYCL_ESIMD_KERNEL {
slm_init<L::TOTAL>();
const int rbg = (int) it.get_group(0), lt = (int) it.get_local_id(1), nt = (int) it.get_global_id(1);
const Pick pk = pick_expert(a, rbg);
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
const uint32_t* T = a.tab;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
constexpr uint32_t TOFF = (F == T_IQ4NL) ? O_IQ4 : O_Q2;
#pragma unroll
for (int k = 0; k < tab_bytes<F>() / WGT / 64; ++k)
slm_block_store<uint32_t, 16>(lt * (tab_bytes<F>() / WGT) + 64 * k, block_load<uint32_t, 16>(T + TOFF + (lt * (tab_bytes<F>() / WGT) + 64 * k) / 4, kAl16), kAl16);
const int s_tile = lt / L::SPLIT, s_part = lt % L::SPLIT;
simd<int, L::PART / 4> stg;
simd<float, 32> stg_dx;
simd<uint32_t, 16> lane(0, 1);
simd<uint32_t, 16> ro = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;
simd<float, 128 * MT> acc = 0.f;
const int nchunks = (a.dbg & 8) ? 0 : (a.nkb >> 2);
if (nchunks > 0) { LOAD_CHUNK(0); STORE_CHUNK(0); }
barrier();
for (int c = 0; c < nchunks; ++c) {
const int stage_off = L::TB + (c & 1) * L::STAGE;
if (c + 1 < nchunks) { LOAD_CHUNK(c + 1); }
#pragma unroll 1
for (int ib4 = 0; ib4 < 4; ++ib4) {
const int kb = c * 4 + ib4;
simd<float, 16> sw0 = 1.f;
simd<uint32_t, 128> fr0 = 0x01010101u;
if (!(a.dbg & 1)) {
if constexpr (F == T_IQ4NL) {
const uint32_t P = (uint32_t) kb * 18;
sw0 = half_of(ld16(We, ro + P));
#pragma unroll
for (int j = 0; j < 4; ++j) {
simd<uint32_t, 16> qd = ld32c(We, ro + (P + 2 + 4 * j), a.lim);
simd<uint32_t, 16> t0 = slm_gather<uint32_t, 16>((qd & 0xffu) << 2);
simd<uint32_t, 16> t1 = slm_gather<uint32_t, 16>(((qd >> 8) & 0xffu) << 2);
simd<uint32_t, 16> t2 = slm_gather<uint32_t, 16>(((qd >> 16) & 0xffu) << 2);
simd<uint32_t, 16> t3 = slm_gather<uint32_t, 16>((qd >> 24) << 2);
fr0.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);
fr0.template select<16, 1>(16 * (j + 4)) =
((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);
}
} else { // T_Q2_0
const uint32_t P = (uint32_t) (kb >> 1) * 18;
const int hb = kb & 1;
sw0 = half_of(ld16(We, ro + P));
simd<uint32_t, 16> q0 = ld32c(We, ro + (P + 2 + 8 * hb), a.lim);
simd<uint32_t, 16> q1 = ld32c(We, ro + (P + 6 + 8 * hb), a.lim);
#pragma unroll
for (int r = 0; r < 8; ++r) {
simd<uint32_t, 16> qb = ((r < 4 ? q0 : q1) >> (8 * (r & 3))) & 0xffu;
fr0.template select<16, 1>(16 * r) = slm_gather<uint32_t, 16>(qb << 2);
}
}
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
simd<int, 64> av = 0x01010101;
simd<float, 8> dxv = 1.f;
if (!(a.dbg & 2)) {
av = slm_block_load<int, 64>(stage_off + i * 1024 + ib4 * 256, kAl16);
dxv = slm_block_load<float, 8>(stage_off + L::ASZ + i * 128 + ib4 * 32, kAl16);
}
simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
}
}
}
}
if (c + 1 < nchunks) { STORE_CHUNK((c + 1) & 1); }
barrier();
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
});
});
}
// MT from the group's mean number of 8-row tiles per expert.
int pick_mt(const Layout& lo) {
const double mean = lo.n ? (double) lo.tile0[lo.n] / lo.n : 0;
if (mean <= 2.0) return 2;
if (mean <= 4.0) return 4;
return 8;
}
int dbg_flags() {
static const int v = [] { const char* s = std::getenv("STRATA_I8_DBG"); return s ? std::atoi(s) : 0; }();
return v;
}
int mt_for(const Layout& lo) {
static const int mo = [] { const char* s = std::getenv("STRATA_I8_MT"); return s ? std::atoi(s) : 0; }();
return mo ? mo : pick_mt(lo);
}
// register mode per tile height: STRATA_I8_GRF=128 (more threads per EU, spills at MT 8) or 256; default 256
int wg_for() {
static const int v = [] { const char* s = std::getenv("STRATA_I8_WG"); return s ? std::atoi(s) : 1; }();
return v;
}
int grf_for(int mt) {
static const int g = [] { const char* s = std::getenv("STRATA_I8_GRF"); return s ? std::atoi(s) : 256; }();
(void) mt;
return g == 128 ? 128 : 256;
}
template <class F>
bool dispatch_mt(int mt, F&& f) {
switch (mt) {
case 2: f(std::integral_constant<int, 2>{}); return true;
case 4: f(std::integral_constant<int, 4>{}); return true;
case 8: f(std::integral_constant<int, 8>{}); return true;
default: return false;
}
}
struct QArgs {
int n;
int row0[kMaxGroup + 1];
int tile0[kMaxGroup + 1];
int nkb;
int ld; // row length in elements of the source
};
class QuantTag;
class SwigluTag;
} // namespace
void make_layout(Layout& lo, const int32_t* cnt, int n) {
lo.n = n;
int r = 0, t = 0;
for (int i = 0; i < n; ++i) {
lo.cnt[i] = cnt[i];
lo.row0[i] = r;
lo.tile0[i] = t;
r += cnt[i];
t += (cnt[i] + 7) / 8;
}
lo.row0[n] = r;
lo.tile0[n] = t;
}
namespace {
bool bmg_arch(const sycl::device& d) {
namespace ex = sycl::ext::oneapi::experimental;
if (!d.is_gpu()) return false;
const auto a = d.get_info<ex::info::device::architecture>();
return a == ex::architecture::intel_gpu_bmg_g21 || a == ex::architecture::intel_gpu_bmg_g31;
}
} // namespace
bool battlemage(void* stream) {
sycl::queue& q = *strata::q_of(stream);
return bmg_arch(q.get_device());
}
bool any_battlemage() {
static const bool v = [] {
try {
for (const auto& d : sycl::device::get_devices(sycl::info::device_type::gpu))
if (bmg_arch(d)) return true;
} catch (...) {}
return false;
}();
return v;
}
bool available(void* stream) {
sycl::queue& q = *strata::q_of(stream);
static std::mutex mu;
static std::unordered_map<sycl::device, bool> per_dev;
std::lock_guard<std::mutex> lk(mu);
const sycl::device d = q.get_device();
if (auto it = per_dev.find(d); it != per_dev.end()) return it->second;
const bool ok = [&] {
const char* v = std::getenv("STRATA_I8_FORCE");
if (v && v[0] == '0') return false;
if (!d.has(sycl::aspect::ext_intel_esimd)) return false;
namespace mx = sycl::ext::oneapi::experimental::matrix;
bool s8 = false;
for (const auto& c : d.get_info<sycl::ext::oneapi::experimental::info::device::matrix_combinations>())
if (c.atype == mx::matrix_type::sint8 && c.btype == mx::matrix_type::sint8 && c.nsize == 16 && c.ksize == 32) s8 = true;
return s8;
}();
per_dev[d] = ok;
return ok;
}
size_t xq_bytes(const Layout& lo, int K) { return (size_t) lo.tile0[lo.n] * 8 * K; }
size_t dx_bytes(const Layout& lo, int K) { return (size_t) lo.tile0[lo.n] * (K / 32) * 8 * sizeof(float); }
bool workspace(const Layout& lo, int K, int n_ff, Workspace& ws, void* stream) {
sycl::queue& q = *strata::q_of(stream);
static std::mutex mu;
struct St { Workspace cur; size_t cap_x = 0, cap_h = 0; };
static std::unordered_map<sycl::device, St> per_dev;
std::lock_guard<std::mutex> lk(mu);
St& st = per_dev[q.get_device()];
Workspace& cur = st.cur;
size_t& cap_x = st.cap_x;
size_t& cap_h = st.cap_h;
const size_t nx = xq_bytes(lo, K), nh = xq_bytes(lo, n_ff);
if (nx > cap_x || nh > cap_h) {
q.wait();
if (cur.xq) { sycl::free(cur.xq, q); sycl::free(cur.dx, q); sycl::free(cur.hq, q); sycl::free(cur.dh, q); }
cap_x = std::max(nx, cap_x + cap_x / 4);
cap_h = std::max(nh, cap_h + cap_h / 4);
cur.xq = sycl::malloc_device<int8_t>(cap_x, q);
cur.dx = sycl::malloc_device<float>(cap_x / 32 / 8 * 8 + 64, q); // rows * K/32 floats = bytes / 32
cur.hq = sycl::malloc_device<int8_t>(cap_h, q);
cur.dh = sycl::malloc_device<float>(cap_h / 32 + 64, q);
if (!cur.xq || !cur.dx || !cur.hq || !cur.dh) { cap_x = cap_h = 0; cur = Workspace{}; ws = Workspace{}; return false; }
}
ws = cur;
return true;
}
void quantize_rows(const uint16_t* X, const Layout& lo, int K, int8_t* Xq, float* dx, void* stream) {
if (lo.tile0[lo.n] == 0) return;
sycl::queue& q = *strata::q_of(stream);
QArgs a{};
a.n = lo.n;
for (int i = 0; i <= lo.n; ++i) { a.row0[i] = lo.row0[i]; a.tile0[i] = lo.tile0[i]; }
a.nkb = K / 32;
a.ld = K;
const size_t nthreads = (size_t) lo.tile0[lo.n] * a.nkb;
const half* Xh = reinterpret_cast<const half*>(X);
uint32_t* Xd = reinterpret_cast<uint32_t*>(Xq);
q.submit([&](sycl::handler& h) {
h.parallel_for<QuantTag>(sycl::range<2>((size_t) lo.tile0[lo.n], (size_t) a.nkb), [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int T = (int) id[0], kb = (int) id[1];
// the expert of tile T, by constant-index reads of the argument arrays (see pick_expert)
int t0e = a.tile0[0], r0e = a.row0[0], r1e = a.row0[1];
#pragma unroll
for (int i = 1; i < kMaxGroup; ++i)
if (i < a.n && T >= a.tile0[i]) { t0e = a.tile0[i]; r0e = a.row0[i]; r1e = a.row0[i + 1]; }
const int cnt = r1e - r0e;
simd<float, 8> dxv = 0.f;
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (T - t0e) * 8 + r;
simd<uint32_t, 8> qd = 0u;
if (rl < cnt) {
simd<half, 32> xh = block_load<half, 32>(Xh + ((size_t) (r0e + rl)) * a.ld + kb * 32);
simd<float, 32> x = xh;
float amax = hmax<float>(abs(x));
const float dd = amax / 127.f;
const float id127 = amax > 0.f ? 127.f / amax : 0.f;
simd<float, 32> y = x * id127;
simd<float, 32> adj = 0.5f;
adj.merge(simd<float, 32>(-0.5f), y < 0.f);
simd<int, 32> qi = convert<int>(y + adj);
simd<int8_t, 32> q8 = convert<int8_t>(qi);
qd = q8.template bit_cast_view<uint32_t>();
dxv[r] = dd;
}
block_store<uint32_t, 8>(Xd + ((size_t) T * a.nkb + kb) * 64 + r * 8, qd);
}
block_store<float, 8>(dx + ((size_t) T * a.nkb + kb) * 8, dxv);
});
});
}
void swiglu_quantize(const float* GU, const Layout& lo, int n_ff, int8_t* Hq, float* dh, void* stream) {
if (lo.tile0[lo.n] == 0) return;
sycl::queue& q = *strata::q_of(stream);
QArgs a{};
a.n = lo.n;
for (int i = 0; i <= lo.n; ++i) { a.row0[i] = lo.row0[i]; a.tile0[i] = lo.tile0[i]; }
a.nkb = n_ff / 32;
a.ld = 2 * n_ff;
const size_t nthreads = (size_t) lo.tile0[lo.n] * a.nkb;
uint32_t* Hd = reinterpret_cast<uint32_t*>(Hq);
q.submit([&](sycl::handler& h) {
h.parallel_for<SwigluTag>(sycl::range<2>((size_t) lo.tile0[lo.n], (size_t) a.nkb), [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int T = (int) id[0], kb = (int) id[1];
// the expert of tile T, by constant-index reads of the argument arrays (see pick_expert)
int t0e = a.tile0[0], r0e = a.row0[0], r1e = a.row0[1];
#pragma unroll
for (int i = 1; i < kMaxGroup; ++i)
if (i < a.n && T >= a.tile0[i]) { t0e = a.tile0[i]; r0e = a.row0[i]; r1e = a.row0[i + 1]; }
const int cnt = r1e - r0e;
simd<float, 8> dxv = 0.f;
#pragma unroll
for (int r = 0; r < 8; ++r) {
const int rl = (T - t0e) * 8 + r;
simd<uint32_t, 8> qd = 0u;
if (rl < cnt) {
simd<float, 64> v = block_load<float, 64>(GU + ((size_t) (r0e + rl)) * a.ld + kb * 64);
simd<float, 32> g = v.template select<32, 2>(0);
simd<float, 32> u = v.template select<32, 2>(1);
simd<float, 32> x = g * u / (1.f + exp(-g));
float amax = hmax<float>(abs(x));
const float dd = amax / 127.f;
const float id127 = amax > 0.f ? 127.f / amax : 0.f;
simd<float, 32> y = x * id127;
simd<float, 32> adj = 0.5f;
adj.merge(simd<float, 32>(-0.5f), y < 0.f);
simd<int, 32> qi = convert<int>(y + adj);
simd<int8_t, 32> q8 = convert<int8_t>(qi);
qd = q8.template bit_cast_view<uint32_t>();
dxv[r] = dd;
}
block_store<uint32_t, 8>(Hd + ((size_t) T * a.nkb + kb) * 64 + r * 8, qd);
}
block_store<float, 8>(dh + ((size_t) T * a.nkb + kb) * 8, dxv);
});
});
}
bool gate_up(int gu_type, const uint8_t* const* blob, size_t up_off, int n_ff, const Layout& lo, int K, const int8_t* Xq,
const float* dx, float* GU, void* stream) {
if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;
if ((gu_type != T_IQ3S && gu_type != T_IQ3XXS && gu_type != T_IQ2S) || K % 256 != 0 || (2 * n_ff) % 16 != 0 ||
(up_off & 3) != 0 || lo.n > kMaxGroup)
return false;
sycl::queue& q = *strata::q_of(stream);
GArgs a{};
for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];
a.sec_off = (uint32_t) up_off;
a.Xq = Xq; a.dx = dx; a.Y = GU;
a.tab = tables(q);
a.ntiles = 2 * n_ff / 16;
a.nkb = K / 32;
a.rowb = (K / 256) * block_bytes_of(gu_type);
a.ldy = 2 * n_ff;
a.dbg = dbg_flags();
const int mt = mt_for(lo);
const int rb = fill_args_mt(a, lo, mt);
return dispatch_mt(mt, [&](auto mtc) {
constexpr int MT = decltype(mtc)::value;
if (wg_for() && gu_type != T_IQ2S && a.nkb % 4 == 0 && a.ntiles % WGT == 0) { // IQ2_S: its 8 KB codebook copied into local memory by every group costs more than it saves (35 vs 16 us)
if (gu_type == T_IQ3S) launch_gu_wg<T_IQ3S, MT, 256>(q, a, rb);
else if (gu_type == T_IQ3XXS) launch_gu_wg<T_IQ3XXS, MT, 256>(q, a, rb);
else launch_gu_wg<T_IQ2S, MT, 256>(q, a, rb);
} else if (grf_for(MT) == 128) {
if (gu_type == T_IQ3S) launch_gu<T_IQ3S, MT, 128>(q, a, rb);
else if (gu_type == T_IQ3XXS) launch_gu<T_IQ3XXS, MT, 128>(q, a, rb);
else launch_gu<T_IQ2S, MT, 128>(q, a, rb);
} else {
if (gu_type == T_IQ3S) launch_gu<T_IQ3S, MT, 256>(q, a, rb);
else if (gu_type == T_IQ3XXS) launch_gu<T_IQ3XXS, MT, 256>(q, a, rb);
else launch_gu<T_IQ2S, MT, 256>(q, a, rb);
}
});
}
bool down(int d_type, const uint8_t* const* blob, size_t down_off, int n_embd, const Layout& lo, int n_ff, const int8_t* Hq,
const float* dh, float* Dm, void* stream) {
if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;
if ((d_type != T_IQ4NL && d_type != T_Q2_0) || n_ff % block_vals_of(d_type) != 0 || n_embd % 16 != 0 || (down_off & 3) != 0 ||
lo.n > kMaxGroup)
return false;
sycl::queue& q = *strata::q_of(stream);
GArgs a{};
for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];
a.sec_off = (uint32_t) down_off;
a.Xq = Hq; a.dx = dh; a.Y = Dm;
a.tab = tables(q);
a.ntiles = n_embd / 16;
a.nkb = n_ff / 32;
a.rowb = (n_ff / block_vals_of(d_type)) * block_bytes_of(d_type);
a.lim = (uint32_t) (down_off + (size_t) n_embd * (size_t) a.rowb) - 4u;
a.ldy = n_embd;
a.dbg = dbg_flags();
const int mt = mt_for(lo);
const int rb = fill_args_mt(a, lo, mt);
return dispatch_mt(mt, [&](auto mtc) {
constexpr int MT = decltype(mtc)::value;
if (wg_for() && a.nkb % 4 == 0 && a.ntiles % WGT == 0) {
if (d_type == T_IQ4NL) launch_dn_wg<T_IQ4NL, MT, 256>(q, a, rb);
else launch_dn_wg<T_Q2_0, MT, 256>(q, a, rb);
} else if (grf_for(MT) == 128) {
if (d_type == T_IQ4NL) launch_dn<T_IQ4NL, MT, 128>(q, a, rb);
else launch_dn<T_Q2_0, MT, 128>(q, a, rb);
} else {
if (d_type == T_IQ4NL) launch_dn<T_IQ4NL, MT, 256>(q, a, rb);
else launch_dn<T_Q2_0, MT, 256>(q, a, rb);
}
});
}
} // namespace strata::prefill::i8
#endif // STRATA_I8_NO_ESIMD