Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

sycl: int8 DPAS expert products for IQ3_XXS and IQ2_S gate/up and Q2_0 down too (every layer type of the IQ3_S pack but one), thread tile NT=1 with MT 2/4/8, parity test over all six type pairs; STRATA_PREFILL_MMQ=1 prompts +10% (4K), +16% (2K), +20% (1K) with identical tokens

Niko1221ebdfaacd1cd9
diff --git a/sycl/src/kernels/esimd_moe_parity.cpp b/sycl/src/kernels/esimd_moe_parity.cpp
index 269e67284..ebaddf2db 100644
--- a/sycl/src/kernels/esimd_moe_parity.cpp
+++ b/sycl/src/kernels/esimd_moe_parity.cpp
@@ -1,10 +1,12 @@
// esimd_moe_parity (151-b70): the fused int8 DPAS expert products (prefill/esimd_moe.dp.cpp) against
// (a) the engine's dequantize + oneMKL path (FP16 weights, FP16 activations), the path they replace,
-// (b) an FP64 host reference from the blobs (IQ3_S gate/up, IQ4_NL down) with the unquantized activations,
+// (b) an FP64 host reference from the blobs with the unquantized activations,
// (c) an exact host reference on the SAME int8 activations the device produced (isolates the kernel arithmetic from the
-// activation quantization: only the float accumulation order differs).
-// and times both paths per expert. esimd_moe_parity [mean_rows=80] [experts=16] [reps=10] [seed=1]
-// Exit status 1 when (c) is off by more than 1e-5 of the row's scale or (a)/(b) exceed the int8 activation error budget.
+// activation quantization: only the float accumulation order differs),
+// for every gate/up x down type pair the kernels handle, and times both paths per expert.
+// esimd_moe_parity [mean_rows=80] [experts=16] [reps=10] [seed=1] [gate_up_type down_type]
+// types: gate/up 21 IQ3_S, 18 IQ3_XXS, 22 IQ2_S; down 20 IQ4_NL, 42 Q2_0.
+// Exit status 1 when (c) is off by more than 2e-5 of the largest value or (b) exceeds the int8 activation error budget.
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/iq_dequant_expert.hpp"
#include "strata/prefill/esimd_moe.hpp"
@@ -30,46 +32,66 @@ namespace i8 = strata::prefill::i8;
static uint16_t f2h(float f) { return sycl::bit_cast<uint16_t>(sycl::half(f)); }
static float h2f(uint16_t h) { return (float) sycl::bit_cast<sycl::half>(h); }
-// host decode of one IQ3_S row (K values) -> double
-static void dec_iq3s(const uint8_t* row, int K, double* out) {
- for (int sb = 0; sb < K / 256; ++sb) {
- const uint8_t* blk = row + sb * 110;
+static int block_bytes(int t) { return t == 21 ? 110 : t == 18 ? 98 : t == 22 ? 82 : 18; }
+static int block_vals(int t) { return (t == 21 || t == 18 || t == 22) ? 256 : t == 42 ? 64 : 32; }
+
+// host decode of one row (K values) -> double
+static void dec_row(int t, const uint8_t* row, int K, double* out) {
+ const int bb = block_bytes(t), bv = block_vals(t);
+ for (int b = 0; b < K / bv; ++b) {
+ const uint8_t* blk = row + (size_t) b * bb;
uint16_t dh; std::memcpy(&dh, blk, 2);
- const uint8_t *qs = blk + 2, *qh = blk + 66, *sgn = blk + 74, *scl = blk + 106;
- for (int ib = 0; ib < 8; ++ib) {
- const double d = (double) h2f(dh) * (1 + 2 * ((scl[ib / 2] >> (4 * (ib % 2))) & 0xf));
- for (int r = 0; r < 8; ++r) {
- const int idx = qs[8 * ib + r] | (((qh[ib] >> r) & 1) << 8);
- const uint8_t* g = (const uint8_t*) &iq3s_grid[idx];
- const uint8_t sbyte = sgn[4 * ib + r / 2];
- for (int j = 0; j < 4; ++j) {
- const int kk = 4 * (r & 1) + j;
- out[sb * 256 + ib * 32 + 4 * r + j] = d * ((sbyte >> kk & 1) ? -(int) g[j] : (int) g[j]);
+ const double d = h2f(dh);
+ double* o = out + (size_t) b * bv;
+ if (t == 21) {
+ const uint8_t *qs = blk + 2, *qh = blk + 66, *sgn = blk + 74, *scl = blk + 106;
+ for (int ib = 0; ib < 8; ++ib) {
+ const double dd = d * (1 + 2 * ((scl[ib / 2] >> (4 * (ib % 2))) & 0xf));
+ for (int r = 0; r < 8; ++r) {
+ const int idx = qs[8 * ib + r] | (((qh[ib] >> r) & 1) << 8);
+ const uint8_t* g = (const uint8_t*) &iq3s_grid[idx];
+ const uint8_t sb = sgn[4 * ib + r / 2];
+ for (int j = 0; j < 4; ++j) o[ib * 32 + 4 * r + j] = dd * ((sb >> (4 * (r & 1) + j) & 1) ? -(int) g[j] : (int) g[j]);
}
}
- }
- }
-}
-static void dec_iq4nl(const uint8_t* row, int K, double* out) {
- for (int b = 0; b < K / 32; ++b) {
- const uint8_t* blk = row + b * 18;
- uint16_t dh; std::memcpy(&dh, blk, 2);
- for (int j = 0; j < 16; ++j) {
- out[b * 32 + j] = (double) h2f(dh) * kvalues_iq4nl[blk[2 + j] & 15];
- out[b * 32 + 16 + j] = (double) h2f(dh) * kvalues_iq4nl[blk[2 + j] >> 4];
+ } else if (t == 18) {
+ const uint8_t* qs = blk + 2;
+ for (int ib = 0; ib < 8; ++ib) {
+ uint32_t aux; std::memcpy(&aux, blk + 66 + 4 * ib, 4);
+ const double dd = d * (0.5 + (aux >> 28)) * 0.5;
+ for (int l = 0; l < 4; ++l) {
+ const uint8_t sb = ksigns_iq2xs[(aux >> (7 * l)) & 127];
+ const uint8_t *g1 = (const uint8_t*) &iq3xxs_grid[qs[8 * ib + 2 * l]], *g2 = (const uint8_t*) &iq3xxs_grid[qs[8 * ib + 2 * l + 1]];
+ for (int j = 0; j < 4; ++j) {
+ o[ib * 32 + 8 * l + j] = dd * ((sb >> j & 1) ? -(int) g1[j] : (int) g1[j]);
+ o[ib * 32 + 8 * l + 4 + j] = dd * ((sb >> (4 + j) & 1) ? -(int) g2[j] : (int) g2[j]);
+ }
+ }
+ }
+ } else if (t == 22) {
+ const uint8_t *qs = blk + 2, *qh = blk + 66, *scl = blk + 74, *sgn = blk + 34;
+ for (int ib = 0; ib < 8; ++ib)
+ for (int l = 0; l < 4; ++l) {
+ const int idx = qs[4 * ib + l] | ((qh[ib] << (8 - 2 * l)) & 0x300);
+ const uint8_t* g = (const uint8_t*) &iq2s_grid[idx];
+ const double dd = d * (0.5 + ((scl[ib] >> (4 * (l / 2))) & 0xf)) * 0.25;
+ const uint8_t sb = sgn[4 * ib + l];
+ for (int j = 0; j < 8; ++j) o[ib * 32 + 8 * l + j] = dd * ((sb >> j & 1) ? -(int) g[j] : (int) g[j]);
+ }
+ } else if (t == 20) {
+ for (int j = 0; j < 16; ++j) {
+ o[j] = d * kvalues_iq4nl[blk[2 + j] & 15];
+ o[16 + j] = d * kvalues_iq4nl[blk[2 + j] >> 4];
+ }
+ } else { // 42 Q2_0
+ for (int i = 0; i < 64; ++i) o[i] = d * (((blk[2 + i / 4] >> ((i % 4) * 2)) & 3) - 1);
}
}
}
-int main(int argc, char** argv) {
- const int mean = argc > 1 ? std::atoi(argv[1]) : 80;
- const int n = argc > 2 ? std::atoi(argv[2]) : 16;
- const int reps = argc > 3 ? std::atoi(argv[3]) : 10;
- const int seed = argc > 4 ? std::atoi(argv[4]) : 1;
- sycl::queue* q = &dpct::get_in_order_queue();
- if (!i8::available(q)) { std::printf("esimd_moe_parity: SKIP (no sub-group 16 / s8 DPAS device)\n"); return 0; }
- dpct::blas::descriptor_ptr hd = new dpct::blas::descriptor();
- hd->set_queue(q);
+struct Result { bool ok; double t_ref, t_fused; };
+
+static Result run_case(sycl::queue* q, dpct::blas::descriptor_ptr hd, int gt, int dt, int mean, int n, int reps, int seed) {
const int K = 2560, NFF = 640, NE = 2560;
std::mt19937 rng(seed);
std::vector<int32_t> cnt(n);
@@ -80,15 +102,15 @@ int main(int argc, char** argv) {
i8::Layout lo;
i8::make_layout(lo, cnt.data(), n);
- const size_t gate_b = (size_t) NFF * (K / 256) * 110, down_b = (size_t) NE * (NFF / 32) * 18;
- const size_t blob_b = 2 * gate_b + down_b;
+ const size_t rowb_gu = (size_t) (K / block_vals(gt)) * block_bytes(gt), rowb_dn = (size_t) (NFF / block_vals(dt)) * block_bytes(dt);
+ const size_t gate_b = NFF * rowb_gu, down_b = NE * rowb_dn, blob_b = 2 * gate_b + down_b;
std::vector<std::vector<uint8_t>> hb(n);
std::vector<uint8_t*> db(n);
for (int i = 0; i < n; ++i) {
hb[i].resize(blob_b);
for (auto& x : hb[i]) x = (uint8_t) rng();
- for (size_t o = 0; o < 2 * gate_b; o += 110) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }
- for (size_t o = 2 * gate_b; o < blob_b; o += 18) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }
+ for (size_t o = 0; o < 2 * gate_b; o += block_bytes(gt)) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }
+ for (size_t o = 2 * gate_b; o < blob_b; o += block_bytes(dt)) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }
db[i] = sycl::malloc_device<uint8_t>(blob_b + 256, *q);
q->memcpy(db[i], hb[i].data(), blob_b);
}
@@ -103,25 +125,23 @@ int main(int argc, char** argv) {
float* dH = sycl::malloc_device<float>(i8::dx_bytes(lo, NFF) / 4, *q);
float *GU = sycl::malloc_device<float>((size_t) tot * 2 * NFF, *q), *Dm = sycl::malloc_device<float>((size_t) tot * NE, *q);
float *GUr = sycl::malloc_device<float>((size_t) tot * 2 * NFF, *q), *Dmr = sycl::malloc_device<float>((size_t) tot * NE, *q);
- uint16_t *Hh = sycl::malloc_device<uint16_t>((size_t) tot * NFF, *q);
+ uint16_t* Hh = sycl::malloc_device<uint16_t>((size_t) tot * NFF, *q);
uint16_t* dwgu = sycl::malloc_device<uint16_t>((size_t) 2 * NFF * K, *q);
uint16_t* dwd = sycl::malloc_device<uint16_t>((size_t) NE * NFF, *q);
q->wait();
std::vector<const uint8_t*> pb(n);
- if (std::getenv("I8_STAGE")) { for (int i = 0; i < n && i < 4; ++i) std::printf("blob %d at %p size %zu\n", i, (void*) db[i], blob_b); }
for (int i = 0; i < n; ++i) pb[i] = db[i];
- auto fused = [&](bool all = true) {
+ auto fused = [&]() {
i8::quantize_rows(dxh, lo, K, Xq, dX, q);
- if (!i8::gate_up(21, pb.data(), 2 * 0 + gate_b, NFF, lo, K, Xq, dX, GU, q)) { std::printf("gate_up refused\n"); std::exit(2); }
+ if (!i8::gate_up(gt, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q)) { std::printf("gate_up refused (type %d)\n", gt); std::exit(2); }
i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q);
- if (!i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q)) { std::printf("down refused\n"); std::exit(2); }
- (void) all;
+ if (!i8::down(dt, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q)) { std::printf("down refused (type %d)\n", dt); std::exit(2); }
};
auto ref = [&]() {
int64_t o = 0;
for (int i = 0; i < n; ++i) {
- strata::kernels::iq_dequant_expert_f16(21, 20, db[i], db[i] + gate_b, db[i] + 2 * gate_b, NFF, K, dwgu, dwd, q);
+ strata::kernels::iq_dequant_expert_f16(gt, dt, db[i], db[i] + gate_b, db[i] + 2 * gate_b, NFF, K, dwgu, dwd, q);
const float a = 1.f, b = 0.f;
dpct::blas::gemm(hd, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, 2 * NFF, cnt[i], K, &a, dwgu,
dpct::library_data_t::real_half, K, dxh + o * K, dpct::library_data_t::real_half, K, &b,
@@ -143,23 +163,7 @@ int main(int argc, char** argv) {
return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - a).count() / reps;
};
- if (const char* st = std::getenv("I8_STAGE")) { // bring-up: run the first N stages, synchronising after each
- const int ns = std::atoi(st);
- i8::quantize_rows(dxh, lo, K, Xq, dX, q); q->wait(); std::printf("stage 1 quantize ok%c", 10);
- if (ns >= 2) { i8::gate_up(21, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); q->wait(); std::printf("stage 2 gate_up ok%c", 10); }
- if (ns >= 3) { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); q->wait(); std::printf("stage 3 swiglu ok%c", 10); }
- if (ns >= 4) { i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); q->wait(); std::printf("stage 4 down ok%c", 10); }
- if (const char* dd = std::getenv("STRATA_I8_DBG"); dd && (std::atoi(dd) & 128)) { // the down kernel stored rowoff + P per thread
- std::vector<uint32_t> o((size_t) 480 * 16);
- q->memcpy(o.data(), Dm, o.size() * 4).wait();
- uint32_t mx = 0, mn = ~0u;
- for (int th = 0; th < 480; ++th) for (int l = 0; l < 16; ++l) { mx = std::max(mx, o[th * 16 + l]); mn = std::min(mn, o[th * 16 + l]); }
- std::printf("rowoff+P: min %u max %u%c", mn, mx, 10); for (int th : {0, 1, 158, 159, 160, 161, 319, 320, 321, 479}) std::printf(" t%d lane0 %u row %u%c", th, o[th * 16], (o[th * 16] - 1408000) / 360, 10);
- }
- return 0;
- }
fused(); q->wait();
- // ---- (c) exact reference on the device's int8 activations
std::vector<int8_t> hXq(i8::xq_bytes(lo, K));
std::vector<float> hdX(i8::dx_bytes(lo, K) / 4), hGU((size_t) tot * 2 * NFF);
q->memcpy(hXq.data(), Xq, hXq.size());
@@ -168,36 +172,33 @@ int main(int argc, char** argv) {
const int nkb = K / 32;
auto xq_at = [&](int tile, int r, int k) { return (int) hXq[(((size_t) tile * nkb + k / 32) * 8 + r) * 32 + (k % 32)]; };
auto dx_at = [&](int tile, int r, int kb) { return (double) hdX[((size_t) tile * nkb + kb) * 8 + r]; };
- int fails = 0;
- double err_c = 0, mx_c = 0, err_b = 0, rms_b = 0, rms_bref = 0, mx_b = 0;
+ bool ok = true;
std::vector<double> wrow(K);
- {
- int sampled = 0;
- for (int e = 0; e < n; ++e) {
- for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {
- const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;
- for (int v = 0; v < 2 * NFF; v += 37) {
- dec_iq3s(&hb[e][(size_t) (v & 1 ? gate_b : 0) + (size_t) (v >> 1) * (K / 256) * 110], K, wrow.data());
- double refc = 0, refb = 0;
- for (int kb = 0; kb < nkb; ++kb) {
- double s = 0;
- for (int k = 0; k < 32; ++k) s += (double) xq_at(tile, r, kb * 32 + k) * wrow[kb * 32 + k];
- refc += s * dx_at(tile, r, kb);
- }
- for (int k = 0; k < K; ++k) refb += (double) h2f(hx[(size_t) row * K + k]) * wrow[k];
- const double got = hGU[(size_t) row * 2 * NFF + v];
- err_c = std::max(err_c, std::fabs(got - refc)); mx_c = std::max(mx_c, std::fabs(refc));
- err_b += (got - refb) * (got - refb); rms_bref += refb * refb; mx_b = std::max(mx_b, std::fabs(refb));
- ++sampled;
+ double err_c = 0, mx_c = 0, err_b = 0, rms_bref = 0;
+ int sampled = 0;
+ for (int e = 0; e < n; ++e)
+ for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {
+ const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;
+ for (int v = 0; v < 2 * NFF; v += 37) {
+ dec_row(gt, &hb[e][(size_t) (v & 1 ? gate_b : 0) + (size_t) (v >> 1) * rowb_gu], K, wrow.data());
+ double refc = 0, refb = 0;
+ for (int kb = 0; kb < nkb; ++kb) {
+ double s = 0;
+ for (int k = 0; k < 32; ++k) s += (double) xq_at(tile, r, kb * 32 + k) * wrow[kb * 32 + k];
+ refc += s * dx_at(tile, r, kb);
}
+ for (int k = 0; k < K; ++k) refb += (double) h2f(hx[(size_t) row * K + k]) * wrow[k];
+ const double got = hGU[(size_t) row * 2 * NFF + v];
+ err_c = std::max(err_c, std::fabs(got - refc)); mx_c = std::max(mx_c, std::fabs(refc));
+ err_b += (got - refb) * (got - refb); rms_bref += refb * refb;
+ ++sampled;
}
}
- const bool ok = err_c <= 2e-5 * mx_c && std::sqrt(err_b / rms_bref) < 0.02;
- fails += !ok;
- std::printf("gate/up IQ3_S : exact-int8 reference max err %.2e of %.2e (%.1e) vs FP64 true: rms rel err %.2e (%d samples) %s\n",
- err_c, mx_c, err_c / mx_c, std::sqrt(err_b / rms_bref), sampled, ok ? "OK" : "FAIL");
- }
- { // down: needs the device's Hq / dH and the true H from the device GU (float math on the host)
+ const bool ok1 = err_c <= 2e-5 * mx_c && std::sqrt(err_b / rms_bref) < 0.02;
+ ok = ok && ok1;
+ std::printf(" gate/up type %2d: exact-int8 ref max err %.2e of %.2e (%.1e) vs FP64 true rms rel err %.2e (%d samples) %s\n", gt,
+ err_c, mx_c, err_c / mx_c, std::sqrt(err_b / rms_bref), sampled, ok1 ? "OK" : "FAIL");
+ {
std::vector<int8_t> hHq(i8::xq_bytes(lo, NFF));
std::vector<float> hdH(i8::dx_bytes(lo, NFF) / 4), hDm((size_t) tot * NE);
q->memcpy(hHq.data(), Hq, hHq.size());
@@ -206,11 +207,10 @@ int main(int argc, char** argv) {
const int nkh = NFF / 32;
double ec = 0, mc = 0, eb = 0, rb = 0, hq_bad = 0;
std::vector<double> drow(NFF);
- int sampled = 0;
+ int smp = 0;
for (int e = 0; e < n; ++e)
for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {
const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;
- // the true H (double) of this row, and a check of the device's swiglu quantization
std::vector<double> H(NFF);
for (int j = 0; j < NFF; ++j) {
const double g = hGU[(size_t) row * 2 * NFF + 2 * j], u = hGU[(size_t) row * 2 * NFF + 2 * j + 1];
@@ -222,7 +222,7 @@ int main(int argc, char** argv) {
if (std::fabs(dd - amax / 127.0) > 1e-5 * amax + 1e-30) hq_bad += 1;
}
for (int v = 0; v < NE; v += 41) {
- dec_iq4nl(&hb[e][2 * gate_b + (size_t) v * nkh * 18], NFF, drow.data());
+ dec_row(dt, &hb[e][2 * gate_b + (size_t) v * rowb_dn], NFF, drow.data());
double refc = 0, refb = 0;
for (int kb = 0; kb < nkh; ++kb) {
double s = 0;
@@ -233,15 +233,14 @@ int main(int argc, char** argv) {
const double got = hDm[(size_t) row * NE + v];
ec = std::max(ec, std::fabs(got - refc)); mc = std::max(mc, std::fabs(refc));
eb += (got - refb) * (got - refb); rb += refb * refb;
- ++sampled;
+ ++smp;
}
}
- const bool ok = ec <= 2e-5 * mc && std::sqrt(eb / rb) < 0.03 && hq_bad == 0;
- fails += !ok;
- std::printf("down IQ4_NL : exact-int8 reference max err %.2e of %.2e (%.1e) vs FP64 true: rms rel err %.2e, swiglu scale mismatches %.0f (%d samples) %s\n",
- ec, mc, ec / mc, std::sqrt(eb / rb), hq_bad, sampled, ok ? "OK" : "FAIL");
+ const bool ok2 = ec <= 2e-5 * mc && std::sqrt(eb / rb) < 0.03 && hq_bad == 0;
+ ok = ok && ok2;
+ std::printf(" down type %2d : exact-int8 ref max err %.2e of %.2e (%.1e) vs FP64 true rms rel err %.2e, swiglu scale mismatches %.0f (%d samples) %s\n",
+ dt, ec, mc, ec / mc, std::sqrt(eb / rb), hq_bad, smp, ok2 ? "OK" : "FAIL");
}
- // ---- (a) against the dequantize + oneMKL path, whole chain
ref(); q->wait();
{
std::vector<float> a((size_t) tot * NE), b((size_t) tot * NE);
@@ -249,15 +248,37 @@ int main(int argc, char** argv) {
q->memcpy(b.data(), Dm, b.size() * 4).wait();
double e2 = 0, r2 = 0;
for (size_t i = 0; i < a.size(); ++i) { e2 += (double) (a[i] - b[i]) * (a[i] - b[i]); r2 += (double) a[i] * a[i]; }
- std::printf("chain vs dequant + oneMKL (FP16 path): rms rel diff %.2e (rows %d)\n", std::sqrt(e2 / r2), tot);
+ std::printf(" chain vs dequantize + oneMKL (FP16 path): rms rel diff %.2e (rows %d)\n", std::sqrt(e2 / r2), tot);
+ }
+ Result res{ok, tm(ref) / n, tm(fused) / n};
+ const double t_q = tm([&] { i8::quantize_rows(dxh, lo, K, Xq, dX, q); }) / n;
+ const double t_gu = tm([&] { i8::gate_up(gt, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); }) / n;
+ const double t_sw = tm([&] { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); }) / n;
+ const double t_dn = tm([&] { i8::down(dt, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); }) / n;
+ std::printf(" rows %d (%d experts, mean %.0f): dequant+oneMKL %.1f us/expert | fused int8 %.1f us/expert (%.2fx) [quantize %.1f, gate/up %.1f, swiglu+q %.1f, down %.1f]\n",
+ tot, n, (double) tot / n, res.t_ref, res.t_fused, res.t_ref / res.t_fused, t_q, t_gu, t_sw, t_dn);
+ for (int i = 0; i < n; ++i) sycl::free(db[i], *q);
+ sycl::free(dxh, *q); sycl::free(Xq, *q); sycl::free(dX, *q); sycl::free(Hq, *q); sycl::free(dH, *q); sycl::free(GU, *q);
+ sycl::free(Dm, *q); sycl::free(GUr, *q); sycl::free(Dmr, *q); sycl::free(Hh, *q); sycl::free(dwgu, *q); sycl::free(dwd, *q);
+ return res;
+}
+
+int main(int argc, char** argv) {
+ const int mean = argc > 1 ? std::atoi(argv[1]) : 80;
+ const int n = argc > 2 ? std::atoi(argv[2]) : 16;
+ const int reps = argc > 3 ? std::atoi(argv[3]) : 10;
+ const int seed = argc > 4 ? std::atoi(argv[4]) : 1;
+ sycl::queue* q = &dpct::get_in_order_queue();
+ if (!i8::available(q)) { std::printf("esimd_moe_parity: SKIP (no ESIMD device with the s8 8x16x32 DPAS)\n"); return 0; }
+ dpct::blas::descriptor_ptr hd = new dpct::blas::descriptor();
+ hd->set_queue(q);
+ std::vector<std::pair<int, int>> cases = {{21, 20}, {18, 20}, {22, 20}, {21, 42}, {18, 42}, {22, 42}};
+ if (argc > 6) cases = {{std::atoi(argv[5]), std::atoi(argv[6])}};
+ int fails = 0;
+ for (auto [gt, dt] : cases) {
+ std::printf("gate/up %d, down %d, mean %d rows, %d experts\n", gt, dt, mean, n);
+ fails += !run_case(q, hd, gt, dt, mean, n, reps, seed).ok;
}
- const double t_ref = tm(ref), t_f = tm([&] { fused(); });
- const double t_q = tm([&] { i8::quantize_rows(dxh, lo, K, Xq, dX, q); });
- const double t_gu = tm([&] { i8::gate_up(21, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); });
- const double t_sw = tm([&] { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); });
- const double t_dn = tm([&] { i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); });
- std::printf("rows %d (%d experts, mean %.0f): dequant+oneMKL chain %.1f us/expert | fused int8 chain %.1f us/expert (%.2fx) [quantize %.1f, gate/up %.1f, swiglu+q %.1f, down %.1f]\n",
- tot, n, (double) tot / n, t_ref / n, t_f / n, t_ref / t_f, t_q / n, t_gu / n, t_sw / n, t_dn / n);
std::printf("%s\n", fails ? "FAILED" : "all OK");
return fails ? 1 : 0;
}
diff --git a/sycl/src/prefill/esimd_moe.dp.cpp b/sycl/src/prefill/esimd_moe.dp.cpp
index 40794a832..0d4d4b3ad 100644
--- a/sycl/src/prefill/esimd_moe.dp.cpp
+++ b/sycl/src/prefill/esimd_moe.dp.cpp
@@ -1,24 +1,26 @@
-// 151-b70: fused int8 DPAS expert products (ESIMD, Battlemage) - see include/strata/prefill/esimd_moe.hpp.
+// 151-b70: fused int8 DPAS expert products (ESIMD, Battlemage) - see include/strata/prefill/esimd_moe.hpp and
+// docs/INTEL_INT8_EXPERTS.md (the behaviour note).
//
-// Behaviour note (docs/INTEL.md, "int8 expert path"):
// * Tile. One hardware thread owns 16 output rows (a DPAS N tile; for gate/up the 16 virtual rows alternate gate/up) and up to
-// MT x 8 activation rows (MT = 2, 4, 8 or 12, picked per group from the experts' mean row count). It walks K in steps of 32:
+// MT x 8 activation rows (MT = 2, 4 or 8, picked per group from the experts' mean row count). It walks K in steps of 32:
// decode the 16 weight rows' 32 values into the B operand (registers only), then for each of its row tiles one s8 DPAS 8x16x32
// into a zeroed int32 tile, converted to float and scaled by (weight scale of the row and 32-block) x (activation scale of the
-// row and 32-block) into a float accumulator. A 128-wide work-group tile (the owner's 16/32/64 rows x 128 outputs) is eight of
-// these threads side by side; there is no shared memory and no barrier.
+// row and 32-block) into a float accumulator. There is no shared memory and no barrier.
// * Decode once per tile: the B operand is decoded once per thread and applied to all its MT row tiles. An expert with more rows
-// than 8 x MT is split over row-block threads, each decoding again (the redundancy is ceil(rows / (8 MT)); MT 12 keeps it 1 up
-// to 96 rows).
+// than 8 x MT is split over row-block threads, each decoding again. A thread's accumulators are MT tiles of 512 bytes in an
+// 8 KB register file (256 GRF of 32 bytes): MT 12, or two output tiles per thread, spilled (3-15 KB) and ran 2-5x slower.
// * Scales. Activations: q8_1 as llama.cpp (d = amax / 127, q = round half away from zero), one float per row and 32 of K.
-// Weights: IQ3_S d * (1 + 2 s) per 32 (exact in float), IQ4_NL d per 32; the int8 values are the codebook entries with their
-// signs, never rounded. The int32 dot product of a 32-block is at most 32 * 127 * 15 (IQ3_S) so it is exact, and int32 -> float
-// is exact below 2^24. The float accumulation order is fixed (K ascending).
+// Weights: IQ3_S d (1 + 2 s), IQ3_XXS d (2 s + 1) / 4, IQ2_S d (2 s + 1) / 8 per 16 (two DPAS per 32 with the other half
+// zeroed), IQ4_NL d per 32, Q2_0 d per 64; the int8 values are the codebook entries with their signs, never rounded. The int32
+// dot product of a 32-block is exact and exactly convertible to float. The float accumulation order is fixed (K ascending).
// * Edge cases. Rows are padded to tiles of 8 with zeros (stores are masked to the real rows); a group of up to 16 experts per
-// launch; K a multiple of 256 (IQ3_S) / 32 (IQ4_NL); output rows a multiple of 16. Blob reads never go past a matrix end
-// (the last dword of a block is read as two dwords only when the block is not dword aligned).
-// * Fallback. available() is false without sub-group 16 DPAS (a non-Battlemage device) and the caller keeps the dequantize +
+// launch; K a multiple of 256 (IQ3_S / IQ3_XXS / IQ2_S) / 64 (Q2_0) / 32 (IQ4_NL); output rows a multiple of 16. Blob reads
+// never go past a matrix end.
+// * Fallback. available() is false without the s8 8x16x32 DPAS (a non-Battlemage device) and the caller keeps the dequantize +
// oneMKL path; gate_up()/down() return false for a type / shape they do not handle.
+// * ESIMD on this compiler (oneAPI 2026.1, bmg-g31): a dynamic index into the kernel-argument arrays, and `/` or `%` by a run-time
+// value, gave wrong values / garbage pointers (the first weight read faulted the GPU); the kernels use constant-index reads
+// (pick_expert) and 2-D grids. gather<T, N, VS> with VS > 1 is not available on Battlemage.
#include <sycl/sycl.hpp>
#include <sycl/ext/intel/esimd.hpp>
#include <sycl/ext/intel/experimental/grf_size_properties.hpp>
@@ -30,7 +32,7 @@
#include <mutex>
#include <type_traits>
#include <string>
-#include <unordered_map>
+#include <vector>
#include "strata/sycl_queue.hpp"
#include "strata/prefill/esimd_moe.hpp"
@@ -46,26 +48,41 @@ using namespace sycl::ext::intel::esimd;
namespace xmx = sycl::ext::intel::esimd::xmx;
using half = sycl::half;
-constexpr int BLK_IQ3S = 110; // sizeof(block_iq3_s)
-constexpr int BLK_IQ4NL = 18; // sizeof(block_iq4_nl)
+// ggml types handled
+constexpr int T_IQ3XXS = 18, T_IQ4NL = 20, T_IQ3S = 21, T_IQ2S = 22, T_Q2_0 = 42;
+template <int F> constexpr int blk_bytes() { return F == T_IQ3S ? 110 : F == T_IQ3XXS ? 98 : F == T_IQ2S ? 82 : F == T_IQ4NL ? 18 : 18; }
+constexpr int block_bytes_of(int f) { return f == T_IQ3S ? 110 : f == T_IQ3XXS ? 98 : f == T_IQ2S ? 82 : 18; }
+constexpr int block_vals_of(int f) { return (f == T_IQ3S || f == T_IQ3XXS || f == T_IQ2S) ? 256 : f == T_Q2_0 ? 64 : 32; }
-struct Tables {
- uint32_t* grid3s = nullptr; // iq3s_grid[512]
- uint32_t* iq4 = nullptr; // [256]: byte b -> kvalues[b & 15] in byte 0, kvalues[b >> 4] in byte 1
-};
-Tables& tables(sycl::queue& q) {
+// One device table buffer (uint32 units): the codebooks the decoders gather from.
+constexpr int O_IQ3S = 0; // iq3s_grid[512]
+constexpr int O_IQ3XXS = 512; // iq3xxs_grid[256]
+constexpr int O_IQ2S = 768; // iq2s_grid[1024] as 2048 dwords (low dword of an entry first)
+constexpr int O_KSIGNS = 2816; // ksigns_iq2xs[128], one byte in the low byte of each dword
+constexpr int O_IQ4 = 2944; // [256]: byte b -> kvalues_iq4nl[b & 15] in byte 0, kvalues_iq4nl[b >> 4] in byte 1
+constexpr int O_Q2 = 3200; // [256]: byte b -> its four 2-bit codes minus 1, as four int8 (Q2_0 values)
+constexpr int N_TABLE = 3456;
+
+uint32_t* tables(sycl::queue& q) {
static std::mutex mu;
- static Tables t; // one device per process in the port
+ static uint32_t* t = nullptr; // one device per process in the port
std::lock_guard<std::mutex> lk(mu);
- if (!t.grid3s) {
- t.grid3s = sycl::malloc_device<uint32_t>(512, q);
- t.iq4 = sycl::malloc_device<uint32_t>(256, q);
- uint32_t h4[256];
- for (int b = 0; b < 256; ++b)
- h4[b] = (uint32_t) (uint8_t) kvalues_iq4nl[b & 15] | ((uint32_t) (uint8_t) kvalues_iq4nl[b >> 4] << 8);
- q.memcpy(t.grid3s, iq3s_grid, 512 * 4);
- q.memcpy(t.iq4, h4, 256 * 4);
+ if (!t) {
+ std::vector<uint32_t> h(N_TABLE, 0);
+ for (int i = 0; i < 512; ++i) h[O_IQ3S + i] = iq3s_grid[i];
+ for (int i = 0; i < 256; ++i) h[O_IQ3XXS + i] = iq3xxs_grid[i];
+ for (int i = 0; i < 1024; ++i) { h[O_IQ2S + 2 * i] = (uint32_t) iq2s_grid[i]; h[O_IQ2S + 2 * i + 1] = (uint32_t) (iq2s_grid[i] >> 32); }
+ for (int i = 0; i < 128; ++i) h[O_KSIGNS + i] = ksigns_iq2xs[i];
+ for (int b = 0; b < 256; ++b) {
+ h[O_IQ4 + b] = (uint32_t) (uint8_t) kvalues_iq4nl[b & 15] | ((uint32_t) (uint8_t) kvalues_iq4nl[b >> 4] << 8);
+ uint32_t q2 = 0;
+ for (int k = 0; k < 4; ++k) q2 |= (uint32_t) (uint8_t) (int8_t) (((b >> (2 * k)) & 3) - 1) << (8 * k);
+ h[O_Q2 + b] = q2;
+ }
+ uint32_t* d = sycl::malloc_device<uint32_t>(N_TABLE, q);
+ q.memcpy(d, h.data(), N_TABLE * 4);
q.wait();
+ t = d;
}
return t;
}
@@ -93,6 +110,12 @@ SYCL_ESIMD_FUNCTION inline simd<int, 128> dpas_s8(simd<uint32_t, 128> b, simd<in
simd<int, 128> bi = b.template bit_cast_view<int>();
return xmx::dpas<8, 8, int, int, int, int, xmx::dpas_argument_type::s8, xmx::dpas_argument_type::s8>(simd<int, 128>(0), bi, a);
}
+// sign nibble -> four negation bytes: (g ^ mask) + m negates the bytes whose sign bit is set (g is never 0xFF per byte)
+SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> apply_signs(simd<uint32_t, 16> g, simd<uint32_t, 16> nib) {
+ simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);
+ simd<uint32_t, 16> mask = m * 255u;
+ return (g ^ mask) + m;
+}
struct GArgs {
const uint8_t* blob[kMaxGroup];
@@ -109,12 +132,10 @@ struct GArgs {
int nkb; // K / 32
int rowb; // bytes per weight row
int ldy;
- int dbg; // bring-up switches (STRATA_I8_DBG): 1 skip the decode, 2 skip the activation loads, 4 skip the stores
+ int dbg; // bring-up switches (STRATA_I8_DBG): 1 skip the decode, 2 skip the activation loads, 4 skip the stores, 8 skip the K loop
};
-
-// The expert a work item belongs to, with constant-index reads of the kernel argument arrays (a dynamic index into the argument
-// struct gave a garbage pointer in an ESIMD kernel on the B70: the first weight read faulted the GPU).
+// The expert a work item belongs to, with constant-index reads of the kernel argument arrays.
struct Pick { int e, row0, cnt, tile0, rb0; const uint8_t* blob; };
SYCL_ESIMD_FUNCTION inline Pick pick_expert(const GArgs& a, int rbg) {
Pick p{0, a.row0[0], a.row0[1] - a.row0[0], a.tile0[0], a.rb0[0], a.blob[0]};
@@ -124,9 +145,7 @@ SYCL_ESIMD_FUNCTION inline Pick pick_expert(const GArgs& a, int rbg) {
return p;
}
-struct LaunchInfo { int nthreads; };
-template <int MT>
-void fill_args(GArgs& a, const Layout& lo, int ntiles_out, int& nthreads) {
+int fill_args_mt(GArgs& a, const Layout& lo, int MT) {
int rb = 0;
for (int i = 0; i < lo.n; ++i) {
a.row0[i] = lo.row0[i];
@@ -139,65 +158,112 @@ void fill_args(GArgs& a, const Layout& lo, int ntiles_out, int& nthreads) {
a.tile0[lo.n] = lo.tile0[lo.n];
a.rb0[lo.n] = rb;
a.n = lo.n;
- nthreads = rb * ntiles_out;
+ return rb;
}
-template <int MT> struct TagGuS3;
-template <int MT> struct TagDnNL;
+template <int F, int MT> struct TagGu;
+template <int F, int MT> struct TagDn;
-// ------------------------------------------------------------------------------------------------ gate/up, IQ3_S
-template <int MT>
-void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t* grid) {
+// ------------------------------------------------------------------------------------------------ gate/up
+template <int F, int MT>
+void launch_gu(sycl::queue& q, const GArgs a, int nrowblocks) {
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<256>};
+ constexpr int NS = (F == T_IQ2S) ? 2 : 1; // IQ2_S: two scales per 32 values -> two DPAS with the other half zeroed
q.submit([&](sycl::handler& h) {
- h.parallel_for<TagGuS3<MT>>(sycl::range<2>((size_t) (nthreads / a.ntiles), (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
- // a 2-D grid: this device's ESIMD compiler gave `t % n` = t for a run-time n (integer / and % by a non-constant are not usable here)
+ h.parallel_for<TagGu<F, MT>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int rbg = (int) id[0], nt = (int) id[1];
- const int t = rbg * a.ntiles + nt;
const Pick pk = pick_expert(a, rbg);
- const int e = pk.e;
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
- const int nsb = a.nkb >> 3;
+ const uint32_t* T = a.tab;
simd<uint32_t, 16> lane(0, 1);
- simd<uint32_t, 16> rowoff = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;
+ simd<uint32_t, 16> ro = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;
simd<float, 128 * MT> acc = 0.f;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
- for (int sb = 0; sb < ((a.dbg & 8) ? 0 : nsb); ++sb) {
- const uint32_t P = (uint32_t) sb * BLK_IQ3S;
- simd<float, 16> d = half_of(ld16(We, rowoff + P));
- simd<uint32_t, 16> sc4 = ld32(We, rowoff + (P + 106));
- simd<uint32_t, 16> qh4 = ld32(We, rowoff + (P + 66));
- simd<uint32_t, 16> qh8 = ld32(We, rowoff + (P + 70));
-#pragma unroll
+ for (int sb = 0; sb < ((a.dbg & 8) ? 0 : (a.nkb >> 3)); ++sb) {
+ const uint32_t P = (uint32_t) sb * blk_bytes<F>();
+ const simd<float, 16> d = half_of(ld16(We, ro + P));
+ simd<uint32_t, 16> w0 = 0, w1 = 0, w2 = 0, w3 = 0; // sb-level fields
+ if constexpr (F == T_IQ3S) {
+ w0 = ld32(We, ro + (P + 106)); // scales
+ w1 = ld32(We, ro + (P + 66)); // qh 0..3
+ w2 = ld32(We, ro + (P + 70)); // qh 4..7
+ } else if constexpr (F == T_IQ2S) {
+ w0 = ld32(We, ro + (P + 74)); // scales 0..3
+ w1 = ld32(We, ro + (P + 78)); // scales 4..7
+ w2 = ld32(We, ro + (P + 66)); // qh 0..3
+ w3 = ld32(We, ro + (P + 70)); // qh 4..7
+ }
+#pragma unroll 1
for (int ib = 0; ib < 8; ++ib) {
- simd<uint32_t, 128> frag;
- simd<float, 16> swv = d;
- if (a.dbg & 1) {
- frag = 0x01010101u;
- } else {
- simd<uint32_t, 16> q0 = ld32(We, rowoff + (P + 2 + 8 * ib));
- simd<uint32_t, 16> q1 = ld32(We, rowoff + (P + 6 + 8 * ib));
- simd<uint32_t, 16> sg = ld32(We, rowoff + (P + 74 + 4 * ib));
- simd<uint32_t, 16> qhv = ((ib < 4 ? qh4 : qh8) >> (8 * (ib & 3))) & 0xffu;
- simd<uint32_t, 16> scn = ((sc4 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;
- swv = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));
+ const int kb = sb * 8 + ib;
+ simd<uint32_t, 128> fr0 = 0x01010101u, fr1 = 0u;
+ simd<float, 16> sw0 = 1.f, sw1 = 1.f;
+ if (!(a.dbg & 1)) {
+ if constexpr (F == T_IQ3S) {
+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
+ simd<uint32_t, 16> sg = ld32(We, ro + (P + 74 + 4 * ib));
+ simd<uint32_t, 16> qhv = (((ib < 4) ? w1 : w2) >> (8 * (ib & 3))) & 0xffu;
+ simd<uint32_t, 16> scn = ((w0 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;
+ sw0 = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));
#pragma unroll
- for (int r = 0; r < 8; ++r) {
- simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
- simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);
- simd<uint32_t, 16> g = gather<uint32_t, 16>(grid, idx << 2);
- simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;
- simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);
- simd<uint32_t, 16> mask = m * 255u;
- frag.template select<16, 1>(16 * r) = (g ^ mask) + m;
- }
+ for (int r = 0; r < 8; ++r) {
+ simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
+ simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);
+ simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3S, idx << 2);
+ simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;
+ fr0.template select<16, 1>(16 * r) = apply_signs(g, nib);
+ }
+ } else if constexpr (F == T_IQ3XXS) {
+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));
+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));
+ simd<uint32_t, 16> aux = ld32(We, ro + (P + 66 + 4 * ib));
+ sw0 = d * simd<float, 16>(simd<int, 16>((aux >> 28) * 2u + 1u)) * 0.25f;
+#pragma unroll
+ for (int il = 0; il < 4; ++il) {
+ simd<uint32_t, 16> sbyte = gather<uint32_t, 16>(T + O_KSIGNS, ((aux >> (7 * il)) & 127u) << 2);
+#pragma unroll
+ for (int hh = 0; hh < 2; ++hh) {
+ const int r = 2 * il + hh;
+ simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;
+ simd<uint32_t, 16> idx = (qsw >> (8 * (r & 3))) & 0xffu;
+ simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3XXS, idx << 2);
+ fr0.template select<16, 1>(16 * r) = apply_signs(g, (sbyte >> (4 * hh)) & 0xfu);
+ }
+ }
+ } else if constexpr (F == T_IQ2S) {
+ simd<uint32_t, 16> idw = ld32(We, ro + (P + 2 + 4 * ib));
+ simd<uint32_t, 16> sgw = ld32(We, ro + (P + 34 + 4 * ib));
+ simd<uint32_t, 16> qhb = (((ib < 4) ? w2 : w3) >> (8 * (ib & 3))) & 0xffu;
+ simd<uint32_t, 16> scb = (((ib < 4) ? w0 : w1) >> (8 * (ib & 3))) & 0xffu;
+ sw0 = d * simd<float, 16>(simd<int, 16>((scb & 0xfu) * 2u + 1u)) * 0.125f;
+ sw1 = d * simd<float, 16>(simd<int, 16>((scb >> 4) * 2u + 1u)) * 0.125f;
+#pragma unroll
+ for (int il = 0; il < 4; ++il) {
+ simd<uint32_t, 16> idx = ((idw >> (8 * il)) & 0xffu) | ((qhb << (8 - 2 * il)) & 0x300u);
+ simd<uint32_t, 16> gl = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3));
+ simd<uint32_t, 16> gh = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3) + 4u);
+ simd<uint32_t, 16> sbyte = (sgw >> (8 * il)) & 0xffu;
+ simd<uint32_t, 16> vl = apply_signs(gl, sbyte & 0xfu);
+ simd<uint32_t, 16> vh = apply_signs(gh, sbyte >> 4);
+ if (il < 2) {
+ fr0.template select<16, 1>(16 * (2 * il)) = vl;
+ fr0.template select<16, 1>(16 * (2 * il + 1)) = vh;
+ } else {
+ fr1.template select<16, 1>(16 * (2 * il)) = vl;
+ fr1.template select<16, 1>(16 * (2 * il + 1)) = vh;
+ }
+ }
+ // fr0 holds values 0..15 (dwords 0..3, the rest zero), fr1 values 16..31 (dwords 4..7)
+ fr0.template select<64, 1>(64) = 0u;
+ fr1.template select<64, 1>(0) = 0u;
+ }
}
- const int kb = sb * 8 + ib;
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i < nvalid) {
@@ -207,11 +273,19 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*
av = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);
dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);
}
- simd<float, 128> tf = dpas_s8(frag, av);
+ simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
- acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (swv * dxr);
+ acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
+ }
+ if constexpr (NS == 2) {
+ simd<float, 128> tg = dpas_s8(fr1, av);
+#pragma unroll
+ for (int r = 0; r < 8; ++r) {
+ const float dxr = dxv[r];
+ acc.template select<16, 1>(128 * i + 16 * r) += tg.template select<16, 1>(16 * r) * (sw1 * dxr);
+ }
}
}
}
@@ -224,8 +298,7 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
- block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt,
- acc.template select<16, 1>(128 * i + 16 * r));
+ block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
@@ -233,42 +306,56 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*
});
}
-// ------------------------------------------------------------------------------------------------ down, IQ4_NL
-template <int MT>
-void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t* tab) {
+// ------------------------------------------------------------------------------------------------ down
+template <int F, int MT>
+void launch_dn(sycl::queue& q, const GArgs a, int nrowblocks) {
auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<256>};
q.submit([&](sycl::handler& h) {
- h.parallel_for<TagDnNL<MT>>(sycl::range<2>((size_t) (nthreads / a.ntiles), (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
+ h.parallel_for<TagDn<F, MT>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {
const int rbg = (int) id[0], nt = (int) id[1];
- const int t = rbg * a.ntiles + nt;
- const Pick pk = (a.dbg & 32) ? Pick{0, a.row0[0], a.row0[1] - a.row0[0], a.tile0[0], a.rb0[0], a.blob[0]} : pick_expert(a, rbg);
- const int e = pk.e;
+ const Pick pk = pick_expert(a, rbg);
const int cnt = pk.cnt;
const int ntl = (cnt + 7) >> 3;
const int tl0 = (rbg - pk.rb0) * MT;
const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;
const int tbase = pk.tile0 + tl0;
const uint8_t* We = pk.blob;
+ const uint32_t* T = a.tab;
simd<uint32_t, 16> lane(0, 1);
- simd<uint32_t, 16> rowoff = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;
+ simd<uint32_t, 16> ro = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;
simd<float, 128 * MT> acc = 0.f;
const int* Xi = reinterpret_cast<const int*>(a.Xq);
for (int kb = 0; kb < ((a.dbg & 8) ? 0 : a.nkb); ++kb) {
- const uint32_t P = (uint32_t) kb * BLK_IQ4NL;
- simd<float, 16> swv = 1.f;
- if (a.dbg & 128) { if (kb == 0) block_store<uint32_t, 16>(reinterpret_cast<uint32_t*>(a.Y) + (size_t) t * 16, rowoff + P); }
- else if (!(a.dbg & 16)) swv = half_of(ld16(We, rowoff + P));
- simd<uint32_t, 128> frag = 0x01010101u;
+ simd<float, 16> sw0 = 1.f;
+ simd<uint32_t, 128> fr0 = 0x01010101u;
+ if (!(a.dbg & 1)) {
+ if constexpr (F == T_IQ4NL) {
+ const uint32_t P = (uint32_t) kb * 18;
+ sw0 = half_of(ld16(We, ro + P));
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ simd<uint32_t, 16> qd = ld32(We, ro + (P + 2 + 4 * j));
+ // four byte-pair lookups (lo value in byte 0, hi value in byte 1), repacked into the two dwords
+ simd<uint32_t, 16> t0 = gather<uint32_t, 16>(T + O_IQ4, (qd & 0xffu) << 2);
+ simd<uint32_t, 16> t1 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 8) & 0xffu) << 2);
+ simd<uint32_t, 16> t2 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 16) & 0xffu) << 2);
+ simd<uint32_t, 16> t3 = gather<uint32_t, 16>(T + O_IQ4, (qd >> 24) << 2);
+ fr0.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);
+ fr0.template select<16, 1>(16 * (j + 4)) =
+ ((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);
+ }
+ } else { // T_Q2_0: 64 values per block (2 + 16 bytes), two 32-blocks per block, one table lookup per byte
+ const uint32_t P = (uint32_t) (kb >> 1) * 18;
+ const int hb = kb & 1;
+ sw0 = half_of(ld16(We, ro + P));
+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * hb));
+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * hb));
#pragma unroll
- for (int j = 0; j < ((a.dbg & 1) ? 0 : 4); ++j) {
- simd<uint32_t, 16> qd = ld32(We, rowoff + (P + 2 + 4 * j));
- // four byte-pair lookups (lo value in byte 0, hi value in byte 1), repacked into the two dwords
- simd<uint32_t, 16> t0 = gather<uint32_t, 16>(tab, (qd & 0xffu) << 2);
- simd<uint32_t, 16> t1 = gather<uint32_t, 16>(tab, ((qd >> 8) & 0xffu) << 2);
- simd<uint32_t, 16> t2 = gather<uint32_t, 16>(tab, ((qd >> 16) & 0xffu) << 2);
- simd<uint32_t, 16> t3 = gather<uint32_t, 16>(tab, (qd >> 24) << 2);
- frag.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);
- frag.template select<16, 1>(16 * (j + 4)) = ((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);
+ for (int r = 0; r < 8; ++r) {
+ simd<uint32_t, 16> qb = ((r < 4 ? q0 : q1) >> (8 * (r & 3))) & 0xffu;
+ fr0.template select<16, 1>(16 * r) = gather<uint32_t, 16>(T + O_Q2, qb << 2);
+ }
+ }
}
#pragma unroll
for (int i = 0; i < MT; ++i) {
@@ -279,11 +366,11 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t
av = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);
dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);
}
- simd<float, 128> tf = dpas_s8(frag, av);
+ simd<float, 128> tf = dpas_s8(fr0, av);
#pragma unroll
for (int r = 0; r < 8; ++r) {
const float dxr = dxv[r];
- acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (swv * dxr);
+ acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);
}
}
}
@@ -295,8 +382,7 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t
for (int r = 0; r < 8; ++r) {
const int rl = (tl0 + i) * 8 + r;
if (rl < cnt && !(a.dbg & 4))
- block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt,
- acc.template select<16, 1>(128 * i + 16 * r));
+ block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));
}
}
}
@@ -304,34 +390,32 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t
});
}
+// MT from the group's mean number of 8-row tiles per expert.
int pick_mt(const Layout& lo) {
- int tiles = lo.tile0[lo.n];
- const double mean = lo.n ? (double) tiles / lo.n : 0;
+ const double mean = lo.n ? (double) lo.tile0[lo.n] / lo.n : 0;
if (mean <= 2.0) return 2;
if (mean <= 4.0) return 4;
- if (mean <= 8.0) return 8;
- return 12;
+ return 8;
}
int dbg_flags() {
static const int v = [] { const char* s = std::getenv("STRATA_I8_DBG"); return s ? std::atoi(s) : 0; }();
return v;
}
-int mt_override() {
- static const int v = [] { const char* s = std::getenv("STRATA_I8_MT"); return s ? std::atoi(s) : 0; }();
- return v;
+int mt_for(const Layout& lo) {
+ static const int mo = [] { const char* s = std::getenv("STRATA_I8_MT"); return s ? std::atoi(s) : 0; }();
+ return mo ? mo : pick_mt(lo);
}
template <class F>
-void dispatch_mt(int mt, F&& f) {
+bool dispatch_mt(int mt, F&& f) {
switch (mt) {
- case 2: f(std::integral_constant<int, 2>{}); break;
- case 4: f(std::integral_constant<int, 4>{}); break;
- case 8: f(std::integral_constant<int, 8>{}); break;
- default: f(std::integral_constant<int, 12>{}); break;
+ case 2: f(std::integral_constant<int, 2>{}); return true;
+ case 4: f(std::integral_constant<int, 4>{}); return true;
+ case 8: f(std::integral_constant<int, 8>{}); return true;
+ default: return false;
}
}
-// ------------------------------------------------------------------------------------------------ quantize / swiglu
struct QArgs {
int n;
int row0[kMaxGroup + 1];
@@ -489,57 +573,54 @@ void swiglu_quantize(const float* GU, const Layout& lo, int n_ff, int8_t* Hq, fl
bool gate_up(int gu_type, const uint8_t* const* blob, size_t up_off, int n_ff, const Layout& lo, int K, const int8_t* Xq,
const float* dx, float* GU, void* stream) {
if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;
- if (gu_type != 21 || K % 256 != 0 || (2 * n_ff) % 16 != 0 || (up_off & 3) != 0) return false;
+ if ((gu_type != T_IQ3S && gu_type != T_IQ3XXS && gu_type != T_IQ2S) || K % 256 != 0 || (2 * n_ff) % 16 != 0 ||
+ (up_off & 3) != 0 || lo.n > kMaxGroup)
+ return false;
sycl::queue& q = *strata::q_of(stream);
- Tables& tb = tables(q);
GArgs a{};
for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];
a.sec_off = (uint32_t) up_off;
a.Xq = Xq; a.dx = dx; a.Y = GU;
+ a.tab = tables(q);
a.ntiles = 2 * n_ff / 16;
a.nkb = K / 32;
- a.rowb = (K / 256) * BLK_IQ3S;
+ a.rowb = (K / 256) * block_bytes_of(gu_type);
a.ldy = 2 * n_ff;
a.dbg = dbg_flags();
- const int mt = mt_override() ? mt_override() : pick_mt(lo);
- dispatch_mt(mt, [&](auto mtc) {
+ const int mt = mt_for(lo);
+ const int rb = fill_args_mt(a, lo, mt);
+ return dispatch_mt(mt, [&](auto mtc) {
constexpr int MT = decltype(mtc)::value;
- int nthreads = 0;
- fill_args<MT>(a, lo, a.ntiles, nthreads);
- launch_gu_iq3s<MT>(q, a, nthreads, tb.grid3s);
+ if (gu_type == T_IQ3S) launch_gu<T_IQ3S, MT>(q, a, rb);
+ else if (gu_type == T_IQ3XXS) launch_gu<T_IQ3XXS, MT>(q, a, rb);
+ else launch_gu<T_IQ2S, MT>(q, a, rb);
});
- return true;
}
bool down(int d_type, const uint8_t* const* blob, size_t down_off, int n_embd, const Layout& lo, int n_ff, const int8_t* Hq,
const float* dh, float* Dm, void* stream) {
if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;
- if (d_type != 20 || n_ff % 32 != 0 || n_embd % 16 != 0 || (down_off & 3) != 0) return false;
+ if ((d_type != T_IQ4NL && d_type != T_Q2_0) || n_ff % block_vals_of(d_type) != 0 || n_embd % 16 != 0 || (down_off & 3) != 0 ||
+ lo.n > kMaxGroup)
+ return false;
sycl::queue& q = *strata::q_of(stream);
- Tables& tb = tables(q);
GArgs a{};
for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];
a.sec_off = (uint32_t) down_off;
a.Xq = Hq; a.dx = dh; a.Y = Dm;
+ a.tab = tables(q);
a.ntiles = n_embd / 16;
a.nkb = n_ff / 32;
- a.rowb = (n_ff / 32) * BLK_IQ4NL;
+ a.rowb = (n_ff / block_vals_of(d_type)) * block_bytes_of(d_type);
a.ldy = n_embd;
a.dbg = dbg_flags();
- const int mt = mt_override() ? mt_override() : pick_mt(lo);
- dispatch_mt(mt, [&](auto mtc) {
+ const int mt = mt_for(lo);
+ const int rb = fill_args_mt(a, lo, mt);
+ return dispatch_mt(mt, [&](auto mtc) {
constexpr int MT = decltype(mtc)::value;
- int nthreads = 0;
- fill_args<MT>(a, lo, a.ntiles, nthreads);
- if (a.dbg & 64) {
- std::fprintf(stderr, "i8 down: MT %d n %d threads %d ntiles %d nkb %d rowb %d ldy %d sec_off %u blob0 %p Xq %p dx %p Y %p tab %p\n",
- MT, a.n, nthreads, a.ntiles, a.nkb, a.rowb, a.ldy, a.sec_off, (const void*) a.blob[0], (const void*) a.Xq,
- (const void*) a.dx, (void*) a.Y, (const void*) tb.iq4);
- for (int i = 0; i <= a.n; ++i) std::fprintf(stderr, " e%d row0 %d tile0 %d rb0 %d blob %p\n", i, a.row0[i], a.tile0[i], a.rb0[i], i < a.n ? (const void*) a.blob[i] : nullptr);
- }
- launch_dn_iq4nl<MT>(q, a, nthreads, tb.iq4);
+ if (d_type == T_IQ4NL) launch_dn<T_IQ4NL, MT>(q, a, rb);
+ else launch_dn<T_Q2_0, MT>(q, a, rb);
});
- return true;
}
} // namespace strata::prefill::i8
diff --git a/sycl/src/prefill/iq_tables.inc b/sycl/src/prefill/iq_tables.inc
index 0e9c2cd49..98a238ee9 100644
--- a/sycl/src/prefill/iq_tables.inc
+++ b/sycl/src/prefill/iq_tables.inc
@@ -1,5 +1,5 @@
// IQ lookup tables copied from llama.cpp ggml/src/ggml-common.h (MIT License, Copyright (c) 2023-2026 The ggml authors):
-// kmask_iq2xs, ksigns_iq2xs, iq3xxs_grid, iq3s_grid, kvalues_iq4nl. Same values the dequantize kernels read.
+// kmask_iq2xs, ksigns_iq2xs, iq3xxs_grid, iq3s_grid, kvalues_iq4nl, iq2s_grid. Same values the dequantize kernels read.
// Included by esimd_moe.dp.cpp after defining GGML_TABLE_BEGIN / GGML_TABLE_END.
GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)
1, 2, 4, 8, 16, 32, 64, 128
@@ -117,3 +117,261 @@ GGML_TABLE_END()
GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)
-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,
GGML_TABLE_END()
+GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)
+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,
+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,
+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,
+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,
+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,
+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,
+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,
+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,
+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,
+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,
+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,
+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,
+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,
+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,
+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,
+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,
+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,
+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,
+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,
+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,
+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,
+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,
+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,
+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,
+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,
+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,
+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,
+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,
+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,
+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,
+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,
+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,
+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,
+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,
+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,
+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,
+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,
+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,
+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,
+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,
+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,
+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,
+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,
+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,
+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,
+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,
+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,
+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,
+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,
+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,
+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,
+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,
+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,
+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,
+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,
+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,
+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,
+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,
+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,
+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,
+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,
+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,
+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,
+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,
+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,
+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,
+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,
+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,
+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,
+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,
+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,
+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,
+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,
+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,
+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,
+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,
+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,
+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,
+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,
+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,
+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,
+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,
+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,
+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,
+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,
+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,
+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,
+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,
+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,
+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,
+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,
+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,
+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,
+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,
+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,
+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,
+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,
+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,
+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,
+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,
+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,
+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,
+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,
+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,
+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,
+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,
+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,
+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,
+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,
+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,
+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,
+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,
+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,
+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,
+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,
+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,
+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,
+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,
+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,
+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,
+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,
+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,
+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,
+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,
+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,
+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,
+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,
+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,
+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,
+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,
+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,
+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,
+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,
+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,
+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,
+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,
+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,
+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,
+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,
+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,
+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,
+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,
+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,
+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,
+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,
+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,
+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,
+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,
+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,
+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,
+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,
+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,
+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,
+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,
+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,
+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,
+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,
+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,
+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,
+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,
+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,
+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,
+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,
+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,
+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,
+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,
+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,
+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,
+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,
+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,
+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,
+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,
+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,
+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,
+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,
+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,
+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,
+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,
+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,
+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,
+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,
+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,
+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,
+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,
+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,
+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,
+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,
+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,
+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,
+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,
+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,
+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,
+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,
+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,
+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,
+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,
+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,
+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,
+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,
+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,
+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,
+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,
+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,
+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,
+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,
+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,
+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,
+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,
+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,
+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,
+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,
+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,
+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,
+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,
+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,
+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,
+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,
+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,
+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,
+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,
+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,
+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,
+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,
+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,
+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,
+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,
+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,
+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,
+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,
+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,
+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,
+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,
+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,
+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,
+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,
+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,
+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,
+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,
+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,
+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,
+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,
+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,
+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,
+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,
+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,
+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,
+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,
+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,
+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,
+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,
+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,
+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,
+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,
+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,
+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,
+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,
+GGML_TABLE_END()
diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cpp
index 236136fd8..650dd77fc 100644
--- a/sycl/src/prefill/prefill.cpp
+++ b/sycl/src/prefill/prefill.cpp
@@ -42,6 +42,8 @@
#include "strata/prefill/kernels.hpp"
#include <algorithm>
+#include <set>
+#include <tuple>
#include <chrono>
#include <cmath>
#include <cstdio>
@@ -3963,10 +3965,18 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same
// held group of staged experts as above
const bool i8on = !xmx2_only && i8_mode() != 0 && !use_mmq && lay.native &&
- lay.fmt[(size_t) l].gu_type == 21 && lay.fmt[(size_t) l].d_type == 20 &&
+ (lay.fmt[(size_t) l].gu_type == 21 || lay.fmt[(size_t) l].gu_type == 18 || lay.fmt[(size_t) l].gu_type == 22) &&
+ (lay.fmt[(size_t) l].d_type == 20 || lay.fmt[(size_t) l].d_type == 42) &&
lay.fmt[(size_t) l].n_ff == 640 && lay.fmt[(size_t) l].n_embd == 2560 &&
strata::prefill::i8::available(m.cs);
const bool xmx2 = xmx2_only || i8on;
+ if (i8_mode() != 0 && lay.native) {
+ static std::set<std::tuple<int, int, int, int, int>> said_fmt;
+ const auto& ff = lay.fmt[(size_t) l];
+ if (said_fmt.insert({ff.gu_type, ff.d_type, (int) ff.n_ff, (int) ff.n_embd, (int) i8on}).second)
+ std::fprintf(stderr, "strata: layer %lld experts gate/up type %d, down type %d, n_ff %d, n_embd %d -> int8 path %s\n",
+ (long long) l, ff.gu_type, ff.d_type, (int) ff.n_ff, (int) ff.n_embd, i8on ? "yes" : "no");
+ }
if (strata::prefill::xmx::mode() == 2 && !xmx2) {
static bool said = false;
if (!said) {