Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
sycl: int8 DPAS expert products for IQ3_XXS and IQ2_S gate/up and Q2_0 down too (every layer type of the IQ3_S pack but one), thread tile NT=1 with MT 2/4/8, parity test over all six type pairs; STRATA_PREFILL_MMQ=1 prompts +10% (4K), +16% (2K), +20% (1K) with identical tokens
Niko1221ebdfaacd1cd9
diff --git a/sycl/src/kernels/esimd_moe_parity.cpp b/sycl/src/kernels/esimd_moe_parity.cppindex 269e67284..ebaddf2db 100644--- a/sycl/src/kernels/esimd_moe_parity.cpp+++ b/sycl/src/kernels/esimd_moe_parity.cpp@@ -1,10 +1,12 @@// esimd_moe_parity (151-b70): the fused int8 DPAS expert products (prefill/esimd_moe.dp.cpp) against// (a) the engine's dequantize + oneMKL path (FP16 weights, FP16 activations), the path they replace,-// (b) an FP64 host reference from the blobs (IQ3_S gate/up, IQ4_NL down) with the unquantized activations,+// (b) an FP64 host reference from the blobs with the unquantized activations,// (c) an exact host reference on the SAME int8 activations the device produced (isolates the kernel arithmetic from the-// activation quantization: only the float accumulation order differs).-// and times both paths per expert. esimd_moe_parity [mean_rows=80] [experts=16] [reps=10] [seed=1]-// Exit status 1 when (c) is off by more than 1e-5 of the row's scale or (a)/(b) exceed the int8 activation error budget.+// activation quantization: only the float accumulation order differs),+// for every gate/up x down type pair the kernels handle, and times both paths per expert.+// esimd_moe_parity [mean_rows=80] [experts=16] [reps=10] [seed=1] [gate_up_type down_type]+// types: gate/up 21 IQ3_S, 18 IQ3_XXS, 22 IQ2_S; down 20 IQ4_NL, 42 Q2_0.+// Exit status 1 when (c) is off by more than 2e-5 of the largest value or (b) exceeds the int8 activation error budget.#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/iq_dequant_expert.hpp"#include "strata/prefill/esimd_moe.hpp"@@ -30,46 +32,66 @@ namespace i8 = strata::prefill::i8;static uint16_t f2h(float f) { return sycl::bit_cast<uint16_t>(sycl::half(f)); }static float h2f(uint16_t h) { return (float) sycl::bit_cast<sycl::half>(h); }-// host decode of one IQ3_S row (K values) -> double-static void dec_iq3s(const uint8_t* row, int K, double* out) {- for (int sb = 0; sb < K / 256; ++sb) {- const uint8_t* blk = row + sb * 110;+static int block_bytes(int t) { return t == 21 ? 110 : t == 18 ? 98 : t == 22 ? 82 : 18; }+static int block_vals(int t) { return (t == 21 || t == 18 || t == 22) ? 256 : t == 42 ? 64 : 32; }++// host decode of one row (K values) -> double+static void dec_row(int t, const uint8_t* row, int K, double* out) {+ const int bb = block_bytes(t), bv = block_vals(t);+ for (int b = 0; b < K / bv; ++b) {+ const uint8_t* blk = row + (size_t) b * bb;uint16_t dh; std::memcpy(&dh, blk, 2);- const uint8_t *qs = blk + 2, *qh = blk + 66, *sgn = blk + 74, *scl = blk + 106;- for (int ib = 0; ib < 8; ++ib) {- const double d = (double) h2f(dh) * (1 + 2 * ((scl[ib / 2] >> (4 * (ib % 2))) & 0xf));- for (int r = 0; r < 8; ++r) {- const int idx = qs[8 * ib + r] | (((qh[ib] >> r) & 1) << 8);- const uint8_t* g = (const uint8_t*) &iq3s_grid[idx];- const uint8_t sbyte = sgn[4 * ib + r / 2];- for (int j = 0; j < 4; ++j) {- const int kk = 4 * (r & 1) + j;- out[sb * 256 + ib * 32 + 4 * r + j] = d * ((sbyte >> kk & 1) ? -(int) g[j] : (int) g[j]);+ const double d = h2f(dh);+ double* o = out + (size_t) b * bv;+ if (t == 21) {+ const uint8_t *qs = blk + 2, *qh = blk + 66, *sgn = blk + 74, *scl = blk + 106;+ for (int ib = 0; ib < 8; ++ib) {+ const double dd = d * (1 + 2 * ((scl[ib / 2] >> (4 * (ib % 2))) & 0xf));+ for (int r = 0; r < 8; ++r) {+ const int idx = qs[8 * ib + r] | (((qh[ib] >> r) & 1) << 8);+ const uint8_t* g = (const uint8_t*) &iq3s_grid[idx];+ const uint8_t sb = sgn[4 * ib + r / 2];+ for (int j = 0; j < 4; ++j) o[ib * 32 + 4 * r + j] = dd * ((sb >> (4 * (r & 1) + j) & 1) ? -(int) g[j] : (int) g[j]);}}- }- }-}-static void dec_iq4nl(const uint8_t* row, int K, double* out) {- for (int b = 0; b < K / 32; ++b) {- const uint8_t* blk = row + b * 18;- uint16_t dh; std::memcpy(&dh, blk, 2);- for (int j = 0; j < 16; ++j) {- out[b * 32 + j] = (double) h2f(dh) * kvalues_iq4nl[blk[2 + j] & 15];- out[b * 32 + 16 + j] = (double) h2f(dh) * kvalues_iq4nl[blk[2 + j] >> 4];+ } else if (t == 18) {+ const uint8_t* qs = blk + 2;+ for (int ib = 0; ib < 8; ++ib) {+ uint32_t aux; std::memcpy(&aux, blk + 66 + 4 * ib, 4);+ const double dd = d * (0.5 + (aux >> 28)) * 0.5;+ for (int l = 0; l < 4; ++l) {+ const uint8_t sb = ksigns_iq2xs[(aux >> (7 * l)) & 127];+ const uint8_t *g1 = (const uint8_t*) &iq3xxs_grid[qs[8 * ib + 2 * l]], *g2 = (const uint8_t*) &iq3xxs_grid[qs[8 * ib + 2 * l + 1]];+ for (int j = 0; j < 4; ++j) {+ o[ib * 32 + 8 * l + j] = dd * ((sb >> j & 1) ? -(int) g1[j] : (int) g1[j]);+ o[ib * 32 + 8 * l + 4 + j] = dd * ((sb >> (4 + j) & 1) ? -(int) g2[j] : (int) g2[j]);+ }+ }+ }+ } else if (t == 22) {+ const uint8_t *qs = blk + 2, *qh = blk + 66, *scl = blk + 74, *sgn = blk + 34;+ for (int ib = 0; ib < 8; ++ib)+ for (int l = 0; l < 4; ++l) {+ const int idx = qs[4 * ib + l] | ((qh[ib] << (8 - 2 * l)) & 0x300);+ const uint8_t* g = (const uint8_t*) &iq2s_grid[idx];+ const double dd = d * (0.5 + ((scl[ib] >> (4 * (l / 2))) & 0xf)) * 0.25;+ const uint8_t sb = sgn[4 * ib + l];+ for (int j = 0; j < 8; ++j) o[ib * 32 + 8 * l + j] = dd * ((sb >> j & 1) ? -(int) g[j] : (int) g[j]);+ }+ } else if (t == 20) {+ for (int j = 0; j < 16; ++j) {+ o[j] = d * kvalues_iq4nl[blk[2 + j] & 15];+ o[16 + j] = d * kvalues_iq4nl[blk[2 + j] >> 4];+ }+ } else { // 42 Q2_0+ for (int i = 0; i < 64; ++i) o[i] = d * (((blk[2 + i / 4] >> ((i % 4) * 2)) & 3) - 1);}}}-int main(int argc, char** argv) {- const int mean = argc > 1 ? std::atoi(argv[1]) : 80;- const int n = argc > 2 ? std::atoi(argv[2]) : 16;- const int reps = argc > 3 ? std::atoi(argv[3]) : 10;- const int seed = argc > 4 ? std::atoi(argv[4]) : 1;- sycl::queue* q = &dpct::get_in_order_queue();- if (!i8::available(q)) { std::printf("esimd_moe_parity: SKIP (no sub-group 16 / s8 DPAS device)\n"); return 0; }- dpct::blas::descriptor_ptr hd = new dpct::blas::descriptor();- hd->set_queue(q);+struct Result { bool ok; double t_ref, t_fused; };++static Result run_case(sycl::queue* q, dpct::blas::descriptor_ptr hd, int gt, int dt, int mean, int n, int reps, int seed) {const int K = 2560, NFF = 640, NE = 2560;std::mt19937 rng(seed);std::vector<int32_t> cnt(n);@@ -80,15 +102,15 @@ int main(int argc, char** argv) {i8::Layout lo;i8::make_layout(lo, cnt.data(), n);- const size_t gate_b = (size_t) NFF * (K / 256) * 110, down_b = (size_t) NE * (NFF / 32) * 18;- const size_t blob_b = 2 * gate_b + down_b;+ const size_t rowb_gu = (size_t) (K / block_vals(gt)) * block_bytes(gt), rowb_dn = (size_t) (NFF / block_vals(dt)) * block_bytes(dt);+ const size_t gate_b = NFF * rowb_gu, down_b = NE * rowb_dn, blob_b = 2 * gate_b + down_b;std::vector<std::vector<uint8_t>> hb(n);std::vector<uint8_t*> db(n);for (int i = 0; i < n; ++i) {hb[i].resize(blob_b);for (auto& x : hb[i]) x = (uint8_t) rng();- for (size_t o = 0; o < 2 * gate_b; o += 110) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }- for (size_t o = 2 * gate_b; o < blob_b; o += 18) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }+ for (size_t o = 0; o < 2 * gate_b; o += block_bytes(gt)) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }+ for (size_t o = 2 * gate_b; o < blob_b; o += block_bytes(dt)) { const uint16_t d = f2h(0.0004f + 0.00002f * (rng() % 50)); std::memcpy(&hb[i][o], &d, 2); }db[i] = sycl::malloc_device<uint8_t>(blob_b + 256, *q);q->memcpy(db[i], hb[i].data(), blob_b);}@@ -103,25 +125,23 @@ int main(int argc, char** argv) {float* dH = sycl::malloc_device<float>(i8::dx_bytes(lo, NFF) / 4, *q);float *GU = sycl::malloc_device<float>((size_t) tot * 2 * NFF, *q), *Dm = sycl::malloc_device<float>((size_t) tot * NE, *q);float *GUr = sycl::malloc_device<float>((size_t) tot * 2 * NFF, *q), *Dmr = sycl::malloc_device<float>((size_t) tot * NE, *q);- uint16_t *Hh = sycl::malloc_device<uint16_t>((size_t) tot * NFF, *q);+ uint16_t* Hh = sycl::malloc_device<uint16_t>((size_t) tot * NFF, *q);uint16_t* dwgu = sycl::malloc_device<uint16_t>((size_t) 2 * NFF * K, *q);uint16_t* dwd = sycl::malloc_device<uint16_t>((size_t) NE * NFF, *q);q->wait();std::vector<const uint8_t*> pb(n);- if (std::getenv("I8_STAGE")) { for (int i = 0; i < n && i < 4; ++i) std::printf("blob %d at %p size %zu\n", i, (void*) db[i], blob_b); }for (int i = 0; i < n; ++i) pb[i] = db[i];- auto fused = [&](bool all = true) {+ auto fused = [&]() {i8::quantize_rows(dxh, lo, K, Xq, dX, q);- if (!i8::gate_up(21, pb.data(), 2 * 0 + gate_b, NFF, lo, K, Xq, dX, GU, q)) { std::printf("gate_up refused\n"); std::exit(2); }+ if (!i8::gate_up(gt, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q)) { std::printf("gate_up refused (type %d)\n", gt); std::exit(2); }i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q);- if (!i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q)) { std::printf("down refused\n"); std::exit(2); }- (void) all;+ if (!i8::down(dt, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q)) { std::printf("down refused (type %d)\n", dt); std::exit(2); }};auto ref = [&]() {int64_t o = 0;for (int i = 0; i < n; ++i) {- strata::kernels::iq_dequant_expert_f16(21, 20, db[i], db[i] + gate_b, db[i] + 2 * gate_b, NFF, K, dwgu, dwd, q);+ strata::kernels::iq_dequant_expert_f16(gt, dt, db[i], db[i] + gate_b, db[i] + 2 * gate_b, NFF, K, dwgu, dwd, q);const float a = 1.f, b = 0.f;dpct::blas::gemm(hd, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, 2 * NFF, cnt[i], K, &a, dwgu,dpct::library_data_t::real_half, K, dxh + o * K, dpct::library_data_t::real_half, K, &b,@@ -143,23 +163,7 @@ int main(int argc, char** argv) {return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - a).count() / reps;};- if (const char* st = std::getenv("I8_STAGE")) { // bring-up: run the first N stages, synchronising after each- const int ns = std::atoi(st);- i8::quantize_rows(dxh, lo, K, Xq, dX, q); q->wait(); std::printf("stage 1 quantize ok%c", 10);- if (ns >= 2) { i8::gate_up(21, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); q->wait(); std::printf("stage 2 gate_up ok%c", 10); }- if (ns >= 3) { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); q->wait(); std::printf("stage 3 swiglu ok%c", 10); }- if (ns >= 4) { i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); q->wait(); std::printf("stage 4 down ok%c", 10); }- if (const char* dd = std::getenv("STRATA_I8_DBG"); dd && (std::atoi(dd) & 128)) { // the down kernel stored rowoff + P per thread- std::vector<uint32_t> o((size_t) 480 * 16);- q->memcpy(o.data(), Dm, o.size() * 4).wait();- uint32_t mx = 0, mn = ~0u;- for (int th = 0; th < 480; ++th) for (int l = 0; l < 16; ++l) { mx = std::max(mx, o[th * 16 + l]); mn = std::min(mn, o[th * 16 + l]); }- std::printf("rowoff+P: min %u max %u%c", mn, mx, 10); for (int th : {0, 1, 158, 159, 160, 161, 319, 320, 321, 479}) std::printf(" t%d lane0 %u row %u%c", th, o[th * 16], (o[th * 16] - 1408000) / 360, 10);- }- return 0;- }fused(); q->wait();- // ---- (c) exact reference on the device's int8 activationsstd::vector<int8_t> hXq(i8::xq_bytes(lo, K));std::vector<float> hdX(i8::dx_bytes(lo, K) / 4), hGU((size_t) tot * 2 * NFF);q->memcpy(hXq.data(), Xq, hXq.size());@@ -168,36 +172,33 @@ int main(int argc, char** argv) {const int nkb = K / 32;auto xq_at = [&](int tile, int r, int k) { return (int) hXq[(((size_t) tile * nkb + k / 32) * 8 + r) * 32 + (k % 32)]; };auto dx_at = [&](int tile, int r, int kb) { return (double) hdX[((size_t) tile * nkb + kb) * 8 + r]; };- int fails = 0;- double err_c = 0, mx_c = 0, err_b = 0, rms_b = 0, rms_bref = 0, mx_b = 0;+ bool ok = true;std::vector<double> wrow(K);- {- int sampled = 0;- for (int e = 0; e < n; ++e) {- for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {- const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;- for (int v = 0; v < 2 * NFF; v += 37) {- dec_iq3s(&hb[e][(size_t) (v & 1 ? gate_b : 0) + (size_t) (v >> 1) * (K / 256) * 110], K, wrow.data());- double refc = 0, refb = 0;- for (int kb = 0; kb < nkb; ++kb) {- double s = 0;- for (int k = 0; k < 32; ++k) s += (double) xq_at(tile, r, kb * 32 + k) * wrow[kb * 32 + k];- refc += s * dx_at(tile, r, kb);- }- for (int k = 0; k < K; ++k) refb += (double) h2f(hx[(size_t) row * K + k]) * wrow[k];- const double got = hGU[(size_t) row * 2 * NFF + v];- err_c = std::max(err_c, std::fabs(got - refc)); mx_c = std::max(mx_c, std::fabs(refc));- err_b += (got - refb) * (got - refb); rms_bref += refb * refb; mx_b = std::max(mx_b, std::fabs(refb));- ++sampled;+ double err_c = 0, mx_c = 0, err_b = 0, rms_bref = 0;+ int sampled = 0;+ for (int e = 0; e < n; ++e)+ for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {+ const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;+ for (int v = 0; v < 2 * NFF; v += 37) {+ dec_row(gt, &hb[e][(size_t) (v & 1 ? gate_b : 0) + (size_t) (v >> 1) * rowb_gu], K, wrow.data());+ double refc = 0, refb = 0;+ for (int kb = 0; kb < nkb; ++kb) {+ double s = 0;+ for (int k = 0; k < 32; ++k) s += (double) xq_at(tile, r, kb * 32 + k) * wrow[kb * 32 + k];+ refc += s * dx_at(tile, r, kb);}+ for (int k = 0; k < K; ++k) refb += (double) h2f(hx[(size_t) row * K + k]) * wrow[k];+ const double got = hGU[(size_t) row * 2 * NFF + v];+ err_c = std::max(err_c, std::fabs(got - refc)); mx_c = std::max(mx_c, std::fabs(refc));+ err_b += (got - refb) * (got - refb); rms_bref += refb * refb;+ ++sampled;}}- const bool ok = err_c <= 2e-5 * mx_c && std::sqrt(err_b / rms_bref) < 0.02;- fails += !ok;- std::printf("gate/up IQ3_S : exact-int8 reference max err %.2e of %.2e (%.1e) vs FP64 true: rms rel err %.2e (%d samples) %s\n",- err_c, mx_c, err_c / mx_c, std::sqrt(err_b / rms_bref), sampled, ok ? "OK" : "FAIL");- }- { // down: needs the device's Hq / dH and the true H from the device GU (float math on the host)+ const bool ok1 = err_c <= 2e-5 * mx_c && std::sqrt(err_b / rms_bref) < 0.02;+ ok = ok && ok1;+ std::printf(" gate/up type %2d: exact-int8 ref max err %.2e of %.2e (%.1e) vs FP64 true rms rel err %.2e (%d samples) %s\n", gt,+ err_c, mx_c, err_c / mx_c, std::sqrt(err_b / rms_bref), sampled, ok1 ? "OK" : "FAIL");+ {std::vector<int8_t> hHq(i8::xq_bytes(lo, NFF));std::vector<float> hdH(i8::dx_bytes(lo, NFF) / 4), hDm((size_t) tot * NE);q->memcpy(hHq.data(), Hq, hHq.size());@@ -206,11 +207,10 @@ int main(int argc, char** argv) {const int nkh = NFF / 32;double ec = 0, mc = 0, eb = 0, rb = 0, hq_bad = 0;std::vector<double> drow(NFF);- int sampled = 0;+ int smp = 0;for (int e = 0; e < n; ++e)for (int rl = 0; rl < cnt[e]; rl += (cnt[e] > 12 ? 5 : 1)) {const int tile = lo.tile0[e] + rl / 8, r = rl % 8, row = lo.row0[e] + rl;- // the true H (double) of this row, and a check of the device's swiglu quantizationstd::vector<double> H(NFF);for (int j = 0; j < NFF; ++j) {const double g = hGU[(size_t) row * 2 * NFF + 2 * j], u = hGU[(size_t) row * 2 * NFF + 2 * j + 1];@@ -222,7 +222,7 @@ int main(int argc, char** argv) {if (std::fabs(dd - amax / 127.0) > 1e-5 * amax + 1e-30) hq_bad += 1;}for (int v = 0; v < NE; v += 41) {- dec_iq4nl(&hb[e][2 * gate_b + (size_t) v * nkh * 18], NFF, drow.data());+ dec_row(dt, &hb[e][2 * gate_b + (size_t) v * rowb_dn], NFF, drow.data());double refc = 0, refb = 0;for (int kb = 0; kb < nkh; ++kb) {double s = 0;@@ -233,15 +233,14 @@ int main(int argc, char** argv) {const double got = hDm[(size_t) row * NE + v];ec = std::max(ec, std::fabs(got - refc)); mc = std::max(mc, std::fabs(refc));eb += (got - refb) * (got - refb); rb += refb * refb;- ++sampled;+ ++smp;}}- const bool ok = ec <= 2e-5 * mc && std::sqrt(eb / rb) < 0.03 && hq_bad == 0;- fails += !ok;- std::printf("down IQ4_NL : exact-int8 reference max err %.2e of %.2e (%.1e) vs FP64 true: rms rel err %.2e, swiglu scale mismatches %.0f (%d samples) %s\n",- ec, mc, ec / mc, std::sqrt(eb / rb), hq_bad, sampled, ok ? "OK" : "FAIL");+ const bool ok2 = ec <= 2e-5 * mc && std::sqrt(eb / rb) < 0.03 && hq_bad == 0;+ ok = ok && ok2;+ std::printf(" down type %2d : exact-int8 ref max err %.2e of %.2e (%.1e) vs FP64 true rms rel err %.2e, swiglu scale mismatches %.0f (%d samples) %s\n",+ dt, ec, mc, ec / mc, std::sqrt(eb / rb), hq_bad, smp, ok2 ? "OK" : "FAIL");}- // ---- (a) against the dequantize + oneMKL path, whole chainref(); q->wait();{std::vector<float> a((size_t) tot * NE), b((size_t) tot * NE);@@ -249,15 +248,37 @@ int main(int argc, char** argv) {q->memcpy(b.data(), Dm, b.size() * 4).wait();double e2 = 0, r2 = 0;for (size_t i = 0; i < a.size(); ++i) { e2 += (double) (a[i] - b[i]) * (a[i] - b[i]); r2 += (double) a[i] * a[i]; }- std::printf("chain vs dequant + oneMKL (FP16 path): rms rel diff %.2e (rows %d)\n", std::sqrt(e2 / r2), tot);+ std::printf(" chain vs dequantize + oneMKL (FP16 path): rms rel diff %.2e (rows %d)\n", std::sqrt(e2 / r2), tot);+ }+ Result res{ok, tm(ref) / n, tm(fused) / n};+ const double t_q = tm([&] { i8::quantize_rows(dxh, lo, K, Xq, dX, q); }) / n;+ const double t_gu = tm([&] { i8::gate_up(gt, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); }) / n;+ const double t_sw = tm([&] { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); }) / n;+ const double t_dn = tm([&] { i8::down(dt, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); }) / n;+ std::printf(" rows %d (%d experts, mean %.0f): dequant+oneMKL %.1f us/expert | fused int8 %.1f us/expert (%.2fx) [quantize %.1f, gate/up %.1f, swiglu+q %.1f, down %.1f]\n",+ tot, n, (double) tot / n, res.t_ref, res.t_fused, res.t_ref / res.t_fused, t_q, t_gu, t_sw, t_dn);+ for (int i = 0; i < n; ++i) sycl::free(db[i], *q);+ sycl::free(dxh, *q); sycl::free(Xq, *q); sycl::free(dX, *q); sycl::free(Hq, *q); sycl::free(dH, *q); sycl::free(GU, *q);+ sycl::free(Dm, *q); sycl::free(GUr, *q); sycl::free(Dmr, *q); sycl::free(Hh, *q); sycl::free(dwgu, *q); sycl::free(dwd, *q);+ return res;+}++int main(int argc, char** argv) {+ const int mean = argc > 1 ? std::atoi(argv[1]) : 80;+ const int n = argc > 2 ? std::atoi(argv[2]) : 16;+ const int reps = argc > 3 ? std::atoi(argv[3]) : 10;+ const int seed = argc > 4 ? std::atoi(argv[4]) : 1;+ sycl::queue* q = &dpct::get_in_order_queue();+ if (!i8::available(q)) { std::printf("esimd_moe_parity: SKIP (no ESIMD device with the s8 8x16x32 DPAS)\n"); return 0; }+ dpct::blas::descriptor_ptr hd = new dpct::blas::descriptor();+ hd->set_queue(q);+ std::vector<std::pair<int, int>> cases = {{21, 20}, {18, 20}, {22, 20}, {21, 42}, {18, 42}, {22, 42}};+ if (argc > 6) cases = {{std::atoi(argv[5]), std::atoi(argv[6])}};+ int fails = 0;+ for (auto [gt, dt] : cases) {+ std::printf("gate/up %d, down %d, mean %d rows, %d experts\n", gt, dt, mean, n);+ fails += !run_case(q, hd, gt, dt, mean, n, reps, seed).ok;}- const double t_ref = tm(ref), t_f = tm([&] { fused(); });- const double t_q = tm([&] { i8::quantize_rows(dxh, lo, K, Xq, dX, q); });- const double t_gu = tm([&] { i8::gate_up(21, pb.data(), gate_b, NFF, lo, K, Xq, dX, GU, q); });- const double t_sw = tm([&] { i8::swiglu_quantize(GU, lo, NFF, Hq, dH, q); });- const double t_dn = tm([&] { i8::down(20, pb.data(), 2 * gate_b, NE, lo, NFF, Hq, dH, Dm, q); });- std::printf("rows %d (%d experts, mean %.0f): dequant+oneMKL chain %.1f us/expert | fused int8 chain %.1f us/expert (%.2fx) [quantize %.1f, gate/up %.1f, swiglu+q %.1f, down %.1f]\n",- tot, n, (double) tot / n, t_ref / n, t_f / n, t_ref / t_f, t_q / n, t_gu / n, t_sw / n, t_dn / n);std::printf("%s\n", fails ? "FAILED" : "all OK");return fails ? 1 : 0;}diff --git a/sycl/src/prefill/esimd_moe.dp.cpp b/sycl/src/prefill/esimd_moe.dp.cppindex 40794a832..0d4d4b3ad 100644--- a/sycl/src/prefill/esimd_moe.dp.cpp+++ b/sycl/src/prefill/esimd_moe.dp.cpp@@ -1,24 +1,26 @@-// 151-b70: fused int8 DPAS expert products (ESIMD, Battlemage) - see include/strata/prefill/esimd_moe.hpp.+// 151-b70: fused int8 DPAS expert products (ESIMD, Battlemage) - see include/strata/prefill/esimd_moe.hpp and+// docs/INTEL_INT8_EXPERTS.md (the behaviour note).//-// Behaviour note (docs/INTEL.md, "int8 expert path"):// * Tile. One hardware thread owns 16 output rows (a DPAS N tile; for gate/up the 16 virtual rows alternate gate/up) and up to-// MT x 8 activation rows (MT = 2, 4, 8 or 12, picked per group from the experts' mean row count). It walks K in steps of 32:+// MT x 8 activation rows (MT = 2, 4 or 8, picked per group from the experts' mean row count). It walks K in steps of 32:// decode the 16 weight rows' 32 values into the B operand (registers only), then for each of its row tiles one s8 DPAS 8x16x32// into a zeroed int32 tile, converted to float and scaled by (weight scale of the row and 32-block) x (activation scale of the-// row and 32-block) into a float accumulator. A 128-wide work-group tile (the owner's 16/32/64 rows x 128 outputs) is eight of-// these threads side by side; there is no shared memory and no barrier.+// row and 32-block) into a float accumulator. There is no shared memory and no barrier.// * Decode once per tile: the B operand is decoded once per thread and applied to all its MT row tiles. An expert with more rows-// than 8 x MT is split over row-block threads, each decoding again (the redundancy is ceil(rows / (8 MT)); MT 12 keeps it 1 up-// to 96 rows).+// than 8 x MT is split over row-block threads, each decoding again. A thread's accumulators are MT tiles of 512 bytes in an+// 8 KB register file (256 GRF of 32 bytes): MT 12, or two output tiles per thread, spilled (3-15 KB) and ran 2-5x slower.// * Scales. Activations: q8_1 as llama.cpp (d = amax / 127, q = round half away from zero), one float per row and 32 of K.-// Weights: IQ3_S d * (1 + 2 s) per 32 (exact in float), IQ4_NL d per 32; the int8 values are the codebook entries with their-// signs, never rounded. The int32 dot product of a 32-block is at most 32 * 127 * 15 (IQ3_S) so it is exact, and int32 -> float-// is exact below 2^24. The float accumulation order is fixed (K ascending).+// Weights: IQ3_S d (1 + 2 s), IQ3_XXS d (2 s + 1) / 4, IQ2_S d (2 s + 1) / 8 per 16 (two DPAS per 32 with the other half+// zeroed), IQ4_NL d per 32, Q2_0 d per 64; the int8 values are the codebook entries with their signs, never rounded. The int32+// dot product of a 32-block is exact and exactly convertible to float. The float accumulation order is fixed (K ascending).// * Edge cases. Rows are padded to tiles of 8 with zeros (stores are masked to the real rows); a group of up to 16 experts per-// launch; K a multiple of 256 (IQ3_S) / 32 (IQ4_NL); output rows a multiple of 16. Blob reads never go past a matrix end-// (the last dword of a block is read as two dwords only when the block is not dword aligned).-// * Fallback. available() is false without sub-group 16 DPAS (a non-Battlemage device) and the caller keeps the dequantize ++// launch; K a multiple of 256 (IQ3_S / IQ3_XXS / IQ2_S) / 64 (Q2_0) / 32 (IQ4_NL); output rows a multiple of 16. Blob reads+// never go past a matrix end.+// * Fallback. available() is false without the s8 8x16x32 DPAS (a non-Battlemage device) and the caller keeps the dequantize +// oneMKL path; gate_up()/down() return false for a type / shape they do not handle.+// * ESIMD on this compiler (oneAPI 2026.1, bmg-g31): a dynamic index into the kernel-argument arrays, and `/` or `%` by a run-time+// value, gave wrong values / garbage pointers (the first weight read faulted the GPU); the kernels use constant-index reads+// (pick_expert) and 2-D grids. gather<T, N, VS> with VS > 1 is not available on Battlemage.#include <sycl/sycl.hpp>#include <sycl/ext/intel/esimd.hpp>#include <sycl/ext/intel/experimental/grf_size_properties.hpp>@@ -30,7 +32,7 @@#include <mutex>#include <type_traits>#include <string>-#include <unordered_map>+#include <vector>#include "strata/sycl_queue.hpp"#include "strata/prefill/esimd_moe.hpp"@@ -46,26 +48,41 @@ using namespace sycl::ext::intel::esimd;namespace xmx = sycl::ext::intel::esimd::xmx;using half = sycl::half;-constexpr int BLK_IQ3S = 110; // sizeof(block_iq3_s)-constexpr int BLK_IQ4NL = 18; // sizeof(block_iq4_nl)+// ggml types handled+constexpr int T_IQ3XXS = 18, T_IQ4NL = 20, T_IQ3S = 21, T_IQ2S = 22, T_Q2_0 = 42;+template <int F> constexpr int blk_bytes() { return F == T_IQ3S ? 110 : F == T_IQ3XXS ? 98 : F == T_IQ2S ? 82 : F == T_IQ4NL ? 18 : 18; }+constexpr int block_bytes_of(int f) { return f == T_IQ3S ? 110 : f == T_IQ3XXS ? 98 : f == T_IQ2S ? 82 : 18; }+constexpr int block_vals_of(int f) { return (f == T_IQ3S || f == T_IQ3XXS || f == T_IQ2S) ? 256 : f == T_Q2_0 ? 64 : 32; }-struct Tables {- uint32_t* grid3s = nullptr; // iq3s_grid[512]- uint32_t* iq4 = nullptr; // [256]: byte b -> kvalues[b & 15] in byte 0, kvalues[b >> 4] in byte 1-};-Tables& tables(sycl::queue& q) {+// One device table buffer (uint32 units): the codebooks the decoders gather from.+constexpr int O_IQ3S = 0; // iq3s_grid[512]+constexpr int O_IQ3XXS = 512; // iq3xxs_grid[256]+constexpr int O_IQ2S = 768; // iq2s_grid[1024] as 2048 dwords (low dword of an entry first)+constexpr int O_KSIGNS = 2816; // ksigns_iq2xs[128], one byte in the low byte of each dword+constexpr int O_IQ4 = 2944; // [256]: byte b -> kvalues_iq4nl[b & 15] in byte 0, kvalues_iq4nl[b >> 4] in byte 1+constexpr int O_Q2 = 3200; // [256]: byte b -> its four 2-bit codes minus 1, as four int8 (Q2_0 values)+constexpr int N_TABLE = 3456;++uint32_t* tables(sycl::queue& q) {static std::mutex mu;- static Tables t; // one device per process in the port+ static uint32_t* t = nullptr; // one device per process in the portstd::lock_guard<std::mutex> lk(mu);- if (!t.grid3s) {- t.grid3s = sycl::malloc_device<uint32_t>(512, q);- t.iq4 = sycl::malloc_device<uint32_t>(256, q);- uint32_t h4[256];- for (int b = 0; b < 256; ++b)- h4[b] = (uint32_t) (uint8_t) kvalues_iq4nl[b & 15] | ((uint32_t) (uint8_t) kvalues_iq4nl[b >> 4] << 8);- q.memcpy(t.grid3s, iq3s_grid, 512 * 4);- q.memcpy(t.iq4, h4, 256 * 4);+ if (!t) {+ std::vector<uint32_t> h(N_TABLE, 0);+ for (int i = 0; i < 512; ++i) h[O_IQ3S + i] = iq3s_grid[i];+ for (int i = 0; i < 256; ++i) h[O_IQ3XXS + i] = iq3xxs_grid[i];+ for (int i = 0; i < 1024; ++i) { h[O_IQ2S + 2 * i] = (uint32_t) iq2s_grid[i]; h[O_IQ2S + 2 * i + 1] = (uint32_t) (iq2s_grid[i] >> 32); }+ for (int i = 0; i < 128; ++i) h[O_KSIGNS + i] = ksigns_iq2xs[i];+ for (int b = 0; b < 256; ++b) {+ h[O_IQ4 + b] = (uint32_t) (uint8_t) kvalues_iq4nl[b & 15] | ((uint32_t) (uint8_t) kvalues_iq4nl[b >> 4] << 8);+ uint32_t q2 = 0;+ for (int k = 0; k < 4; ++k) q2 |= (uint32_t) (uint8_t) (int8_t) (((b >> (2 * k)) & 3) - 1) << (8 * k);+ h[O_Q2 + b] = q2;+ }+ uint32_t* d = sycl::malloc_device<uint32_t>(N_TABLE, q);+ q.memcpy(d, h.data(), N_TABLE * 4);q.wait();+ t = d;}return t;}@@ -93,6 +110,12 @@ SYCL_ESIMD_FUNCTION inline simd<int, 128> dpas_s8(simd<uint32_t, 128> b, simd<insimd<int, 128> bi = b.template bit_cast_view<int>();return xmx::dpas<8, 8, int, int, int, int, xmx::dpas_argument_type::s8, xmx::dpas_argument_type::s8>(simd<int, 128>(0), bi, a);}+// sign nibble -> four negation bytes: (g ^ mask) + m negates the bytes whose sign bit is set (g is never 0xFF per byte)+SYCL_ESIMD_FUNCTION inline simd<uint32_t, 16> apply_signs(simd<uint32_t, 16> g, simd<uint32_t, 16> nib) {+ simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);+ simd<uint32_t, 16> mask = m * 255u;+ return (g ^ mask) + m;+}struct GArgs {const uint8_t* blob[kMaxGroup];@@ -109,12 +132,10 @@ struct GArgs {int nkb; // K / 32int rowb; // bytes per weight rowint ldy;- int dbg; // bring-up switches (STRATA_I8_DBG): 1 skip the decode, 2 skip the activation loads, 4 skip the stores+ int dbg; // bring-up switches (STRATA_I8_DBG): 1 skip the decode, 2 skip the activation loads, 4 skip the stores, 8 skip the K loop};--// The expert a work item belongs to, with constant-index reads of the kernel argument arrays (a dynamic index into the argument-// struct gave a garbage pointer in an ESIMD kernel on the B70: the first weight read faulted the GPU).+// The expert a work item belongs to, with constant-index reads of the kernel argument arrays.struct Pick { int e, row0, cnt, tile0, rb0; const uint8_t* blob; };SYCL_ESIMD_FUNCTION inline Pick pick_expert(const GArgs& a, int rbg) {Pick p{0, a.row0[0], a.row0[1] - a.row0[0], a.tile0[0], a.rb0[0], a.blob[0]};@@ -124,9 +145,7 @@ SYCL_ESIMD_FUNCTION inline Pick pick_expert(const GArgs& a, int rbg) {return p;}-struct LaunchInfo { int nthreads; };-template <int MT>-void fill_args(GArgs& a, const Layout& lo, int ntiles_out, int& nthreads) {+int fill_args_mt(GArgs& a, const Layout& lo, int MT) {int rb = 0;for (int i = 0; i < lo.n; ++i) {a.row0[i] = lo.row0[i];@@ -139,65 +158,112 @@ void fill_args(GArgs& a, const Layout& lo, int ntiles_out, int& nthreads) {a.tile0[lo.n] = lo.tile0[lo.n];a.rb0[lo.n] = rb;a.n = lo.n;- nthreads = rb * ntiles_out;+ return rb;}-template <int MT> struct TagGuS3;-template <int MT> struct TagDnNL;+template <int F, int MT> struct TagGu;+template <int F, int MT> struct TagDn;-// ------------------------------------------------------------------------------------------------ gate/up, IQ3_S-template <int MT>-void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t* grid) {+// ------------------------------------------------------------------------------------------------ gate/up+template <int F, int MT>+void launch_gu(sycl::queue& q, const GArgs a, int nrowblocks) {auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<256>};+ constexpr int NS = (F == T_IQ2S) ? 2 : 1; // IQ2_S: two scales per 32 values -> two DPAS with the other half zeroedq.submit([&](sycl::handler& h) {- h.parallel_for<TagGuS3<MT>>(sycl::range<2>((size_t) (nthreads / a.ntiles), (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {- // a 2-D grid: this device's ESIMD compiler gave `t % n` = t for a run-time n (integer / and % by a non-constant are not usable here)+ h.parallel_for<TagGu<F, MT>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {const int rbg = (int) id[0], nt = (int) id[1];- const int t = rbg * a.ntiles + nt;const Pick pk = pick_expert(a, rbg);- const int e = pk.e;const int cnt = pk.cnt;const int ntl = (cnt + 7) >> 3;const int tl0 = (rbg - pk.rb0) * MT;const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;const int tbase = pk.tile0 + tl0;const uint8_t* We = pk.blob;- const int nsb = a.nkb >> 3;+ const uint32_t* T = a.tab;simd<uint32_t, 16> lane(0, 1);- simd<uint32_t, 16> rowoff = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;+ simd<uint32_t, 16> ro = (lane & 1u) * a.sec_off + ((lane >> 1) + (uint32_t) (8 * nt)) * (uint32_t) a.rowb;simd<float, 128 * MT> acc = 0.f;const int* Xi = reinterpret_cast<const int*>(a.Xq);- for (int sb = 0; sb < ((a.dbg & 8) ? 0 : nsb); ++sb) {- const uint32_t P = (uint32_t) sb * BLK_IQ3S;- simd<float, 16> d = half_of(ld16(We, rowoff + P));- simd<uint32_t, 16> sc4 = ld32(We, rowoff + (P + 106));- simd<uint32_t, 16> qh4 = ld32(We, rowoff + (P + 66));- simd<uint32_t, 16> qh8 = ld32(We, rowoff + (P + 70));-#pragma unroll+ for (int sb = 0; sb < ((a.dbg & 8) ? 0 : (a.nkb >> 3)); ++sb) {+ const uint32_t P = (uint32_t) sb * blk_bytes<F>();+ const simd<float, 16> d = half_of(ld16(We, ro + P));+ simd<uint32_t, 16> w0 = 0, w1 = 0, w2 = 0, w3 = 0; // sb-level fields+ if constexpr (F == T_IQ3S) {+ w0 = ld32(We, ro + (P + 106)); // scales+ w1 = ld32(We, ro + (P + 66)); // qh 0..3+ w2 = ld32(We, ro + (P + 70)); // qh 4..7+ } else if constexpr (F == T_IQ2S) {+ w0 = ld32(We, ro + (P + 74)); // scales 0..3+ w1 = ld32(We, ro + (P + 78)); // scales 4..7+ w2 = ld32(We, ro + (P + 66)); // qh 0..3+ w3 = ld32(We, ro + (P + 70)); // qh 4..7+ }+#pragma unroll 1for (int ib = 0; ib < 8; ++ib) {- simd<uint32_t, 128> frag;- simd<float, 16> swv = d;- if (a.dbg & 1) {- frag = 0x01010101u;- } else {- simd<uint32_t, 16> q0 = ld32(We, rowoff + (P + 2 + 8 * ib));- simd<uint32_t, 16> q1 = ld32(We, rowoff + (P + 6 + 8 * ib));- simd<uint32_t, 16> sg = ld32(We, rowoff + (P + 74 + 4 * ib));- simd<uint32_t, 16> qhv = ((ib < 4 ? qh4 : qh8) >> (8 * (ib & 3))) & 0xffu;- simd<uint32_t, 16> scn = ((sc4 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;- swv = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));+ const int kb = sb * 8 + ib;+ simd<uint32_t, 128> fr0 = 0x01010101u, fr1 = 0u;+ simd<float, 16> sw0 = 1.f, sw1 = 1.f;+ if (!(a.dbg & 1)) {+ if constexpr (F == T_IQ3S) {+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));+ simd<uint32_t, 16> sg = ld32(We, ro + (P + 74 + 4 * ib));+ simd<uint32_t, 16> qhv = (((ib < 4) ? w1 : w2) >> (8 * (ib & 3))) & 0xffu;+ simd<uint32_t, 16> scn = ((w0 >> (8 * (ib >> 1))) >> (4 * (ib & 1))) & 0xfu;+ sw0 = d * simd<float, 16>(simd<int, 16>(scn * 2u + 1u));#pragma unroll- for (int r = 0; r < 8; ++r) {- simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;- simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);- simd<uint32_t, 16> g = gather<uint32_t, 16>(grid, idx << 2);- simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;- simd<uint32_t, 16> m = ((nib * 0x00204081u) & 0x01010101u);- simd<uint32_t, 16> mask = m * 255u;- frag.template select<16, 1>(16 * r) = (g ^ mask) + m;- }+ for (int r = 0; r < 8; ++r) {+ simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;+ simd<uint32_t, 16> idx = ((qsw >> (8 * (r & 3))) & 0xffu) | (((qhv >> r) & 1u) << 8);+ simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3S, idx << 2);+ simd<uint32_t, 16> nib = ((sg >> (8 * (r >> 1))) >> (4 * (r & 1))) & 0xfu;+ fr0.template select<16, 1>(16 * r) = apply_signs(g, nib);+ }+ } else if constexpr (F == T_IQ3XXS) {+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * ib));+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * ib));+ simd<uint32_t, 16> aux = ld32(We, ro + (P + 66 + 4 * ib));+ sw0 = d * simd<float, 16>(simd<int, 16>((aux >> 28) * 2u + 1u)) * 0.25f;+#pragma unroll+ for (int il = 0; il < 4; ++il) {+ simd<uint32_t, 16> sbyte = gather<uint32_t, 16>(T + O_KSIGNS, ((aux >> (7 * il)) & 127u) << 2);+#pragma unroll+ for (int hh = 0; hh < 2; ++hh) {+ const int r = 2 * il + hh;+ simd<uint32_t, 16> qsw = r < 4 ? q0 : q1;+ simd<uint32_t, 16> idx = (qsw >> (8 * (r & 3))) & 0xffu;+ simd<uint32_t, 16> g = gather<uint32_t, 16>(T + O_IQ3XXS, idx << 2);+ fr0.template select<16, 1>(16 * r) = apply_signs(g, (sbyte >> (4 * hh)) & 0xfu);+ }+ }+ } else if constexpr (F == T_IQ2S) {+ simd<uint32_t, 16> idw = ld32(We, ro + (P + 2 + 4 * ib));+ simd<uint32_t, 16> sgw = ld32(We, ro + (P + 34 + 4 * ib));+ simd<uint32_t, 16> qhb = (((ib < 4) ? w2 : w3) >> (8 * (ib & 3))) & 0xffu;+ simd<uint32_t, 16> scb = (((ib < 4) ? w0 : w1) >> (8 * (ib & 3))) & 0xffu;+ sw0 = d * simd<float, 16>(simd<int, 16>((scb & 0xfu) * 2u + 1u)) * 0.125f;+ sw1 = d * simd<float, 16>(simd<int, 16>((scb >> 4) * 2u + 1u)) * 0.125f;+#pragma unroll+ for (int il = 0; il < 4; ++il) {+ simd<uint32_t, 16> idx = ((idw >> (8 * il)) & 0xffu) | ((qhb << (8 - 2 * il)) & 0x300u);+ simd<uint32_t, 16> gl = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3));+ simd<uint32_t, 16> gh = gather<uint32_t, 16>(T + O_IQ2S, (idx << 3) + 4u);+ simd<uint32_t, 16> sbyte = (sgw >> (8 * il)) & 0xffu;+ simd<uint32_t, 16> vl = apply_signs(gl, sbyte & 0xfu);+ simd<uint32_t, 16> vh = apply_signs(gh, sbyte >> 4);+ if (il < 2) {+ fr0.template select<16, 1>(16 * (2 * il)) = vl;+ fr0.template select<16, 1>(16 * (2 * il + 1)) = vh;+ } else {+ fr1.template select<16, 1>(16 * (2 * il)) = vl;+ fr1.template select<16, 1>(16 * (2 * il + 1)) = vh;+ }+ }+ // fr0 holds values 0..15 (dwords 0..3, the rest zero), fr1 values 16..31 (dwords 4..7)+ fr0.template select<64, 1>(64) = 0u;+ fr1.template select<64, 1>(0) = 0u;+ }}- const int kb = sb * 8 + ib;#pragma unrollfor (int i = 0; i < MT; ++i) {if (i < nvalid) {@@ -207,11 +273,19 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*av = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);}- simd<float, 128> tf = dpas_s8(frag, av);+ simd<float, 128> tf = dpas_s8(fr0, av);#pragma unrollfor (int r = 0; r < 8; ++r) {const float dxr = dxv[r];- acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (swv * dxr);+ acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);+ }+ if constexpr (NS == 2) {+ simd<float, 128> tg = dpas_s8(fr1, av);+#pragma unroll+ for (int r = 0; r < 8; ++r) {+ const float dxr = dxv[r];+ acc.template select<16, 1>(128 * i + 16 * r) += tg.template select<16, 1>(16 * r) * (sw1 * dxr);+ }}}}@@ -224,8 +298,7 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*for (int r = 0; r < 8; ++r) {const int rl = (tl0 + i) * 8 + r;if (rl < cnt && !(a.dbg & 4))- block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt,- acc.template select<16, 1>(128 * i + 16 * r));+ block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));}}}@@ -233,42 +306,56 @@ void launch_gu_iq3s(sycl::queue& q, const GArgs a, int nthreads, const uint32_t*});}-// ------------------------------------------------------------------------------------------------ down, IQ4_NL-template <int MT>-void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t* tab) {+// ------------------------------------------------------------------------------------------------ down+template <int F, int MT>+void launch_dn(sycl::queue& q, const GArgs a, int nrowblocks) {auto props = sycl::ext::oneapi::experimental::properties{sycl::ext::intel::experimental::grf_size<256>};q.submit([&](sycl::handler& h) {- h.parallel_for<TagDnNL<MT>>(sycl::range<2>((size_t) (nthreads / a.ntiles), (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {+ h.parallel_for<TagDn<F, MT>>(sycl::range<2>((size_t) nrowblocks, (size_t) a.ntiles), props, [=](sycl::id<2> id) SYCL_ESIMD_KERNEL {const int rbg = (int) id[0], nt = (int) id[1];- const int t = rbg * a.ntiles + nt;- const Pick pk = (a.dbg & 32) ? Pick{0, a.row0[0], a.row0[1] - a.row0[0], a.tile0[0], a.rb0[0], a.blob[0]} : pick_expert(a, rbg);- const int e = pk.e;+ const Pick pk = pick_expert(a, rbg);const int cnt = pk.cnt;const int ntl = (cnt + 7) >> 3;const int tl0 = (rbg - pk.rb0) * MT;const int nvalid = (ntl - tl0) < MT ? (ntl - tl0) : MT;const int tbase = pk.tile0 + tl0;const uint8_t* We = pk.blob;+ const uint32_t* T = a.tab;simd<uint32_t, 16> lane(0, 1);- simd<uint32_t, 16> rowoff = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;+ simd<uint32_t, 16> ro = (lane + (uint32_t) (16 * nt)) * (uint32_t) a.rowb + a.sec_off;simd<float, 128 * MT> acc = 0.f;const int* Xi = reinterpret_cast<const int*>(a.Xq);for (int kb = 0; kb < ((a.dbg & 8) ? 0 : a.nkb); ++kb) {- const uint32_t P = (uint32_t) kb * BLK_IQ4NL;- simd<float, 16> swv = 1.f;- if (a.dbg & 128) { if (kb == 0) block_store<uint32_t, 16>(reinterpret_cast<uint32_t*>(a.Y) + (size_t) t * 16, rowoff + P); }- else if (!(a.dbg & 16)) swv = half_of(ld16(We, rowoff + P));- simd<uint32_t, 128> frag = 0x01010101u;+ simd<float, 16> sw0 = 1.f;+ simd<uint32_t, 128> fr0 = 0x01010101u;+ if (!(a.dbg & 1)) {+ if constexpr (F == T_IQ4NL) {+ const uint32_t P = (uint32_t) kb * 18;+ sw0 = half_of(ld16(We, ro + P));+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ simd<uint32_t, 16> qd = ld32(We, ro + (P + 2 + 4 * j));+ // four byte-pair lookups (lo value in byte 0, hi value in byte 1), repacked into the two dwords+ simd<uint32_t, 16> t0 = gather<uint32_t, 16>(T + O_IQ4, (qd & 0xffu) << 2);+ simd<uint32_t, 16> t1 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 8) & 0xffu) << 2);+ simd<uint32_t, 16> t2 = gather<uint32_t, 16>(T + O_IQ4, ((qd >> 16) & 0xffu) << 2);+ simd<uint32_t, 16> t3 = gather<uint32_t, 16>(T + O_IQ4, (qd >> 24) << 2);+ fr0.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);+ fr0.template select<16, 1>(16 * (j + 4)) =+ ((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);+ }+ } else { // T_Q2_0: 64 values per block (2 + 16 bytes), two 32-blocks per block, one table lookup per byte+ const uint32_t P = (uint32_t) (kb >> 1) * 18;+ const int hb = kb & 1;+ sw0 = half_of(ld16(We, ro + P));+ simd<uint32_t, 16> q0 = ld32(We, ro + (P + 2 + 8 * hb));+ simd<uint32_t, 16> q1 = ld32(We, ro + (P + 6 + 8 * hb));#pragma unroll- for (int j = 0; j < ((a.dbg & 1) ? 0 : 4); ++j) {- simd<uint32_t, 16> qd = ld32(We, rowoff + (P + 2 + 4 * j));- // four byte-pair lookups (lo value in byte 0, hi value in byte 1), repacked into the two dwords- simd<uint32_t, 16> t0 = gather<uint32_t, 16>(tab, (qd & 0xffu) << 2);- simd<uint32_t, 16> t1 = gather<uint32_t, 16>(tab, ((qd >> 8) & 0xffu) << 2);- simd<uint32_t, 16> t2 = gather<uint32_t, 16>(tab, ((qd >> 16) & 0xffu) << 2);- simd<uint32_t, 16> t3 = gather<uint32_t, 16>(tab, (qd >> 24) << 2);- frag.template select<16, 1>(16 * j) = (t0 & 0xffu) | ((t1 & 0xffu) << 8) | ((t2 & 0xffu) << 16) | (t3 << 24);- frag.template select<16, 1>(16 * (j + 4)) = ((t0 >> 8) & 0xffu) | (t1 & 0xff00u) | ((t2 & 0xff00u) << 8) | ((t3 & 0xff00u) << 16);+ for (int r = 0; r < 8; ++r) {+ simd<uint32_t, 16> qb = ((r < 4 ? q0 : q1) >> (8 * (r & 3))) & 0xffu;+ fr0.template select<16, 1>(16 * r) = gather<uint32_t, 16>(T + O_Q2, qb << 2);+ }+ }}#pragma unrollfor (int i = 0; i < MT; ++i) {@@ -279,11 +366,11 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_tav = block_load<int, 64>(Xi + ((size_t) (tbase + i) * a.nkb + kb) * 64);dxv = block_load<float, 8>(a.dx + ((size_t) (tbase + i) * a.nkb + kb) * 8);}- simd<float, 128> tf = dpas_s8(frag, av);+ simd<float, 128> tf = dpas_s8(fr0, av);#pragma unrollfor (int r = 0; r < 8; ++r) {const float dxr = dxv[r];- acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (swv * dxr);+ acc.template select<16, 1>(128 * i + 16 * r) += tf.template select<16, 1>(16 * r) * (sw0 * dxr);}}}@@ -295,8 +382,7 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_tfor (int r = 0; r < 8; ++r) {const int rl = (tl0 + i) * 8 + r;if (rl < cnt && !(a.dbg & 4))- block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt,- acc.template select<16, 1>(128 * i + 16 * r));+ block_store<float, 16>(a.Y + ((size_t) (pk.row0 + rl)) * a.ldy + 16 * nt, acc.template select<16, 1>(128 * i + 16 * r));}}}@@ -304,34 +390,32 @@ void launch_dn_iq4nl(sycl::queue& q, const GArgs a, int nthreads, const uint32_t});}+// MT from the group's mean number of 8-row tiles per expert.int pick_mt(const Layout& lo) {- int tiles = lo.tile0[lo.n];- const double mean = lo.n ? (double) tiles / lo.n : 0;+ const double mean = lo.n ? (double) lo.tile0[lo.n] / lo.n : 0;if (mean <= 2.0) return 2;if (mean <= 4.0) return 4;- if (mean <= 8.0) return 8;- return 12;+ return 8;}int dbg_flags() {static const int v = [] { const char* s = std::getenv("STRATA_I8_DBG"); return s ? std::atoi(s) : 0; }();return v;}-int mt_override() {- static const int v = [] { const char* s = std::getenv("STRATA_I8_MT"); return s ? std::atoi(s) : 0; }();- return v;+int mt_for(const Layout& lo) {+ static const int mo = [] { const char* s = std::getenv("STRATA_I8_MT"); return s ? std::atoi(s) : 0; }();+ return mo ? mo : pick_mt(lo);}template <class F>-void dispatch_mt(int mt, F&& f) {+bool dispatch_mt(int mt, F&& f) {switch (mt) {- case 2: f(std::integral_constant<int, 2>{}); break;- case 4: f(std::integral_constant<int, 4>{}); break;- case 8: f(std::integral_constant<int, 8>{}); break;- default: f(std::integral_constant<int, 12>{}); break;+ case 2: f(std::integral_constant<int, 2>{}); return true;+ case 4: f(std::integral_constant<int, 4>{}); return true;+ case 8: f(std::integral_constant<int, 8>{}); return true;+ default: return false;}}-// ------------------------------------------------------------------------------------------------ quantize / swiglustruct QArgs {int n;int row0[kMaxGroup + 1];@@ -489,57 +573,54 @@ void swiglu_quantize(const float* GU, const Layout& lo, int n_ff, int8_t* Hq, flbool gate_up(int gu_type, const uint8_t* const* blob, size_t up_off, int n_ff, const Layout& lo, int K, const int8_t* Xq,const float* dx, float* GU, void* stream) {if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;- if (gu_type != 21 || K % 256 != 0 || (2 * n_ff) % 16 != 0 || (up_off & 3) != 0) return false;+ if ((gu_type != T_IQ3S && gu_type != T_IQ3XXS && gu_type != T_IQ2S) || K % 256 != 0 || (2 * n_ff) % 16 != 0 ||+ (up_off & 3) != 0 || lo.n > kMaxGroup)+ return false;sycl::queue& q = *strata::q_of(stream);- Tables& tb = tables(q);GArgs a{};for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];a.sec_off = (uint32_t) up_off;a.Xq = Xq; a.dx = dx; a.Y = GU;+ a.tab = tables(q);a.ntiles = 2 * n_ff / 16;a.nkb = K / 32;- a.rowb = (K / 256) * BLK_IQ3S;+ a.rowb = (K / 256) * block_bytes_of(gu_type);a.ldy = 2 * n_ff;a.dbg = dbg_flags();- const int mt = mt_override() ? mt_override() : pick_mt(lo);- dispatch_mt(mt, [&](auto mtc) {+ const int mt = mt_for(lo);+ const int rb = fill_args_mt(a, lo, mt);+ return dispatch_mt(mt, [&](auto mtc) {constexpr int MT = decltype(mtc)::value;- int nthreads = 0;- fill_args<MT>(a, lo, a.ntiles, nthreads);- launch_gu_iq3s<MT>(q, a, nthreads, tb.grid3s);+ if (gu_type == T_IQ3S) launch_gu<T_IQ3S, MT>(q, a, rb);+ else if (gu_type == T_IQ3XXS) launch_gu<T_IQ3XXS, MT>(q, a, rb);+ else launch_gu<T_IQ2S, MT>(q, a, rb);});- return true;}bool down(int d_type, const uint8_t* const* blob, size_t down_off, int n_embd, const Layout& lo, int n_ff, const int8_t* Hq,const float* dh, float* Dm, void* stream) {if (lo.n == 0 || lo.tile0[lo.n] == 0) return true;- if (d_type != 20 || n_ff % 32 != 0 || n_embd % 16 != 0 || (down_off & 3) != 0) return false;+ if ((d_type != T_IQ4NL && d_type != T_Q2_0) || n_ff % block_vals_of(d_type) != 0 || n_embd % 16 != 0 || (down_off & 3) != 0 ||+ lo.n > kMaxGroup)+ return false;sycl::queue& q = *strata::q_of(stream);- Tables& tb = tables(q);GArgs a{};for (int i = 0; i < lo.n; ++i) a.blob[i] = blob[i];a.sec_off = (uint32_t) down_off;a.Xq = Hq; a.dx = dh; a.Y = Dm;+ a.tab = tables(q);a.ntiles = n_embd / 16;a.nkb = n_ff / 32;- a.rowb = (n_ff / 32) * BLK_IQ4NL;+ a.rowb = (n_ff / block_vals_of(d_type)) * block_bytes_of(d_type);a.ldy = n_embd;a.dbg = dbg_flags();- const int mt = mt_override() ? mt_override() : pick_mt(lo);- dispatch_mt(mt, [&](auto mtc) {+ const int mt = mt_for(lo);+ const int rb = fill_args_mt(a, lo, mt);+ return dispatch_mt(mt, [&](auto mtc) {constexpr int MT = decltype(mtc)::value;- int nthreads = 0;- fill_args<MT>(a, lo, a.ntiles, nthreads);- if (a.dbg & 64) {- std::fprintf(stderr, "i8 down: MT %d n %d threads %d ntiles %d nkb %d rowb %d ldy %d sec_off %u blob0 %p Xq %p dx %p Y %p tab %p\n",- MT, a.n, nthreads, a.ntiles, a.nkb, a.rowb, a.ldy, a.sec_off, (const void*) a.blob[0], (const void*) a.Xq,- (const void*) a.dx, (void*) a.Y, (const void*) tb.iq4);- for (int i = 0; i <= a.n; ++i) std::fprintf(stderr, " e%d row0 %d tile0 %d rb0 %d blob %p\n", i, a.row0[i], a.tile0[i], a.rb0[i], i < a.n ? (const void*) a.blob[i] : nullptr);- }- launch_dn_iq4nl<MT>(q, a, nthreads, tb.iq4);+ if (d_type == T_IQ4NL) launch_dn<T_IQ4NL, MT>(q, a, rb);+ else launch_dn<T_Q2_0, MT>(q, a, rb);});- return true;}} // namespace strata::prefill::i8diff --git a/sycl/src/prefill/iq_tables.inc b/sycl/src/prefill/iq_tables.incindex 0e9c2cd49..98a238ee9 100644--- a/sycl/src/prefill/iq_tables.inc+++ b/sycl/src/prefill/iq_tables.inc@@ -1,5 +1,5 @@// IQ lookup tables copied from llama.cpp ggml/src/ggml-common.h (MIT License, Copyright (c) 2023-2026 The ggml authors):-// kmask_iq2xs, ksigns_iq2xs, iq3xxs_grid, iq3s_grid, kvalues_iq4nl. Same values the dequantize kernels read.+// kmask_iq2xs, ksigns_iq2xs, iq3xxs_grid, iq3s_grid, kvalues_iq4nl, iq2s_grid. Same values the dequantize kernels read.// Included by esimd_moe.dp.cpp after defining GGML_TABLE_BEGIN / GGML_TABLE_END.GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)1, 2, 4, 8, 16, 32, 64, 128@@ -117,3 +117,261 @@ GGML_TABLE_END()GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,GGML_TABLE_END()+GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cppindex 236136fd8..650dd77fc 100644--- a/sycl/src/prefill/prefill.cpp+++ b/sycl/src/prefill/prefill.cpp@@ -42,6 +42,8 @@#include "strata/prefill/kernels.hpp"#include <algorithm>+#include <set>+#include <tuple>#include <chrono>#include <cmath>#include <cstdio>@@ -3963,10 +3965,18 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same// held group of staged experts as aboveconst bool i8on = !xmx2_only && i8_mode() != 0 && !use_mmq && lay.native &&- lay.fmt[(size_t) l].gu_type == 21 && lay.fmt[(size_t) l].d_type == 20 &&+ (lay.fmt[(size_t) l].gu_type == 21 || lay.fmt[(size_t) l].gu_type == 18 || lay.fmt[(size_t) l].gu_type == 22) &&+ (lay.fmt[(size_t) l].d_type == 20 || lay.fmt[(size_t) l].d_type == 42) &&lay.fmt[(size_t) l].n_ff == 640 && lay.fmt[(size_t) l].n_embd == 2560 &&strata::prefill::i8::available(m.cs);const bool xmx2 = xmx2_only || i8on;+ if (i8_mode() != 0 && lay.native) {+ static std::set<std::tuple<int, int, int, int, int>> said_fmt;+ const auto& ff = lay.fmt[(size_t) l];+ if (said_fmt.insert({ff.gu_type, ff.d_type, (int) ff.n_ff, (int) ff.n_embd, (int) i8on}).second)+ std::fprintf(stderr, "strata: layer %lld experts gate/up type %d, down type %d, n_ff %d, n_embd %d -> int8 path %s\n",+ (long long) l, ff.gu_type, ff.d_type, (int) ff.n_ff, (int) ff.n_embd, i8on ? "yes" : "no");+ }if (strata::prefill::xmx::mode() == 2 && !xmx2) {static bool said = false;if (!said) {