Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
SYCL Gyro: decode-once multi-entry trellis kernels, the grouped XMX prompt path takes the trellis types (prompt 40 -> 82 tok/s on the A750), tq_sycl_parity --bench; setup keeps Gyro's n-gram table on the SSD on i915
Niko12212411e7cd3b9d
diff --git a/sycl/setup_intel.py b/sycl/setup_intel.pyindex 2f234b74c..cd6818246 100644--- a/sycl/setup_intel.py+++ b/sycl/setup_intel.py@@ -295,7 +295,10 @@ def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0,if driver == "i915":# Arc A-series on i915 (the A750): the experts load into an ordinary RAM arena and the CPU computes the ones# the card does not hold; the n-gram table is kept in RAM (setup's rotational-disk rule) and no mirror is built.- if "--ple-io" not in args:+ # Gyro: its n-gram table is half of the 58-92 GB file and stays on the SSD (read row by row); the RAM holds the+ # experts' arena (24 GB for Gyro-S) and the CPU's work, so a table in RAM would not fit beside them on a 64 GB PC+ gyro = "gyro" in (flag(args, "--native") or "").lower()+ if "--ple-io" not in args and not gyro:args += ["--ple-io", "ram"]args += ["--vram-reserve-mib", reserve]else:diff --git a/sycl/src/kernels/cuda/iq_kernels.dp.cpp b/sycl/src/kernels/cuda/iq_kernels.dp.cppindex 313ed1186..9f649a500 100644--- a/sycl/src/kernels/cuda/iq_kernels.dp.cpp+++ b/sycl/src/kernels/cuda/iq_kernels.dp.cpp@@ -1866,6 +1866,17 @@ template <> struct Multi<42> { // q2_0 (down)// instead of once per entry. Every entry keeps its own lane-strided k order and lanes_sum: the outputs are bitwise// those of row_dot_lanes (native_grouped_parity --kq checks it). STRATA_KQ_SPLIT=0: the per-entry kernels.template <int TY> struct KqSplit { static constexpr bool has = false; };+// APR trellis types: tq_load decodes the lane's 16 codebook values once, tq_apply is the activation side per entry (the same+// operations as vec_dot_tq_q8_1, so the outputs are the per-entry kernels' bits)+template <int TY> struct KqSplitTq {+ static constexpr bool has = true;+ using W = TqLane;+ __dpct_inline__ static W load(const void* __restrict__ vbq, int kbx, int iqs) { return tq_load<TY>(vbq, kbx, iqs); }+ __dpct_inline__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }+};+template <> struct KqSplit<144> : KqSplitTq<144> {};+template <> struct KqSplit<145> : KqSplitTq<145> {};+template <> struct KqSplit<146> : KqSplitTq<146> {};template <> struct KqSplit<12> { // Q4_K (vec_dot_q4_K_q8_1)static constexpr bool has = true;struct W { int v[2]; uint16_t aux[2]; sycl::half2 dm; };@@ -5857,8 +5868,9 @@ void launch(sycl::queue& q, int ty, bool gu, const sycl::half* X, float* Y, cons// the decoder thread that writes values [o, o + 8) of the superblock: the IQ2/IQ3/IQ4_NL/IQ1_M and// Q2_0 decoders write [8 tid, 8 tid + 8) (coalesced runs); IQ4_XS and the rest keep the// (sub-run, 32-block) split+ // (the trellis decoder thread tid writes values [8 tid, 8 tid + 8) of the 256-value call as well)const bool run8 = ty == 16 || ty == 17 || ty == 18 || ty == 20 || ty == 21 || ty == 22 ||- ty == 29 || ty == 42;+ ty == 29 || ty == 42 || ty == 144 || ty == 145 || ty == 146;const int tid = run8 ? (o >> 3) : (sub * 8 + (o32 >> 5));dq_dispatch<sycl::half>(ty, base, ibs, pb + r * LDB - o32, tid);}diff --git a/sycl/src/kernels/tq_sycl_parity.cpp b/sycl/src/kernels/tq_sycl_parity.cppindex c6a161bc5..a5f668792 100644--- a/sycl/src/kernels/tq_sycl_parity.cpp+++ b/sycl/src/kernels/tq_sycl_parity.cpp@@ -3,6 +3,7 @@//// tq_sycl_parity <pack-dir> <model.gguf> [layer ...]// tq_sycl_parity --synthetic [--hadamard] GU/DOWN ... (random trellis codes and scales, e.g. 145/146 144/144)+// tq_sycl_parity --bench GU/DOWN [groups [entries-per-group [iterations]]] (native_expert_grouped on distinct experts in VRAM)//// For one expert of each layer (the gate/up/down rows read from the GGUF, the Hadamard signs and block from the pack):// (a) float reference: fp16-codebook weights, x and h rotated in double, SwiGLU in double.@@ -27,6 +28,7 @@#include <dpct/dpct.hpp>#include <algorithm>+#include <chrono>#include <cmath>#include <cstdio>#include <cstdlib>@@ -231,6 +233,9 @@ int check_layer(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int sfor (int64_t r = 0; r < FF; ++r) {const float eg = emu_dot(f.gu_type, blob.data() + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);const float eu = emu_dot(f.gu_type, blob.data() + f.up_off + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);+ if (std::memcmp(&eg, &gate[k * FF + r], 4) != 0 && gu_diff < 4)+ std::printf(" [debug] gate k %d r %lld: device %g host replay %g (d of block 0: %g)\n", k, (long long) r,+ gate[k * FF + r], eg, h2f((uint16_t) (blob[r * f.gu_row] | (blob[r * f.gu_row + 1] << 8))));gu_diff += std::memcmp(&eg, &gate[k * FF + r], 4) != 0;gu_diff += std::memcmp(&eu, &up[k * FF + r], 4) != 0;}@@ -293,15 +298,88 @@ std::vector<uint8_t> synthetic_blob(const cpu::NativeFmt& f, int seed) {blk[1] = (uint8_t) (h >> 8);}};- fix(blob.data(), f.gu_type, f.gu_row, FF);- fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);- fix(blob.data() + f.down_off, f.d_type, f.d_row, H);+ if (f.gu_type >= 144 && f.d_type >= 144) { // (the bench also times other formats on random bytes)+ fix(blob.data(), f.gu_type, f.gu_row, FF);+ fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);+ fix(blob.data() + f.down_off, f.d_type, f.d_row, H);+ }return blob;}int type_of(const std::string& s) { return std::atoi(s.c_str()); }+// the grouped expert kernels on `groups` distinct experts held in VRAM (more bytes than the card's L2), `epg` entries each:+// microseconds per call and per expert, and the weight bytes read per second+int bench(int gu, int dn, int groups, int epg, int iters, bool had, int phase) {+ strata::kernels::native_expert_set_mode(7, phase); // 0: all, 1: gate/up + SwiGLU + quantize, 2: down only+ cpu::NativeFmt f;+ std::string err;+ if (!cpu::native_fmt(gu, dn, H, FF, f, err) || !strata::kernels::native_expert_supported(gu, dn, H, FF)) {+ std::printf("%d/%d: not a pair the GPU expert kernels take\n", gu, dn);+ return 1;+ }+ static std::vector<float> sx((size_t) H), sh((size_t) FF);+ std::mt19937 srng(5);+ for (auto& v : sx) v = (srng() & 1) ? 1.f : -1.f;+ for (auto& v : sh) v = (srng() & 1) ? 1.f : -1.f;+ if (had) { f.had_block = 128; f.had_gu = f.had_d = true; f.gu_signs = sx.data(); f.d_signs = sh.data(); }+ sycl::queue* s = &dpct::get_in_order_queue();+ const int ent = groups * epg, ntok = 4;+ uint8_t* dblob = dmalloc<uint8_t>((size_t) groups * f.bytes);+ for (int g = 0; g < groups; ++g) {+ const std::vector<uint8_t> b = synthetic_blob(f, g);+ to_dev(dblob + (size_t) g * f.bytes, b.data(), b.size());+ }+ std::vector<float> x((size_t) ntok * H);+ std::mt19937 rng(7);+ std::normal_distribution<float> nd(0.f, 1.f);+ for (auto& v : x) v = nd(rng);+ float* dx = dmalloc<float>(x.size());+ to_dev(dx, x.data(), x.size() * 4);+ uint8_t* dxq = dmalloc<uint8_t>((size_t) ntok * H / 32 * 36);+ uint8_t* dscr = dmalloc<uint8_t>(strata::kernels::native_expert_scratch_bytes(ent, FF));+ float* dout = dmalloc<float>((size_t) ent * H);+ std::vector<unsigned long long> ptr(groups);+ std::vector<int32_t> st(groups + 1), dst(ent), tok(ent);+ for (int g = 0; g < groups; ++g) { ptr[g] = (unsigned long long) (dblob + (size_t) g * f.bytes); st[g] = g * epg; }+ st[groups] = ent;+ for (int e = 0; e < ent; ++e) { dst[e] = e; tok[e] = e % ntok; }+ unsigned long long* dptr = dmalloc<unsigned long long>(groups);+ int32_t *dstart = dmalloc<int32_t>(groups + 1), *dn_ = dmalloc<int32_t>(1), *ddst = dmalloc<int32_t>(ent), *dtok = dmalloc<int32_t>(ent);+ to_dev(dptr, ptr.data(), groups * 8);+ to_dev(dstart, st.data(), (groups + 1) * 4);+ to_dev(dn_, &groups, 4);+ to_dev(ddst, dst.data(), ent * 4);+ to_dev(dtok, tok.data(), ent * 4);+ auto L = strata::kernels::native_expert_layout(gu, dn, H, FF);+ if (had) { L.had_block = f.had_block; L.d_signs = strata::kernels::hadamard_device_signs(f.d_signs, (int) FF); }+ strata::kernels::quantize_q8_1_rows(dx, ntok, H, dxq, s);+ for (int i = 0; i < 3; ++i) strata::kernels::native_expert_grouped(L, dptr, dstart, dn_, ddst, dtok, groups, ent, dxq, dscr, dout, s);+ s->wait();+ std::vector<double> us;+ for (int i = 0; i < iters; ++i) {+ const auto t0 = std::chrono::steady_clock::now();+ strata::kernels::native_expert_grouped(L, dptr, dstart, dn_, ddst, dtok, groups, ent, dxq, dscr, dout, s);+ s->wait();+ us.push_back(std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());+ }+ std::sort(us.begin(), us.end());+ const double med = us[us.size() / 2];+ std::printf("bench %d/%d%s: %d groups x %d entries, %zu B per expert: median %.0f us per call = %.1f us per expert, %.1f GB/s of weights (min %.0f us)\n",+ gu, dn, had ? " hadamard" : "", groups, epg, f.bytes, med, med / groups, (double) groups * f.bytes / (med * 1e-6) / 1e9, us.front());+ sycl::free(dblob, *s); sycl::free(dx, *s); sycl::free(dxq, *s); sycl::free(dscr, *s); sycl::free(dout, *s);+ sycl::free(dptr, *s); sycl::free(dstart, *s); sycl::free(dn_, *s); sycl::free(ddst, *s); sycl::free(dtok, *s);+ return 0;+}+int main(int argc, char** argv) {setvbuf(stdout, nullptr, _IONBF, 0);+ if (argc >= 3 && std::string(argv[1]) == "--bench") {+ const std::string arg = argv[2];+ const size_t slash = arg.find('/');+ const bool had = std::getenv("TQ_BENCH_HADAMARD") != nullptr;+ return bench(type_of(arg.substr(0, slash)), type_of(arg.substr(slash + 1)), argc > 3 ? std::atoi(argv[3]) : 64,+ argc > 4 ? std::atoi(argv[4]) : 1, argc > 5 ? std::atoi(argv[5]) : 50, had, argc > 6 ? std::atoi(argv[6]) : 0);+ }if (argc >= 2 && std::string(argv[1]) == "--synthetic") {int failures = 0, i = 2;const bool had = argc > 2 && std::string(argv[2]) == "--hadamard";diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cppindex 28222d91a..be06adfc5 100644--- a/sycl/src/prefill/prefill.cpp+++ b/sycl/src/prefill/prefill.cpp@@ -4221,9 +4221,10 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,auto xmx2_fmt = [&]() {if (!lay.native) return false;const int gt = lay.fmt[(size_t) l].gu_type, dt = lay.fmt[(size_t) l].d_type;+ if ((gt == 144 || gt == 145) && (dt == 144 || dt == 146)) return true; // Gyro's trellis experts (rotated above)return (gt == 18 || gt == 21 || gt == 22 || gt == 23) && (dt == 20 || dt == 42);};- const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt() && !had_d;+ const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt();// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same// held group of staged experts as aboveconst int i8m = i8_mode(m.cs);diff --git a/tools/strata_mcp.py b/tools/strata_mcp.pyindex c863e3b41..b7600dc53 100644--- a/tools/strata_mcp.py+++ b/tools/strata_mcp.py@@ -1110,7 +1110,7 @@ class Strata:if model == "IQ2_XS":why = (f"{total}, the 40 GB tier: Swift 1.5's IQ2_XS (thinks for a shorter time)"+ ("; Gyro-M (35 GB) is the other choice there, close to IQ3_XXS in quality but 17-21% slower at "- "writing answers than Q2_0" if "Gyro-M" in models and best.get("vendor") in ("nvidia", "amd") else ""))+ "writing answers than Q2_0" if "Gyro-M" in models and best.get("vendor") in ("nvidia", "amd", "intel") else ""))elif model == "IQ3_XXS":why = f"{total}, the 48 GB tier: IQ3_XXS (better quality, slower)"else:diff --git a/tools/test_setup_gyro.py b/tools/test_setup_gyro.pyindex 08694fdec..7b041b3f2 100644--- a/tools/test_setup_gyro.py+++ b/tools/test_setup_gyro.py@@ -64,10 +64,10 @@ class GyroCatalog(unittest.TestCase):self.assertEqual((got["family"], got["model"]), ("gyro", "Gyro-S"))got = s.recommend(hardware(24.0, 16.0)) # 40 GB in total: the owner's 40 GB tier, Swift 1.5's IQ2_XSself.assertEqual((got["family"], got["model"]), ("swift", "IQ2_XS"))- self.assertIn("Gyro-M", got["why"]) # named as the other choice (NVIDIA and AMD)+ self.assertIn("Gyro-M", got["why"]) # named as the other choice (NVIDIA, AMD and Intel Arc)intel = hardware(24.0, 16.0)intel["gpus"] = [{**g, "vendor": "intel"} for g in intel["gpus"]]- self.assertNotIn("Gyro", s.recommend(intel)["why"]) # never Intel Arc+ self.assertIn("Gyro-M", s.recommend(intel)["why"]) # the SYCL engine has the trellis kernels (0.2.2)self.assertIsNone(s.recommend(hardware(12.0, 8.0))["family"])self.assertIsNone(s.recommend(hardware(16.0, 11.0))["family"]) # 27 GB: below Gyro-S's 27.6 + roomdiff --git a/tools/test_setup_sycl.py b/tools/test_setup_sycl.pyindex ff6333f7a..10ba23aa9 100644--- a/tools/test_setup_sycl.py+++ b/tools/test_setup_sycl.py@@ -206,6 +206,15 @@ class ToSycl(unittest.TestCase):self.assertEqual(c["env"]["STRATA_VERIFY_NO_HOST"], "0")self.assertEqual(c["env"]["STRATA_SYCL_BIN"], "build-sycl/strata")+ def test_a_series_keeps_gyros_ngram_table_on_the_ssd(self):+ # Gyro's table is half of the 58-92 GB file: --ple-io ram would not fit beside the experts' arena on a 64 GB PC+ m = self.mod()+ cfg = self.cfg(m)+ cfg["args"][cfg["args"].index("--native") + 1] = str(m.MOUNT / "d" / "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf")+ c = m.to_sycl(cfg, m.ROOT / "build-sycl" / "strata", 64.0, {}, 8.0, "i915")+ self.assertNotIn("--ple-io", c["args"])+ self.assertEqual(c["env"]["STRATA_VERIFY_NO_HOST"], "0")+def test_split_short_rule(self):# #1980: 2x B60 24 GB - IQ3_XXS (42.9 GB) was refused under NO_HOST=1, IQ2_XS (35.5 GB) loads fullym = self.mod()