Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

SYCL Gyro: decode-once multi-entry trellis kernels, the grouped XMX prompt path takes the trellis types (prompt 40 -> 82 tok/s on the A750), tq_sycl_parity --bench; setup keeps Gyro's n-gram table on the SSD on i915

Niko12212411e7cd3b9d
diff --git a/sycl/setup_intel.py b/sycl/setup_intel.py
index 2f234b74c..cd6818246 100644
--- a/sycl/setup_intel.py
+++ b/sycl/setup_intel.py
@@ -295,7 +295,10 @@ def to_sycl(cfg: dict, exe: Path, ram: float, keep: dict, vram_gb: float = 0.0,
if driver == "i915":
# Arc A-series on i915 (the A750): the experts load into an ordinary RAM arena and the CPU computes the ones
# the card does not hold; the n-gram table is kept in RAM (setup's rotational-disk rule) and no mirror is built.
- if "--ple-io" not in args:
+ # Gyro: its n-gram table is half of the 58-92 GB file and stays on the SSD (read row by row); the RAM holds the
+ # experts' arena (24 GB for Gyro-S) and the CPU's work, so a table in RAM would not fit beside them on a 64 GB PC
+ gyro = "gyro" in (flag(args, "--native") or "").lower()
+ if "--ple-io" not in args and not gyro:
args += ["--ple-io", "ram"]
args += ["--vram-reserve-mib", reserve]
else:
diff --git a/sycl/src/kernels/cuda/iq_kernels.dp.cpp b/sycl/src/kernels/cuda/iq_kernels.dp.cpp
index 313ed1186..9f649a500 100644
--- a/sycl/src/kernels/cuda/iq_kernels.dp.cpp
+++ b/sycl/src/kernels/cuda/iq_kernels.dp.cpp
@@ -1866,6 +1866,17 @@ template <> struct Multi<42> { // q2_0 (down)
// instead of once per entry. Every entry keeps its own lane-strided k order and lanes_sum: the outputs are bitwise
// those of row_dot_lanes (native_grouped_parity --kq checks it). STRATA_KQ_SPLIT=0: the per-entry kernels.
template <int TY> struct KqSplit { static constexpr bool has = false; };
+// APR trellis types: tq_load decodes the lane's 16 codebook values once, tq_apply is the activation side per entry (the same
+// operations as vec_dot_tq_q8_1, so the outputs are the per-entry kernels' bits)
+template <int TY> struct KqSplitTq {
+ static constexpr bool has = true;
+ using W = TqLane;
+ __dpct_inline__ static W load(const void* __restrict__ vbq, int kbx, int iqs) { return tq_load<TY>(vbq, kbx, iqs); }
+ __dpct_inline__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }
+};
+template <> struct KqSplit<144> : KqSplitTq<144> {};
+template <> struct KqSplit<145> : KqSplitTq<145> {};
+template <> struct KqSplit<146> : KqSplitTq<146> {};
template <> struct KqSplit<12> { // Q4_K (vec_dot_q4_K_q8_1)
static constexpr bool has = true;
struct W { int v[2]; uint16_t aux[2]; sycl::half2 dm; };
@@ -5857,8 +5868,9 @@ void launch(sycl::queue& q, int ty, bool gu, const sycl::half* X, float* Y, cons
// the decoder thread that writes values [o, o + 8) of the superblock: the IQ2/IQ3/IQ4_NL/IQ1_M and
// Q2_0 decoders write [8 tid, 8 tid + 8) (coalesced runs); IQ4_XS and the rest keep the
// (sub-run, 32-block) split
+ // (the trellis decoder thread tid writes values [8 tid, 8 tid + 8) of the 256-value call as well)
const bool run8 = ty == 16 || ty == 17 || ty == 18 || ty == 20 || ty == 21 || ty == 22 ||
- ty == 29 || ty == 42;
+ ty == 29 || ty == 42 || ty == 144 || ty == 145 || ty == 146;
const int tid = run8 ? (o >> 3) : (sub * 8 + (o32 >> 5));
dq_dispatch<sycl::half>(ty, base, ibs, pb + r * LDB - o32, tid);
}
diff --git a/sycl/src/kernels/tq_sycl_parity.cpp b/sycl/src/kernels/tq_sycl_parity.cpp
index c6a161bc5..a5f668792 100644
--- a/sycl/src/kernels/tq_sycl_parity.cpp
+++ b/sycl/src/kernels/tq_sycl_parity.cpp
@@ -3,6 +3,7 @@
//
// tq_sycl_parity <pack-dir> <model.gguf> [layer ...]
// tq_sycl_parity --synthetic [--hadamard] GU/DOWN ... (random trellis codes and scales, e.g. 145/146 144/144)
+// tq_sycl_parity --bench GU/DOWN [groups [entries-per-group [iterations]]] (native_expert_grouped on distinct experts in VRAM)
//
// For one expert of each layer (the gate/up/down rows read from the GGUF, the Hadamard signs and block from the pack):
// (a) float reference: fp16-codebook weights, x and h rotated in double, SwiGLU in double.
@@ -27,6 +28,7 @@
#include <dpct/dpct.hpp>
#include <algorithm>
+#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
@@ -231,6 +233,9 @@ int check_layer(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int s
for (int64_t r = 0; r < FF; ++r) {
const float eg = emu_dot(f.gu_type, blob.data() + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);
const float eu = emu_dot(f.gu_type, blob.data() + f.up_off + r * f.gu_row, hxq.data() + (size_t) k * H / 32 * 36, H);
+ if (std::memcmp(&eg, &gate[k * FF + r], 4) != 0 && gu_diff < 4)
+ std::printf(" [debug] gate k %d r %lld: device %g host replay %g (d of block 0: %g)\n", k, (long long) r,
+ gate[k * FF + r], eg, h2f((uint16_t) (blob[r * f.gu_row] | (blob[r * f.gu_row + 1] << 8))));
gu_diff += std::memcmp(&eg, &gate[k * FF + r], 4) != 0;
gu_diff += std::memcmp(&eu, &up[k * FF + r], 4) != 0;
}
@@ -293,15 +298,88 @@ std::vector<uint8_t> synthetic_blob(const cpu::NativeFmt& f, int seed) {
blk[1] = (uint8_t) (h >> 8);
}
};
- fix(blob.data(), f.gu_type, f.gu_row, FF);
- fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);
- fix(blob.data() + f.down_off, f.d_type, f.d_row, H);
+ if (f.gu_type >= 144 && f.d_type >= 144) { // (the bench also times other formats on random bytes)
+ fix(blob.data(), f.gu_type, f.gu_row, FF);
+ fix(blob.data() + f.up_off, f.gu_type, f.gu_row, FF);
+ fix(blob.data() + f.down_off, f.d_type, f.d_row, H);
+ }
return blob;
}
int type_of(const std::string& s) { return std::atoi(s.c_str()); }
+// the grouped expert kernels on `groups` distinct experts held in VRAM (more bytes than the card's L2), `epg` entries each:
+// microseconds per call and per expert, and the weight bytes read per second
+int bench(int gu, int dn, int groups, int epg, int iters, bool had, int phase) {
+ strata::kernels::native_expert_set_mode(7, phase); // 0: all, 1: gate/up + SwiGLU + quantize, 2: down only
+ cpu::NativeFmt f;
+ std::string err;
+ if (!cpu::native_fmt(gu, dn, H, FF, f, err) || !strata::kernels::native_expert_supported(gu, dn, H, FF)) {
+ std::printf("%d/%d: not a pair the GPU expert kernels take\n", gu, dn);
+ return 1;
+ }
+ static std::vector<float> sx((size_t) H), sh((size_t) FF);
+ std::mt19937 srng(5);
+ for (auto& v : sx) v = (srng() & 1) ? 1.f : -1.f;
+ for (auto& v : sh) v = (srng() & 1) ? 1.f : -1.f;
+ if (had) { f.had_block = 128; f.had_gu = f.had_d = true; f.gu_signs = sx.data(); f.d_signs = sh.data(); }
+ sycl::queue* s = &dpct::get_in_order_queue();
+ const int ent = groups * epg, ntok = 4;
+ uint8_t* dblob = dmalloc<uint8_t>((size_t) groups * f.bytes);
+ for (int g = 0; g < groups; ++g) {
+ const std::vector<uint8_t> b = synthetic_blob(f, g);
+ to_dev(dblob + (size_t) g * f.bytes, b.data(), b.size());
+ }
+ std::vector<float> x((size_t) ntok * H);
+ std::mt19937 rng(7);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ for (auto& v : x) v = nd(rng);
+ float* dx = dmalloc<float>(x.size());
+ to_dev(dx, x.data(), x.size() * 4);
+ uint8_t* dxq = dmalloc<uint8_t>((size_t) ntok * H / 32 * 36);
+ uint8_t* dscr = dmalloc<uint8_t>(strata::kernels::native_expert_scratch_bytes(ent, FF));
+ float* dout = dmalloc<float>((size_t) ent * H);
+ std::vector<unsigned long long> ptr(groups);
+ std::vector<int32_t> st(groups + 1), dst(ent), tok(ent);
+ for (int g = 0; g < groups; ++g) { ptr[g] = (unsigned long long) (dblob + (size_t) g * f.bytes); st[g] = g * epg; }
+ st[groups] = ent;
+ for (int e = 0; e < ent; ++e) { dst[e] = e; tok[e] = e % ntok; }
+ unsigned long long* dptr = dmalloc<unsigned long long>(groups);
+ int32_t *dstart = dmalloc<int32_t>(groups + 1), *dn_ = dmalloc<int32_t>(1), *ddst = dmalloc<int32_t>(ent), *dtok = dmalloc<int32_t>(ent);
+ to_dev(dptr, ptr.data(), groups * 8);
+ to_dev(dstart, st.data(), (groups + 1) * 4);
+ to_dev(dn_, &groups, 4);
+ to_dev(ddst, dst.data(), ent * 4);
+ to_dev(dtok, tok.data(), ent * 4);
+ auto L = strata::kernels::native_expert_layout(gu, dn, H, FF);
+ if (had) { L.had_block = f.had_block; L.d_signs = strata::kernels::hadamard_device_signs(f.d_signs, (int) FF); }
+ strata::kernels::quantize_q8_1_rows(dx, ntok, H, dxq, s);
+ for (int i = 0; i < 3; ++i) strata::kernels::native_expert_grouped(L, dptr, dstart, dn_, ddst, dtok, groups, ent, dxq, dscr, dout, s);
+ s->wait();
+ std::vector<double> us;
+ for (int i = 0; i < iters; ++i) {
+ const auto t0 = std::chrono::steady_clock::now();
+ strata::kernels::native_expert_grouped(L, dptr, dstart, dn_, ddst, dtok, groups, ent, dxq, dscr, dout, s);
+ s->wait();
+ us.push_back(std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count());
+ }
+ std::sort(us.begin(), us.end());
+ const double med = us[us.size() / 2];
+ std::printf("bench %d/%d%s: %d groups x %d entries, %zu B per expert: median %.0f us per call = %.1f us per expert, %.1f GB/s of weights (min %.0f us)\n",
+ gu, dn, had ? " hadamard" : "", groups, epg, f.bytes, med, med / groups, (double) groups * f.bytes / (med * 1e-6) / 1e9, us.front());
+ sycl::free(dblob, *s); sycl::free(dx, *s); sycl::free(dxq, *s); sycl::free(dscr, *s); sycl::free(dout, *s);
+ sycl::free(dptr, *s); sycl::free(dstart, *s); sycl::free(dn_, *s); sycl::free(ddst, *s); sycl::free(dtok, *s);
+ return 0;
+}
+
int main(int argc, char** argv) {
setvbuf(stdout, nullptr, _IONBF, 0);
+ if (argc >= 3 && std::string(argv[1]) == "--bench") {
+ const std::string arg = argv[2];
+ const size_t slash = arg.find('/');
+ const bool had = std::getenv("TQ_BENCH_HADAMARD") != nullptr;
+ return bench(type_of(arg.substr(0, slash)), type_of(arg.substr(slash + 1)), argc > 3 ? std::atoi(argv[3]) : 64,
+ argc > 4 ? std::atoi(argv[4]) : 1, argc > 5 ? std::atoi(argv[5]) : 50, had, argc > 6 ? std::atoi(argv[6]) : 0);
+ }
if (argc >= 2 && std::string(argv[1]) == "--synthetic") {
int failures = 0, i = 2;
const bool had = argc > 2 && std::string(argv[2]) == "--hadamard";
diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cpp
index 28222d91a..be06adfc5 100644
--- a/sycl/src/prefill/prefill.cpp
+++ b/sycl/src/prefill/prefill.cpp
@@ -4221,9 +4221,10 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
auto xmx2_fmt = [&]() {
if (!lay.native) return false;
const int gt = lay.fmt[(size_t) l].gu_type, dt = lay.fmt[(size_t) l].d_type;
+ if ((gt == 144 || gt == 145) && (dt == 144 || dt == 146)) return true; // Gyro's trellis experts (rotated above)
return (gt == 18 || gt == 21 || gt == 22 || gt == 23) && (dt == 20 || dt == 42);
};
- const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt() && !had_d;
+ const bool xmx2_only = strata::prefill::xmx::mode() == 2 && !use_mmq && xmx2_fmt();
// 151-b70: the fused int8 DPAS products (IQ3_S gate/up, IQ4_NL down, the 640 x 2560 expert), the same
// held group of staged experts as above
const int i8m = i8_mode(m.cs);
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index c863e3b41..b7600dc53 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -1110,7 +1110,7 @@ class Strata:
if model == "IQ2_XS":
why = (f"{total}, the 40 GB tier: Swift 1.5's IQ2_XS (thinks for a shorter time)"
+ ("; Gyro-M (35 GB) is the other choice there, close to IQ3_XXS in quality but 17-21% slower at "
- "writing answers than Q2_0" if "Gyro-M" in models and best.get("vendor") in ("nvidia", "amd") else ""))
+ "writing answers than Q2_0" if "Gyro-M" in models and best.get("vendor") in ("nvidia", "amd", "intel") else ""))
elif model == "IQ3_XXS":
why = f"{total}, the 48 GB tier: IQ3_XXS (better quality, slower)"
else:
diff --git a/tools/test_setup_gyro.py b/tools/test_setup_gyro.py
index 08694fdec..7b041b3f2 100644
--- a/tools/test_setup_gyro.py
+++ b/tools/test_setup_gyro.py
@@ -64,10 +64,10 @@ class GyroCatalog(unittest.TestCase):
self.assertEqual((got["family"], got["model"]), ("gyro", "Gyro-S"))
got = s.recommend(hardware(24.0, 16.0)) # 40 GB in total: the owner's 40 GB tier, Swift 1.5's IQ2_XS
self.assertEqual((got["family"], got["model"]), ("swift", "IQ2_XS"))
- self.assertIn("Gyro-M", got["why"]) # named as the other choice (NVIDIA and AMD)
+ self.assertIn("Gyro-M", got["why"]) # named as the other choice (NVIDIA, AMD and Intel Arc)
intel = hardware(24.0, 16.0)
intel["gpus"] = [{**g, "vendor": "intel"} for g in intel["gpus"]]
- self.assertNotIn("Gyro", s.recommend(intel)["why"]) # never Intel Arc
+ self.assertIn("Gyro-M", s.recommend(intel)["why"]) # the SYCL engine has the trellis kernels (0.2.2)
self.assertIsNone(s.recommend(hardware(12.0, 8.0))["family"])
self.assertIsNone(s.recommend(hardware(16.0, 11.0))["family"]) # 27 GB: below Gyro-S's 27.6 + room
diff --git a/tools/test_setup_sycl.py b/tools/test_setup_sycl.py
index ff6333f7a..10ba23aa9 100644
--- a/tools/test_setup_sycl.py
+++ b/tools/test_setup_sycl.py
@@ -206,6 +206,15 @@ class ToSycl(unittest.TestCase):
self.assertEqual(c["env"]["STRATA_VERIFY_NO_HOST"], "0")
self.assertEqual(c["env"]["STRATA_SYCL_BIN"], "build-sycl/strata")
+ def test_a_series_keeps_gyros_ngram_table_on_the_ssd(self):
+ # Gyro's table is half of the 58-92 GB file: --ple-io ram would not fit beside the experts' arena on a 64 GB PC
+ m = self.mod()
+ cfg = self.cfg(m)
+ cfg["args"][cfg["args"].index("--native") + 1] = str(m.MOUNT / "d" / "Qwen3.8-Flash-Next-Gyro-S-TQ1_0.gguf")
+ c = m.to_sycl(cfg, m.ROOT / "build-sycl" / "strata", 64.0, {}, 8.0, "i915")
+ self.assertNotIn("--ple-io", c["args"])
+ self.assertEqual(c["env"]["STRATA_VERIFY_NO_HOST"], "0")
+
def test_split_short_rule(self):
# #1980: 2x B60 24 GB - IQ3_XXS (42.9 GB) was refused under NO_HOST=1, IQ2_XS (35.5 GB) loads fully
m = self.mod()