Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Gyro: native_fmt knows the trellis types' geometry itself, so a ggml built from upstream (no such types; its type table is indexed by the id) is never asked about them
Niko1221e1d11fe727d7
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppindex 48ac7d901..132cb99e1 100644--- a/include/strata/kernels/cpu/tq_cpu.hpp+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -35,13 +35,17 @@ enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };bool tq_supported(int ggml_type) noexcept;/// Bits per step: 8, 6, 7 (0 for any other type).int tq_kbits(int ggml_type) noexcept;+/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30).+constexpr int kTqBlock = 128;+inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).TqIsa tq_isa();/// Whether this CPU can run `isa` at all (scalar: always).bool tq_isa_ok(TqIsa isa);-/// Whether the engine's native expert rows take these kernels for the trellis types: on every CPU unless-/// STRATA_NO_TQ_KERNELS=1 (AVX-512 or AVX2 where the CPU has them, else the scalar kernel: a ggml from upstream has-/// no trellis vec_dot to fall back to, and the AVX code is only reached behind tq_isa_ok).+/// Whether STRATA_NO_TQ_KERNELS=1 is absent. The engine's native expert rows take these kernels for the trellis types+/// on every CPU (AVX-512 or AVX2 where the CPU has them, else the scalar kernel: a ggml from upstream has no trellis+/// type to fall back to, and the AVX code is only reached behind tq_isa_ok). STRATA_NO_TQ_KERNELS=1 asks for+/// ggml-cpu's vec_dot instead, honoured only by a ggml that has the types (the agentionai fork; tests, A/B).bool tq_kernels_enabled();const char* tq_isa_name(TqIsa isa);diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cppindex e3072b934..484340188 100644--- a/src/kernels/cpu/native_expert.cpp+++ b/src/kernels/cpu/native_expert.cpp@@ -27,6 +27,14 @@ namespace {const ggml_type_traits_cpu* traits(int type) { return ggml_get_type_traits_cpu((ggml_type) type); }+// Do the trellis rows take Strata's own kernels (tq_cpu.hpp)? Always, unless STRATA_NO_TQ_KERNELS=1 asks for ggml-cpu's+// vec_dot AND this ggml has the type (the agentionai fork; an upstream ggml does not, and must never be asked about it).+bool tq_rows_own(int type) {+ if (!tq_supported(type)) return false;+ if (tq_kernels_enabled()) return true;+ return type >= (int) GGML_TYPE_COUNT || traits(type) == nullptr || traits(type)->vec_dot == nullptr;+}+void init_once() {static std::once_flag once;std::call_once(once, [] { ggml_cpu_init(); });@@ -52,31 +60,39 @@ bool kq256_supported(int type) noexcept { return type == 12 || type == 7 || typebool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt& f, std::string& err) {init_once();- const ggml_type_traits_cpu* tg = traits(gu_type);- const ggml_type_traits_cpu* td = traits(d_type);- if (tg == nullptr || tg->vec_dot == nullptr || td == nullptr || td->vec_dot == nullptr) {- err = "native experts: ggml-cpu has no dot product for type " + std::to_string(tg && tg->vec_dot ? d_type : gu_type);+ // The trellis types (Gyro, ggml types 144-146) are not in a ggml built from upstream (and an upstream ggml indexes its+ // type table by the id, so asking it about 144 reads past the table): their geometry and activation format (Q8_0)+ // are known here, and their rows run on tq_cpu.hpp.+ const bool tq_g = tq_supported(gu_type), tq_d = tq_supported(d_type);+ const ggml_type_traits_cpu* tg = tq_g ? nullptr : traits(gu_type);+ const ggml_type_traits_cpu* td = tq_d ? nullptr : traits(d_type);+ if ((!tq_g && (tg == nullptr || tg->vec_dot == nullptr)) || (!tq_d && (td == nullptr || td->vec_dot == nullptr))) {+ err = "native experts: ggml-cpu has no dot product for type " ++ std::to_string(!tq_g && !(tg && tg->vec_dot) ? gu_type : d_type);return false;}- const ggml_type_traits_cpu* ag = traits(tg->vec_dot_type);- const ggml_type_traits_cpu* ad = traits(td->vec_dot_type);+ const ggml_type vdt_g = tq_g ? GGML_TYPE_Q8_0 : tg->vec_dot_type;+ const ggml_type vdt_d = tq_d ? GGML_TYPE_Q8_0 : td->vec_dot_type;+ const ggml_type_traits_cpu* ag = traits(vdt_g);+ const ggml_type_traits_cpu* ad = traits(vdt_d);if (ag == nullptr || ag->from_float == nullptr || ad == nullptr || ad->from_float == nullptr) {err = "native experts: ggml-cpu cannot quantize an activation for this layer";return false;}- if (n_embd % ggml_blck_size((ggml_type) gu_type) || n_ff % ggml_blck_size((ggml_type) d_type) ||- n_embd % ggml_blck_size(tg->vec_dot_type) || n_ff % ggml_blck_size(td->vec_dot_type)) {+ if ((tq_g ? n_embd % kTqBlock : n_embd % ggml_blck_size((ggml_type) gu_type)) ||+ (tq_d ? n_ff % kTqBlock : n_ff % ggml_blck_size((ggml_type) d_type)) ||+ n_embd % ggml_blck_size(vdt_g) || n_ff % ggml_blck_size(vdt_d)) {err = "native experts: expert geometry is not whole blocks";return false;}f.gu_type = gu_type;f.d_type = d_type;- f.gu_act = (int) tg->vec_dot_type;- f.d_act = (int) td->vec_dot_type;+ f.gu_act = (int) vdt_g;+ f.d_act = (int) vdt_d;f.n_embd = n_embd;f.n_ff = n_ff;- f.gu_row = ggml_row_size((ggml_type) gu_type, n_embd);- f.d_row = ggml_row_size((ggml_type) d_type, n_ff);+ f.gu_row = tq_g ? (size_t) (n_embd / kTqBlock) * tq_block_bytes(gu_type) : ggml_row_size((ggml_type) gu_type, n_embd);+ f.d_row = tq_d ? (size_t) (n_ff / kTqBlock) * tq_block_bytes(d_type) : ggml_row_size((ggml_type) d_type, n_ff);f.up_off = f.gu_row * (size_t) n_ff;f.down_off = 2 * f.up_off;f.bytes = f.down_off + f.d_row * (size_t) n_embd;@@ -86,8 +102,8 @@ bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmtf.tail_off = f.bytes;f.bytes += kNvfp4Tail;}- f.act_bytes = ggml_row_size(tg->vec_dot_type, n_embd);- f.h_bytes = ggml_row_size(td->vec_dot_type, n_ff);+ f.act_bytes = ggml_row_size(vdt_g, n_embd);+ f.h_bytes = ggml_row_size(vdt_d, n_ff);if (f.act_bytes > kNativeActBytes || f.h_bytes > kNativeHBytes) {err = "native experts: activation larger than the pool's buffers";return false;@@ -190,7 +206,7 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*}// The trellis types (TQ2_T / TQK6 / TQK7): Strata's own kernels (tq_cpu.hpp), for every group size - they decode// a block once for all the tokens and give each token the same bits whatever the group (no #152 rule needed).- if (tq_supported(f.gu_type) && tq_kernels_enabled()) {+ if (tq_rows_own(f.gu_type)) {tq_gu_rows(tq_isa(), f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);return;}@@ -261,7 +277,7 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const// Both multi-token kernels below are /arch:AVX2 translation units (kq_avx2.cpp and iq_avx2.cpp),// so a CPU without AVX2 has to reach ggml-cpu's vec_dot instead - same reasoning as the gate/up// rows above, where `avx512` tested cpu_avx512_ok() and `avx2` did not.- if (tq_supported(f.d_type) && tq_kernels_enabled()) { // the trellis types, as in native_gu_rows+ if (tq_rows_own(f.d_type)) { // the trellis types, as in native_gu_rowstq_rows(tq_isa(), f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);return;}