Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Gyro: native_fmt knows the trellis types' geometry itself, so a ggml built from upstream (no such types; its type table is indexed by the id) is never asked about them

Niko1221e1d11fe727d7
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hpp
index 48ac7d901..132cb99e1 100644
--- a/include/strata/kernels/cpu/tq_cpu.hpp
+++ b/include/strata/kernels/cpu/tq_cpu.hpp
@@ -35,13 +35,17 @@ enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };
bool tq_supported(int ggml_type) noexcept;
/// Bits per step: 8, 6, 7 (0 for any other type).
int tq_kbits(int ggml_type) noexcept;
+/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30).
+constexpr int kTqBlock = 128;
+inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }
/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).
TqIsa tq_isa();
/// Whether this CPU can run `isa` at all (scalar: always).
bool tq_isa_ok(TqIsa isa);
-/// Whether the engine's native expert rows take these kernels for the trellis types: on every CPU unless
-/// STRATA_NO_TQ_KERNELS=1 (AVX-512 or AVX2 where the CPU has them, else the scalar kernel: a ggml from upstream has
-/// no trellis vec_dot to fall back to, and the AVX code is only reached behind tq_isa_ok).
+/// Whether STRATA_NO_TQ_KERNELS=1 is absent. The engine's native expert rows take these kernels for the trellis types
+/// on every CPU (AVX-512 or AVX2 where the CPU has them, else the scalar kernel: a ggml from upstream has no trellis
+/// type to fall back to, and the AVX code is only reached behind tq_isa_ok). STRATA_NO_TQ_KERNELS=1 asks for
+/// ggml-cpu's vec_dot instead, honoured only by a ggml that has the types (the agentionai fork; tests, A/B).
bool tq_kernels_enabled();
const char* tq_isa_name(TqIsa isa);
diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp
index e3072b934..484340188 100644
--- a/src/kernels/cpu/native_expert.cpp
+++ b/src/kernels/cpu/native_expert.cpp
@@ -27,6 +27,14 @@ namespace {
const ggml_type_traits_cpu* traits(int type) { return ggml_get_type_traits_cpu((ggml_type) type); }
+// Do the trellis rows take Strata's own kernels (tq_cpu.hpp)? Always, unless STRATA_NO_TQ_KERNELS=1 asks for ggml-cpu's
+// vec_dot AND this ggml has the type (the agentionai fork; an upstream ggml does not, and must never be asked about it).
+bool tq_rows_own(int type) {
+ if (!tq_supported(type)) return false;
+ if (tq_kernels_enabled()) return true;
+ return type >= (int) GGML_TYPE_COUNT || traits(type) == nullptr || traits(type)->vec_dot == nullptr;
+}
+
void init_once() {
static std::once_flag once;
std::call_once(once, [] { ggml_cpu_init(); });
@@ -52,31 +60,39 @@ bool kq256_supported(int type) noexcept { return type == 12 || type == 7 || type
bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt& f, std::string& err) {
init_once();
- const ggml_type_traits_cpu* tg = traits(gu_type);
- const ggml_type_traits_cpu* td = traits(d_type);
- if (tg == nullptr || tg->vec_dot == nullptr || td == nullptr || td->vec_dot == nullptr) {
- err = "native experts: ggml-cpu has no dot product for type " + std::to_string(tg && tg->vec_dot ? d_type : gu_type);
+ // The trellis types (Gyro, ggml types 144-146) are not in a ggml built from upstream (and an upstream ggml indexes its
+ // type table by the id, so asking it about 144 reads past the table): their geometry and activation format (Q8_0)
+ // are known here, and their rows run on tq_cpu.hpp.
+ const bool tq_g = tq_supported(gu_type), tq_d = tq_supported(d_type);
+ const ggml_type_traits_cpu* tg = tq_g ? nullptr : traits(gu_type);
+ const ggml_type_traits_cpu* td = tq_d ? nullptr : traits(d_type);
+ if ((!tq_g && (tg == nullptr || tg->vec_dot == nullptr)) || (!tq_d && (td == nullptr || td->vec_dot == nullptr))) {
+ err = "native experts: ggml-cpu has no dot product for type " +
+ std::to_string(!tq_g && !(tg && tg->vec_dot) ? gu_type : d_type);
return false;
}
- const ggml_type_traits_cpu* ag = traits(tg->vec_dot_type);
- const ggml_type_traits_cpu* ad = traits(td->vec_dot_type);
+ const ggml_type vdt_g = tq_g ? GGML_TYPE_Q8_0 : tg->vec_dot_type;
+ const ggml_type vdt_d = tq_d ? GGML_TYPE_Q8_0 : td->vec_dot_type;
+ const ggml_type_traits_cpu* ag = traits(vdt_g);
+ const ggml_type_traits_cpu* ad = traits(vdt_d);
if (ag == nullptr || ag->from_float == nullptr || ad == nullptr || ad->from_float == nullptr) {
err = "native experts: ggml-cpu cannot quantize an activation for this layer";
return false;
}
- if (n_embd % ggml_blck_size((ggml_type) gu_type) || n_ff % ggml_blck_size((ggml_type) d_type) ||
- n_embd % ggml_blck_size(tg->vec_dot_type) || n_ff % ggml_blck_size(td->vec_dot_type)) {
+ if ((tq_g ? n_embd % kTqBlock : n_embd % ggml_blck_size((ggml_type) gu_type)) ||
+ (tq_d ? n_ff % kTqBlock : n_ff % ggml_blck_size((ggml_type) d_type)) ||
+ n_embd % ggml_blck_size(vdt_g) || n_ff % ggml_blck_size(vdt_d)) {
err = "native experts: expert geometry is not whole blocks";
return false;
}
f.gu_type = gu_type;
f.d_type = d_type;
- f.gu_act = (int) tg->vec_dot_type;
- f.d_act = (int) td->vec_dot_type;
+ f.gu_act = (int) vdt_g;
+ f.d_act = (int) vdt_d;
f.n_embd = n_embd;
f.n_ff = n_ff;
- f.gu_row = ggml_row_size((ggml_type) gu_type, n_embd);
- f.d_row = ggml_row_size((ggml_type) d_type, n_ff);
+ f.gu_row = tq_g ? (size_t) (n_embd / kTqBlock) * tq_block_bytes(gu_type) : ggml_row_size((ggml_type) gu_type, n_embd);
+ f.d_row = tq_d ? (size_t) (n_ff / kTqBlock) * tq_block_bytes(d_type) : ggml_row_size((ggml_type) d_type, n_ff);
f.up_off = f.gu_row * (size_t) n_ff;
f.down_off = 2 * f.up_off;
f.bytes = f.down_off + f.d_row * (size_t) n_embd;
@@ -86,8 +102,8 @@ bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt
f.tail_off = f.bytes;
f.bytes += kNvfp4Tail;
}
- f.act_bytes = ggml_row_size(tg->vec_dot_type, n_embd);
- f.h_bytes = ggml_row_size(td->vec_dot_type, n_ff);
+ f.act_bytes = ggml_row_size(vdt_g, n_embd);
+ f.h_bytes = ggml_row_size(vdt_d, n_ff);
if (f.act_bytes > kNativeActBytes || f.h_bytes > kNativeHBytes) {
err = "native experts: activation larger than the pool's buffers";
return false;
@@ -190,7 +206,7 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
}
// The trellis types (TQ2_T / TQK6 / TQK7): Strata's own kernels (tq_cpu.hpp), for every group size - they decode
// a block once for all the tokens and give each token the same bits whatever the group (no #152 rule needed).
- if (tq_supported(f.gu_type) && tq_kernels_enabled()) {
+ if (tq_rows_own(f.gu_type)) {
tq_gu_rows(tq_isa(), f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
return;
}
@@ -261,7 +277,7 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const
// Both multi-token kernels below are /arch:AVX2 translation units (kq_avx2.cpp and iq_avx2.cpp),
// so a CPU without AVX2 has to reach ggml-cpu's vec_dot instead - same reasoning as the gate/up
// rows above, where `avx512` tested cpu_avx512_ok() and `avx2` did not.
- if (tq_supported(f.d_type) && tq_kernels_enabled()) { // the trellis types, as in native_gu_rows
+ if (tq_rows_own(f.d_type)) { // the trellis types, as in native_gu_rows
tq_rows(tq_isa(), f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);
return;
}