Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

CPU trellis kernels: AVX2 / AVX-512 decode-and-dot for TQ2_T / TQK6 / TQK7

Laurent Zuijdwijkd0a02fe4264c
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3e253561d..44b6ec4e9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1353,7 +1353,8 @@ if(STRATA_NATIVE_EXPERTS)
target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp src/kernels/cpu/iq_avx512.cpp
src/kernels/cpu/iq_avx2.cpp src/kernels/cpu/kq_avx2.cpp
src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp src/kernels/cpu/iq_avx1.cpp
- src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp)
+ src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp
+ src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)
target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
if(MSVC)
set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp
@@ -1361,6 +1362,8 @@ if(STRATA_NATIVE_EXPERTS)
set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
+ set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
+ set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
# the router lookahead's AVX1 dot (older CPUs; called behind cpu_avx1_ok): no FMA, no 256-bit integer ops
set_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")
# the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): 128-bit integer lanes,
@@ -1374,6 +1377,10 @@ if(STRATA_NATIVE_EXPERTS)
set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
+ # the trellis kernels: AVX2 + FMA, and AVX-512 F/BW/VL/VBMI/VNNI (exactly what cpu_avx512_ok() checks)
+ set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
+ set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS
+ "-mavx512f;-mavx512bw;-mavx512vl;-mavx512vnni;-mavx512vbmi;-mavx2;-mfma;-mf16c")
set_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "-mavx")
# the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): SSE4.2 + AVX with
# AVX2/FMA/F16C explicitly off, so a -march=native build host cannot leak AVX2 code into the file that
@@ -1816,6 +1823,13 @@ if(STRATA_BUILD_TESTS)
target_link_libraries(native_hadamard_test PRIVATE strata_kernels_cpu strata_artifact)
add_test(NAME native_hadamard_test COMMAND native_hadamard_test)
set_tests_properties(native_hadamard_test PROPERTIES TIMEOUT 300)
+ # the CPU trellis kernels (tq_cpu.hpp): AVX2 / AVX-512 vs the scalar kernel bit for bit, vs ggml-cpu's vec_dot
+ # and to_float, plain and with the Hadamard rotation; `tq_cpu_parity --bench` is the micro-benchmark
+ add_executable(tq_cpu_parity src/kernels/cpu/tq_cpu_parity.cpp)
+ target_link_libraries(tq_cpu_parity PRIVATE strata_kernels_cpu)
+ add_test(NAME tq_cpu_parity COMMAND tq_cpu_parity)
+ add_test(NAME tq_cpu_parity_avx2 COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_AVX2=1 $<TARGET_FILE:tq_cpu_parity>)
+ set_tests_properties(tq_cpu_parity tq_cpu_parity_avx2 PROPERTIES TIMEOUT 300)
endif()
add_executable(expert_parity src/kernels/cpu/expert_parity.cpp)
diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hpp
new file mode 100644
index 000000000..6f5f83e09
--- /dev/null
+++ b/include/strata/kernels/cpu/tq_cpu.hpp
@@ -0,0 +1,80 @@
+// include/strata/kernels/cpu/tq_cpu.hpp - the trellis expert formats (TQ2_T / TQK6 / TQK7, ggml types 144-146 of the
+// agention llama.cpp fork) on the CPU: decode-and-dot against Q8_0 activations (ggml's block_q8_0), one or several
+// tokens per call, on AVX-512, AVX2 or plain C++.
+//
+// THE FORMAT (third_party/ggml/ggml-common-tq.h): a 128-weight block is {fp16 d; 32 steps of K bits} (K = 8 / 6 / 7).
+// Step t has a 15-bit state; the state's hash picks two points (pairs) of a 2048-point fp16 codebook, which are
+// weights 4t..4t+3, times d.
+//
+// THE DECODE: a 32768 x 4 int16 table maps a state straight to its four weights at scale 2^13 (256 KiB, the
+// approach of the fork's ggml-cpu vec_dot): one 8-byte lookup per four weights, no hash. The int16 rounding is
+// exact for codebook values >= 2^-3 in magnitude and within 2^-14 below that; it is the same table, rounded the
+// same way, as the fork's ggml-cpu kernels.
+//
+// THE ARITHMETIC, identical on every ISA so every path gives the same bits (and so does any token-group size):
+// S[i][k] = sum_j w16[i][32k + j] * q8[i][k][j] exact in int32 (|w16| < 2^15, |q8| <= 127)
+// acc[k] = fma((float) S[i][k], d_w[i] * d_a[i][k], acc[k]) for blocks i in order, k = 0..3
+// dot = ((acc[0] + acc[1]) + (acc[2] + acc[3])) * 2^-13
+// i.e. four float accumulators, one per Q8_0 sub-block position, each a fixed sequential chain. It differs from
+// ggml-cpu's vec_dot (same integers, other float summation order) by float rounding only.
+//
+// THE KERNELS: `tq_isa()` picks AVX-512 (F/BW/VL/VBMI/VNNI, cpu_avx512_ok) or AVX2 (cpu_avx2_ok) at run time; the
+// scalar kernel is the reference (on a CPU without AVX2 the engine keeps ggml-cpu's vec_dot, see tq_kernels_enabled).
+// STRATA_TQ_ISA=scalar|avx2|avx512 caps the choice (tests, A/B); STRATA_NO_TQ_KERNELS=1 makes native_expert.cpp use
+// ggml-cpu's vec_dot instead (the previous path).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::kernels::cpu {
+
+enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };
+
+/// TQ2_T (144), TQK6 (145), TQK7 (146).
+bool tq_supported(int ggml_type) noexcept;
+/// Bits per step: 8, 6, 7 (0 for any other type).
+int tq_kbits(int ggml_type) noexcept;
+/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).
+TqIsa tq_isa();
+/// Whether this CPU can run `isa` at all (scalar: always).
+bool tq_isa_ok(TqIsa isa);
+/// Whether the engine's native expert rows take these kernels for the trellis types: on an AVX2 or AVX-512 CPU
+/// unless STRATA_NO_TQ_KERNELS=1; the scalar kernel only when STRATA_TQ_ISA=scalar asks for it (otherwise a CPU
+/// without AVX2 keeps ggml-cpu's vec_dot, compiled for the build's floor).
+bool tq_kernels_enabled();
+const char* tq_isa_name(TqIsa isa);
+
+/// The 32768 x 4 int16 state table (filled on first use; thread-safe).
+const int16_t* tq_state_table();
+
+/// Rows [r0, r1) of a trellis matrix (`row_bytes` per row, `n` weights per row) against `nt` (<= 8) Q8_0
+/// activations of n values: out[t][r] = w_r . a[t].
+void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
+ float* const* out, int r0, int r1);
+/// Gate and up rows [r0, r1) (gate at blob, up at blob + up_off): ff[t][r] = silu(g) * u, the SwiGLU expression
+/// of native_gu_rows.
+void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,
+ const void* const* act, int nt, float* const* ff, int r0, int r1);
+
+/// Largest token group per call and widest row the kernels take.
+inline constexpr int kTqMaxTokens = 8;
+inline constexpr int kTqMaxN = 4096;
+
+namespace tq_detail {
+/// One token's Q8_0 activation unpacked for the kernels: the int8 values widened to int16 and the scales as floats.
+struct TqAct {
+ alignas(64) int16_t q[kTqMaxN];
+ alignas(64) float d[kTqMaxN / 32];
+};
+void unpack_act(const void* q8_0, int n, TqAct& a);
+// the per-ISA row kernels (dot products only; `out[t][r]` for rows [r0, r1))
+void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1);
+void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1);
+void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1);
+} // namespace tq_detail
+
+} // namespace strata::kernels::cpu
diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp
index 7ee0e36aa..e3072b934 100644
--- a/src/kernels/cpu/native_expert.cpp
+++ b/src/kernels/cpu/native_expert.cpp
@@ -11,6 +11,7 @@
#include "strata/kernels/cpu/nvfp4_avx512.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/cpu/hadamard.hpp"
+#include "strata/kernels/cpu/tq_cpu.hpp"
#include "ggml.h"
#include "ggml-cpu.h"
@@ -187,6 +188,12 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*
kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
return;
}
+ // The trellis types (TQ2_T / TQK6 / TQK7): Strata's own kernels (tq_cpu.hpp), for every group size - they decode
+ // a block once for all the tokens and give each token the same bits whatever the group (no #152 rule needed).
+ if (tq_supported(f.gu_type) && tq_kernels_enabled()) {
+ tq_gu_rows(tq_isa(), f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);
+ return;
+ }
// A format with only an AVX-2 kernel (IQ4_XS, #415) takes it on AVX-2 CPUs only: an AVX-512 CPU keeps ggml-cpu for
// it, as before (its rows would round differently). Each kernel only for the formats it implements: falling
// through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.
@@ -254,6 +261,10 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const
// Both multi-token kernels below are /arch:AVX2 translation units (kq_avx2.cpp and iq_avx2.cpp),
// so a CPU without AVX2 has to reach ggml-cpu's vec_dot instead - same reasoning as the gate/up
// rows above, where `avx512` tested cpu_avx512_ok() and `avx2` did not.
+ if (tq_supported(f.d_type) && tq_kernels_enabled()) { // the trellis types, as in native_gu_rows
+ tq_rows(tq_isa(), f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);
+ return;
+ }
if (cpu_avx2_ok() && kq && nt >= 2 && (f.d_type == 7 || f.d_type == 8)) { // Q5_1 / Q8_0 down: bit-exact, any group size
kq256_rows(f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);
return;
diff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cpp
new file mode 100644
index 000000000..00918edf6
--- /dev/null
+++ b/src/kernels/cpu/tq_avx2.cpp
@@ -0,0 +1,87 @@
+// src/kernels/cpu/tq_avx2.cpp - the trellis expert rows (TQ2_T / TQK6 / TQK7) on AVX2 + FMA (tq_cpu.hpp).
+// Compiled for AVX2 only and reached behind cpu_avx2_ok(), so nothing here can fault on an older CPU.
+//
+// Per 128-weight block: the 32 states in scalar code (tq_states.hpp: one 32-bit read, a shift and a mask each, used
+// as table indices straight away), four 8-byte state-table rows per YMM register as scalar loads, then per token
+// VPMADDWD against the unpacked int16 activations and a VPHADDD tree that leaves the four exact sub-block sums
+// S[k] in one XMM, so the float part is the scalar kernel's, lane for lane. The weights are decoded once per block
+// for all the tokens of the group.
+//
+// Measured on Zen 5 (cycles per block, one token, AVX2): ~48 with this scheme; computing the states with byte
+// shuffles instead (VPSHUFB / VPSRLVD per 8 states) and moving them to GPRs or through memory was slower (60-80),
+// and ggml-cpu's AVX2 vec_dot for these types (agentionai llama.cpp, the same state table) takes about 2.3x as long.
+#include "strata/kernels/cpu/tq_cpu.hpp"
+
+#include "tq_states.hpp"
+
+#include <immintrin.h>
+
+namespace strata::kernels::cpu::tq_detail {
+namespace {
+
+// four state-table rows (8 bytes each) into one YMM: a load and three broadcast loads blended in. Broadcast loads
+// and VPBLENDD avoid the shuffle port that VPINSRQ / VINSERTI128 need (the two measure the same on Zen 5).
+inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2, uint32_t s3) {
+ __m256i v = _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i*) (tab + s0)));
+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s1]), 0x0C);
+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s2]), 0x30);
+ return _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s3]), 0xC0);
+}
+
+template <int KB>
+void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+ int r1) {
+ const int nb = n / 128;
+ constexpr size_t bsz = 2 + 4 * KB;
+ const long long* tab = (const long long*) tq_state_table();
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* x = w + (size_t) r * row_bytes;
+ __m128 acc[kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) acc[t] = _mm_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = x + (size_t) i * bsz;
+ const uint8_t* qs = blk + 2;
+ const uint64_t wr = tq_wrap<KB>(qs);
+ __m256i wv[8]; // wv[m] = weights 16m .. 16m+15 (states 4m .. 4m+3)
+#if defined(__GNUC__)
+#pragma GCC unroll 8
+#endif
+ for (int m = 0; m < 8; ++m)
+ wv[m] = rows4(tab, tq_state<KB>(qs, wr, 4 * m), tq_state<KB>(qs, wr, 4 * m + 1),
+ tq_state<KB>(qs, wr, 4 * m + 2), tq_state<KB>(qs, wr, 4 * m + 3));
+ const __m128 dx = _mm_set1_ps(tq_half_to_float(tq_ld16(blk)));
+ for (int t = 0; t < nt; ++t) {
+ const __m256i* q = (const __m256i*) (a[t]->q + 128 * i);
+ const __m256i p0 = _mm256_add_epi32(_mm256_madd_epi16(wv[0], _mm256_load_si256(q + 0)),
+ _mm256_madd_epi16(wv[1], _mm256_load_si256(q + 1)));
+ const __m256i p1 = _mm256_add_epi32(_mm256_madd_epi16(wv[2], _mm256_load_si256(q + 2)),
+ _mm256_madd_epi16(wv[3], _mm256_load_si256(q + 3)));
+ const __m256i p2 = _mm256_add_epi32(_mm256_madd_epi16(wv[4], _mm256_load_si256(q + 4)),
+ _mm256_madd_epi16(wv[5], _mm256_load_si256(q + 5)));
+ const __m256i p3 = _mm256_add_epi32(_mm256_madd_epi16(wv[6], _mm256_load_si256(q + 6)),
+ _mm256_madd_epi16(wv[7], _mm256_load_si256(q + 7)));
+ // per 128-bit lane [p0, p1, p2, p3] partial sums, then the two lanes added: S[0..3], exact
+ const __m256i h = _mm256_hadd_epi32(_mm256_hadd_epi32(p0, p1), _mm256_hadd_epi32(p2, p3));
+ const __m128i s = _mm_add_epi32(_mm256_castsi256_si128(h), _mm256_extracti128_si256(h, 1));
+ const __m128 sc = _mm_mul_ps(dx, _mm_loadu_ps(a[t]->d + 4 * i));
+ acc[t] = _mm_fmadd_ps(_mm_cvtepi32_ps(s), sc, acc[t]);
+ }
+ }
+ for (int t = 0; t < nt; ++t) {
+ alignas(16) float v[4];
+ _mm_store_ps(v, acc[t]);
+ out[t][r] = ((v[0] + v[1]) + (v[2] + v[3])) * (1.0f / 8192.0f);
+ }
+ }
+}
+
+} // namespace
+
+void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1) {
+ if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+}
+
+} // namespace strata::kernels::cpu::tq_detail
diff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cpp
new file mode 100644
index 000000000..b45118844
--- /dev/null
+++ b/src/kernels/cpu/tq_avx512.cpp
@@ -0,0 +1,95 @@
+// src/kernels/cpu/tq_avx512.cpp - the trellis expert rows (TQ2_T / TQK6 / TQK7) on AVX-512 F/BW/VL/VNNI
+// (tq_cpu.hpp). Reached behind cpu_avx512_ok() only.
+//
+// Per 128-weight block: the AVX2 kernel's decode (scalar states, scalar 8-byte state-table loads, one ZMM of 32
+// int16 weights per Q8_0 sub-block); per token four VPDPWSSD against the unpacked int16 activations, an unpack/add
+// tree to the four exact sub-block sums S[k] in one XMM, and the scalar kernel's float steps lane for lane. The
+// weights are decoded once per block for all the tokens of the group.
+//
+// Measured on Zen 5 (cycles per block): the decode bounds it, so one token is no faster than AVX2 (~48); each extra
+// token of a group costs ~10% less than on AVX2. ggml-cpu's AVX-512 decode for these types - all states from two
+// VPERMB and the table rows by VPGATHERQQ - was 20-30% slower here: Zen gathers are slow, and so are Intel ones
+// under the Gather Data Sampling microcode mitigation.
+#include "strata/kernels/cpu/tq_cpu.hpp"
+
+#include "tq_states.hpp"
+
+#include <immintrin.h>
+
+namespace strata::kernels::cpu::tq_detail {
+namespace {
+
+// four ZMM of 16 int32 partial sums -> [S0, S1, S2, S3] (exact integer adds)
+inline __m128i reduce4(__m512i p0, __m512i p1, __m512i p2, __m512i p3) {
+ const __m512i a = _mm512_add_epi32(_mm512_unpacklo_epi32(p0, p1), _mm512_unpackhi_epi32(p0, p1));
+ const __m512i b = _mm512_add_epi32(_mm512_unpacklo_epi32(p2, p3), _mm512_unpackhi_epi32(p2, p3));
+ const __m512i c = _mm512_add_epi32(_mm512_unpacklo_epi64(a, b), _mm512_unpackhi_epi64(a, b));
+ // c: per 128-bit lane [p0, p1, p2, p3] partial sums; add the four lanes
+ const __m256i d = _mm256_add_epi32(_mm512_castsi512_si256(c), _mm512_extracti64x4_epi64(c, 1));
+ return _mm_add_epi32(_mm256_castsi256_si128(d), _mm256_extracti128_si256(d, 1));
+}
+
+// four state-table rows (8 bytes each) into one YMM: scalar loads, broadcast and blended in; two make a ZMM
+inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2, uint32_t s3) {
+ __m256i v = _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i*) (tab + s0)));
+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s1]), 0x0C);
+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s2]), 0x30);
+ return _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s3]), 0xC0);
+}
+
+template <int KB>
+inline __m512i rows8(const long long* tab, const uint8_t* qs, uint64_t wr, int t0) {
+ const __m256i lo = rows4(tab, tq_state<KB>(qs, wr, t0), tq_state<KB>(qs, wr, t0 + 1), tq_state<KB>(qs, wr, t0 + 2),
+ tq_state<KB>(qs, wr, t0 + 3));
+ const __m256i hi = rows4(tab, tq_state<KB>(qs, wr, t0 + 4), tq_state<KB>(qs, wr, t0 + 5),
+ tq_state<KB>(qs, wr, t0 + 6), tq_state<KB>(qs, wr, t0 + 7));
+ return _mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1);
+}
+
+template <int KB>
+void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,
+ int r1) {
+ const int nb = n / 128;
+ constexpr size_t bsz = 2 + 4 * KB;
+ const long long* tab = (const long long*) tq_state_table();
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* x = w + (size_t) r * row_bytes;
+ __m128 acc[kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) acc[t] = _mm_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = x + (size_t) i * bsz;
+ const uint8_t* qs = blk + 2;
+ const uint64_t wr = tq_wrap<KB>(qs);
+ // states 8k .. 8k+7 -> weights 32k .. 32k+31 = Q8_0 sub-block k
+ const __m512i w0 = rows8<KB>(tab, qs, wr, 0), w1 = rows8<KB>(tab, qs, wr, 8);
+ const __m512i w2 = rows8<KB>(tab, qs, wr, 16), w3 = rows8<KB>(tab, qs, wr, 24);
+ const __m128 dx = _mm_set1_ps(tq_half_to_float(tq_ld16(blk)));
+ for (int t = 0; t < nt; ++t) {
+ const __m512i* q = (const __m512i*) (a[t]->q + 128 * i);
+ const __m512i z = _mm512_setzero_si512();
+ const __m128i s = reduce4(_mm512_dpwssd_epi32(z, w0, _mm512_load_si512(q + 0)),
+ _mm512_dpwssd_epi32(z, w1, _mm512_load_si512(q + 1)),
+ _mm512_dpwssd_epi32(z, w2, _mm512_load_si512(q + 2)),
+ _mm512_dpwssd_epi32(z, w3, _mm512_load_si512(q + 3)));
+ const __m128 sc = _mm_mul_ps(dx, _mm_loadu_ps(a[t]->d + 4 * i));
+ acc[t] = _mm_fmadd_ps(_mm_cvtepi32_ps(s), sc, acc[t]);
+ }
+ }
+ for (int t = 0; t < nt; ++t) {
+ alignas(16) float v[4];
+ _mm_store_ps(v, acc[t]);
+ out[t][r] = ((v[0] + v[1]) + (v[2] + v[3])) * (1.0f / 8192.0f);
+ }
+ }
+}
+
+} // namespace
+
+void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1) {
+ if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+}
+
+} // namespace strata::kernels::cpu::tq_detail
diff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cpp
new file mode 100644
index 000000000..94536b36d
--- /dev/null
+++ b/src/kernels/cpu/tq_cpu.cpp
@@ -0,0 +1,201 @@
+// src/kernels/cpu/tq_cpu.cpp - see include/strata/kernels/cpu/tq_cpu.hpp: the state table, the scalar reference
+// kernel, the activation unpacking and the run-time ISA choice. Compiled for the baseline x86-64 (no AVX), so
+// everything here runs on any CPU; the AVX2 / AVX-512 kernels are in their own translation units.
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+
+#include "tq_states.hpp"
+#include "ggml-common-tq.h"
+
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace strata::kernels::cpu {
+namespace {
+
+alignas(64) int16_t g_table[(tq_detail::kTqStateMask + 1) * 4];
+
+void fill_table() {
+ for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) {
+ const uint32_t x = s * 0x9e3779b1u;
+ const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u}; // the hash's two codebook points
+ for (int h = 0; h < 2; ++h)
+ for (int j = 0; j < 2; ++j) {
+ // fp16 * 2^13 is exact in float; round half away from zero, as the fork's ggml-cpu table does
+ const float v = tq_detail::tq_half_to_float(tq_lut_f16[2 * pts[h] + j]) * 8192.0f;
+ g_table[4 * s + 2 * h + j] = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);
+ }
+ }
+}
+
+int env_isa_cap() {
+ const char* e = std::getenv("STRATA_TQ_ISA");
+ if (e == nullptr || *e == 0) return 2;
+ const std::string v(e);
+ if (v == "scalar") return 0;
+ if (v == "avx2") return 1;
+ return 2;
+}
+
+template <int KB>
+void rows_scalar_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
+ float* const* out, int r0, int r1) {
+ const int nb = n / 128;
+ constexpr size_t bsz = 2 + 4 * KB;
+ const int16_t* tab = tq_state_table();
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* x = w + (size_t) r * row_bytes;
+ float acc[kTqMaxTokens][4] = {};
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = x + (size_t) i * bsz;
+ const float dx = tq_detail::tq_half_to_float(tq_detail::tq_ld16(blk));
+ uint32_t st[32];
+ tq_detail::tq_states<KB>(blk + 2, st);
+ for (int t = 0; t < nt; ++t) {
+ const int16_t* q = a[t]->q + 128 * i;
+ for (int k = 0; k < 4; ++k) {
+ int32_t s = 0;
+ for (int j = 0; j < 8; ++j) {
+ const int16_t* wv = tab + 4 * st[8 * k + j];
+ const int16_t* qv = q + 32 * k + 4 * j;
+ s += wv[0] * qv[0] + wv[1] * qv[1] + wv[2] * qv[2] + wv[3] * qv[3];
+ }
+ const float sc = dx * a[t]->d[4 * i + k];
+ acc[t][k] = std::fma((float) s, sc, acc[t][k]);
+ }
+ }
+ }
+ for (int t = 0; t < nt; ++t)
+ out[t][r] = ((acc[t][0] + acc[t][1]) + (acc[t][2] + acc[t][3])) * (1.0f / 8192.0f);
+ }
+}
+
+} // namespace
+
+const int16_t* tq_state_table() {
+ static std::once_flag once;
+ std::call_once(once, fill_table);
+ return g_table;
+}
+
+bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146; }
+
+int tq_kbits(int t) noexcept { return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : 0; }
+
+bool tq_isa_ok(TqIsa isa) {
+ switch (isa) {
+ case TqIsa::avx512: return cpu_avx512_ok();
+ case TqIsa::avx2: return cpu_avx2_ok();
+ default: return true;
+ }
+}
+
+TqIsa tq_isa() {
+ static const TqIsa isa = [] {
+ const int cap = env_isa_cap();
+ if (cap >= 2 && tq_isa_ok(TqIsa::avx512)) return TqIsa::avx512;
+ if (cap >= 1 && tq_isa_ok(TqIsa::avx2)) return TqIsa::avx2;
+ return TqIsa::scalar;
+ }();
+ return isa;
+}
+
+bool tq_kernels_enabled() {
+ static const bool on = [] {
+ if (const char* e = std::getenv("STRATA_NO_TQ_KERNELS"); e != nullptr && std::atoi(e) != 0) return false;
+ return tq_isa() != TqIsa::scalar || env_isa_cap() == 0;
+ }();
+ return on;
+}
+
+const char* tq_isa_name(TqIsa isa) {
+ return isa == TqIsa::avx512 ? "avx512" : isa == TqIsa::avx2 ? "avx2" : "scalar";
+}
+
+namespace tq_detail {
+
+void unpack_act(const void* q8_0, int n, TqAct& a) {
+ const uint8_t* p = (const uint8_t*) q8_0;
+ for (int b = 0; b < n / 32; ++b, p += 34) { // block_q8_0: fp16 d, 32 x int8
+ a.d[b] = tq_half_to_float(tq_ld16(p));
+ const int8_t* q = (const int8_t*) (p + 2);
+ for (int j = 0; j < 32; ++j) a.q[32 * b + j] = q[j];
+ }
+}
+
+void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,
+ float* const* out, int r0, int r1) {
+ if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);
+ else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);
+ else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);
+}
+
+} // namespace tq_detail
+
+namespace {
+
+// the activations of one call, unpacked once for all its rows
+struct ActSet {
+ tq_detail::TqAct act[kTqMaxTokens];
+ const tq_detail::TqAct* ptr[kTqMaxTokens];
+};
+
+ActSet& unpack_all(const void* const* act, int n, int nt) {
+ thread_local std::unique_ptr<ActSet> set;
+ if (!set) set.reset(new ActSet);
+ for (int t = 0; t < nt; ++t) {
+ tq_detail::unpack_act(act[t], n, set->act[t]);
+ set->ptr[t] = &set->act[t];
+ }
+ return *set;
+}
+
+void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,
+ float* const* out, int r0, int r1) {
+ switch (isa) {
+ case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+ case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+ default: tq_detail::tq_rows_scalar(kb, w, row_bytes, n, a, nt, out, r0, r1); break;
+ }
+}
+
+} // namespace
+
+void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,
+ float* const* out, int r0, int r1) {
+ if (!tq_isa_ok(isa)) std::abort(); // a caller bug: asked for an ISA this CPU does not have
+ tq_state_table();
+ const ActSet& s = unpack_all(act, n, nt);
+ rows_on(isa, tq_kbits(ggml_type), w, row_bytes, n, s.ptr, nt, out, r0, r1);
+}
+
+void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,
+ const void* const* act, int nt, float* const* ff, int r0, int r1) {
+ if (!tq_isa_ok(isa)) std::abort();
+ tq_state_table();
+ const ActSet& s = unpack_all(act, n, nt);
+ thread_local std::vector<float> gbuf, ubuf;
+ const size_t nr = (size_t) r1; // row r of token t at [t * r1 + r]
+ if (gbuf.size() < nr * kTqMaxTokens) { gbuf.resize(nr * kTqMaxTokens); ubuf.resize(nr * kTqMaxTokens); }
+ float* gp[kTqMaxTokens];
+ float* up[kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) {
+ gp[t] = gbuf.data() + (size_t) t * nr;
+ up[t] = ubuf.data() + (size_t) t * nr;
+ }
+ const int kb = tq_kbits(ggml_type);
+ rows_on(isa, kb, blob, gu_row, n, s.ptr, nt, gp, r0, r1);
+ rows_on(isa, kb, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);
+ for (int t = 0; t < nt; ++t)
+ for (int r = r0; r < r1; ++r) {
+ const float g = gp[t][r], u = up[t][r];
+ ff[t][r] = (g / (1.f + std::exp(-g))) * u; // native_gu_rows' SwiGLU expression
+ }
+}
+
+} // namespace strata::kernels::cpu
diff --git a/src/kernels/cpu/tq_cpu_parity.cpp b/src/kernels/cpu/tq_cpu_parity.cpp
new file mode 100644
index 000000000..2f860ee66
--- /dev/null
+++ b/src/kernels/cpu/tq_cpu_parity.cpp
@@ -0,0 +1,565 @@
+// src/kernels/cpu/tq_cpu_parity.cpp - the CPU trellis kernels (tq_cpu.hpp: TQ2_T / TQK6 / TQK7) against their scalar
+// reference, against ggml-cpu's own vec_dot and to_float, and their speed. CPU only (no GPU runtime needed).
+//
+// tq_cpu_parity every check below for tqk6/tqk7 and tq2_t/tq2_t, plain and Hadamard
+// tq_cpu_parity --synthetic GU/DOWN one expert pair (types by ggml name, e.g. tqk6/tqk7)
+// tq_cpu_parity --synthetic-hadamard GU/DOWN the same with the block-128 Hadamard rotation of the activations
+// tq_cpu_parity --bench [--quick] [--only K] the micro-benchmark (run it pinned: taskset -c <cores>);
+// --only: the kernels whose name contains K
+//
+// Checks, per matrix (random stream bytes and scales, so every state is reached, and ggml-quantized Gaussian rows):
+// (1) AVX2 and AVX-512 kernels == the scalar kernel, bit for bit, for groups of 1..8 tokens
+// (2) a token's row in a group of k == the same token alone, bit for bit (no group-size dependence)
+// (3) vs ggml-cpu's vec_dot on the same Q8_0 activations: the same integers summed in another float order, so
+// ~1e-7 relative
+// (4) vs ggml's to_float weights . the dequantized Q8_0 activations in double: the int16 state table's rounding
+// (exact for |codebook| >= 2^-3, within 2^-14 below) plus float rounding, ~1e-6 relative
+// and per expert (native_gu_rows -> native_quant_h -> native_down_rows, the engine's entry points, which now take
+// these kernels) against the same expert computed with ggml-cpu's vec_dot: float rounding plus the odd Q8_0
+// rounding flip of the intermediate.
+#include "strata/kernels/cpu/tq_cpu.hpp"
+#include "strata/kernels/cpu/native_expert.hpp"
+#include "strata/kernels/cpu/expert.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
+
+#include "ggml.h"
+#include "ggml-cpu.h"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <random>
+#include <string>
+#include <thread>
+#include <vector>
+
+using namespace strata::kernels::cpu;
+
+namespace {
+
+constexpr int H = 2560, FF = 640;
+
+int type_by_name(const std::string& s) {
+ for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
+ const char* n = ggml_type_name((ggml_type) t);
+ if (n != nullptr && s == n) return t;
+ }
+ return -1;
+}
+
+uint16_t f2h(float f) { return ggml_fp32_to_fp16(f); }
+
+// rows x n weights of `type`: random stream bytes and scales (every state reachable, both scale signs), and the
+// last rows (up to 32, ggml's trellis encoder is slow) ggml's own encoding of Gaussian rows
+std::vector<uint8_t> make_matrix(int type, int rows, int n, uint32_t seed) {
+ const size_t rb = ggml_row_size((ggml_type) type, n);
+ std::vector<uint8_t> m(rb * rows);
+ std::mt19937 rng(seed);
+ std::uniform_int_distribution<int> byte(0, 255);
+ std::uniform_real_distribution<float> scale(0.002f, 0.05f);
+ const size_t bsz = rb / (size_t) (n / 128);
+ const int half = rows - std::min(rows / 2, 32);
+ for (int r = 0; r < half; ++r)
+ for (int b = 0; b < n / 128; ++b) {
+ uint8_t* blk = m.data() + (size_t) r * rb + (size_t) b * bsz;
+ const uint16_t d = f2h((byte(rng) & 1 ? -1.f : 1.f) * scale(rng));
+ std::memcpy(blk, &d, 2);
+ for (size_t j = 2; j < bsz; ++j) blk[j] = (uint8_t) byte(rng);
+ }
+ std::normal_distribution<float> nd(0.f, 0.02f);
+ std::vector<float> wf((size_t) (rows - half) * n);
+ for (auto& v : wf) v = nd(rng);
+ ggml_quantize_chunk((ggml_type) type, wf.data(), m.data() + (size_t) half * rb, 0, rows - half, n, nullptr);
+ return m;
+}
+
+std::vector<float> make_x(int n, uint32_t seed) {
+ std::mt19937 rng(seed);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> x(n);
+ for (auto& v : x) v = nd(rng);
+ x[(size_t) (seed % (uint32_t) n)] = 40.f; // a massive channel, as the residual stream has
+ return x;
+}
+
+std::vector<uint8_t> q8_0(const std::vector<float>& x) {
+ std::vector<uint8_t> q(ggml_row_size(GGML_TYPE_Q8_0, (int64_t) x.size()));
+ ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float(x.data(), q.data(), (int64_t) x.size());
+ return q;
+}
+
+bool same_bits(const float* a, const float* b, size_t n) { return std::memcmp(a, b, n * sizeof(float)) == 0; }
+
+// one matrix: checks (1) - (4); returns failures
+int check_matrix(int type, int rows, int n, uint32_t seed, bool hadamard) {
+ int fails = 0;
+ const size_t rb = ggml_row_size((ggml_type) type, n);
+ const auto m = make_matrix(type, rows, n, seed);
+ constexpr int NT = kTqMaxTokens;
+ std::vector<std::vector<uint8_t>> act(NT);
+ std::vector<std::vector<float>> xr(NT);
+ std::vector<float> signs(n);
+ for (int i = 0; i < n; ++i) signs[i] = ((i * 2654435761u) >> 7) & 1 ? -1.f : 1.f;
+ const void* ap[NT];
+ for (int t = 0; t < NT; ++t) {
+ xr[t] = make_x(n, seed * 31 + t);
+ if (hadamard) hadamard_rotate(xr[t].data(), n, 128, signs.data());
+ act[t] = q8_0(xr[t]);
+ ap[t] = act[t].data();
+ }
+ auto run = [&](TqIsa isa, int nt, const void* const* a, std::vector<float>& o) {
+ o.assign((size_t) nt * rows, 0.f);
+ float* op[NT];
+ for (int t = 0; t < nt; ++t) op[t] = o.data() + (size_t) t * rows;
+ // two row ranges, as the pool splits them
+ tq_rows(isa, type, m.data(), rb, n, a, nt, op, 0, rows / 3);
+ tq_rows(isa, type, m.data(), rb, n, a, nt, op, rows / 3, rows);
+ };
+ std::vector<float> ref, got;
+ run(TqIsa::scalar, NT, ap, ref);
+ // (1) every ISA this CPU has, every group size, == scalar bit for bit; (2) group-size independence
+ for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {
+ if (!tq_isa_ok(isa)) { std::printf(" %-6s not on this CPU: skipped\n", tq_isa_name(isa)); continue; }
+ bool ok = true;
+ for (int nt = 1; nt <= NT; ++nt)
+ for (int t0 = 0; t0 + nt <= NT; t0 += nt) {
+ run(isa, nt, ap + t0, got);
+ ok = ok && same_bits(got.data(), ref.data() + (size_t) t0 * rows, (size_t) nt * rows);
+ }
+ std::printf(" %-6s groups of 1..%d tokens vs the scalar kernel's 8-token run: %s\n", tq_isa_name(isa), NT,
+ ok ? "bitwise equal" : "DIFFERENT");
+ fails += !ok;
+ }
+ // (3) ggml-cpu's vec_dot, (4) to_float in double
+ const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);
+ const auto* tt = ggml_get_type_traits((ggml_type) type);
+ const auto* q8t = ggml_get_type_traits(GGML_TYPE_Q8_0);
+ double num3 = 0, num4 = 0, den = 0, max3 = 0;
+ std::vector<float> wf(n), af(n);
+ for (int t = 0; t < 2; ++t) {
+ q8t->to_float(act[t].data(), af.data(), n);
+ for (int r = 0; r < rows; ++r) {
+ float g = 0;
+ tc->vec_dot(n, &g, 0, m.data() + (size_t) r * rb, 0, act[t].data(), 0, 1);
+ tt->to_float(m.data() + (size_t) r * rb, wf.data(), n);
+ double d = 0, mag = 0;
+ for (int i = 0; i < n; ++i) { d += (double) wf[i] * af[i]; mag += std::fabs((double) wf[i] * af[i]); }
+ const double ours = ref[(size_t) t * rows + r];
+ num3 += (ours - g) * (ours - g);
+ num4 += (ours - d) * (ours - d);
+ den += d * d;
+ max3 = std::max(max3, std::fabs(ours - g) / (mag + 1e-30));
+ }
+ }
+ const double r3 = std::sqrt(num3 / den), r4 = std::sqrt(num4 / den);
+ const bool ok3 = r3 < 1e-5 && max3 < 1e-5, ok4 = r4 < 1e-4;
+ std::printf(" vs ggml-cpu vec_dot: rel %.2e (worst row %.2e of sum|w a|) %s | vs to_float in double: rel %.2e %s\n",
+ r3, max3, ok3 ? "ok" : "FAILED", r4, ok4 ? "ok" : "FAILED");
+ return fails + !ok3 + !ok4;
+}
+
+// a whole expert through the engine's entry points vs the same expert on ggml-cpu's vec_dot
+int check_expert(int gu, int dn, bool hadamard) {
+ NativeFmt f;
+ std::string err;
+ if (!native_fmt(gu, dn, H, FF, f, err)) { std::printf("native_fmt: %s\n", err.c_str()); return 1; }
+ std::vector<float> sh(H), sf(FF);
+ for (int i = 0; i < H; ++i) sh[i] = (i * 7 % 3) ? 1.f : -1.f;
+ for (int i = 0; i < FF; ++i) sf[i] = (i * 5 % 3) ? -1.f : 1.f;
+ if (hadamard) {
+ f.had_block = 128;
+ f.had_gu = f.had_d = true;
+ f.gu_signs = sh.data();
+ f.d_signs = sf.data();
+ }
+ std::vector<uint8_t> blob(f.bytes);
+ const auto g = make_matrix(gu, FF, H, 101), u = make_matrix(gu, FF, H, 202), d = make_matrix(dn, H, FF, 303);
+ std::memcpy(blob.data(), g.data(), g.size());
+ std::memcpy(blob.data() + f.up_off, u.data(), u.size());
+ std::memcpy(blob.data() + f.down_off, d.data(), d.size());
+ constexpr int NT = 3;
+ double worst = 0;
+ bool bitwise_group = true;
+ std::vector<std::vector<float>> outs(NT, std::vector<float>(H));
+ std::vector<std::vector<uint8_t>> acts(NT, std::vector<uint8_t>(kNativeActBytes));
+ std::vector<std::vector<float>> ffs(NT, std::vector<float>(FF));
+ std::vector<std::vector<uint8_t>> hqs(NT, std::vector<uint8_t>(kNativeHBytes));
+ const void* a[NT];
+ float* ffp[NT];
+ const void* hp[NT];
+ float* op[NT];
+ for (int t = 0; t < NT; ++t) {
+ const auto x = make_x(H, 7 + t);
+ native_quant_act(f, x.data(), acts[t].data());
+ a[t] = acts[t].data();
+ ffp[t] = ffs[t].data();
+ hp[t] = hqs[t].data();
+ op[t] = outs[t].data();
+ }
+ native_gu_rows(f, blob.data(), a, NT, ffp, 0, FF);
+ for (int t = 0; t < NT; ++t) native_quant_h(f, ffs[t].data(), hqs[t].data());
+ native_down_rows(f, blob.data(), hp, NT, op, 0, H);
+ const auto* tg = ggml_get_type_traits_cpu((ggml_type) gu);
+ const auto* td = ggml_get_type_traits_cpu((ggml_type) dn);
+ for (int t = 0; t < NT; ++t) {
+ std::vector<float> ff(FF), out(H);
+ for (int r = 0; r < FF; ++r) {
+ float gg = 0, uu = 0;
+ tg->vec_dot(H, &gg, 0, blob.data() + (size_t) r * f.gu_row, 0, a[t], 0, 1);
+ tg->vec_dot(H, &uu, 0, blob.data() + f.up_off + (size_t) r * f.gu_row, 0, a[t], 0, 1);
+ ff[r] = (gg / (1.f + std::exp(-gg))) * uu;
+ }
+ std::vector<uint8_t> hq(kNativeHBytes);
+ native_quant_h(f, ff.data(), hq.data());
+ for (int r = 0; r < H; ++r)
+ td->vec_dot(FF, &out[r], 0, blob.data() + f.down_off + (size_t) r * f.d_row, 0, hq.data(), 0, 1);
+ double num = 0, den = 0;
+ for (int r = 0; r < H; ++r) { num += std::pow((double) outs[t][r] - out[r], 2); den += (double) out[r] * out[r]; }
+ worst = std::max(worst, std::sqrt(num / den));
+ // the same token alone through the engine path: the same bits as in the group of three
+ std::vector<float> ff1(FF), o1(H);
+ float* f1[1] = {ff1.data()};
+ float* o1p[1] = {o1.data()};
+ const void* a1[1] = {a[t]};
+ native_gu_rows(f, blob.data(), a1, 1, f1, 0, FF);
+ std::vector<uint8_t> h1(kNativeHBytes);
+ native_quant_h(f, ff1.data(), h1.data());
+ const void* h1p[1] = {h1.data()};
+ native_down_rows(f, blob.data(), h1p, 1, o1p, 0, H);
+ bitwise_group = bitwise_group && same_bits(o1.data(), outs[t].data(), H);
+ }
+ const bool ok = worst < 2e-3 && bitwise_group;
+ std::printf(" expert %s/%s%s through native_gu_rows/native_down_rows (%s, %s): vs ggml-cpu vec_dot rel %.2e, "
+ "alone vs in a group of %d: %s %s\n",
+ ggml_type_name((ggml_type) gu), ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "",
+ tq_kernels_enabled() ? "tq kernels" : "ggml-cpu", tq_isa_name(tq_isa()), worst, NT,
+ bitwise_group ? "bitwise equal" : "DIFFERENT", ok ? "ok" : "FAILED");
+ return ok ? 0 : 1;
+}
+
+int synthetic(int gu, int dn, bool hadamard) {
+ int fails = 0;
+ std::printf("%s/%s%s (tq kernels: %s, best ISA here %s)\n", ggml_type_name((ggml_type) gu),
+ ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "", tq_kernels_enabled() ? "on" : "off",
+ tq_isa_name(tq_isa()));
+ std::printf(" %s, 96 rows of %d (gate/up shape)\n", ggml_type_name((ggml_type) gu), H);
+ fails += check_matrix(gu, 96, H, 1000 + gu, hadamard);
+ std::printf(" %s, 256 rows of %d (down shape)\n", ggml_type_name((ggml_type) dn), FF);
+ fails += check_matrix(dn, 256, FF, 2000 + dn, hadamard);
+ fails += check_expert(gu, dn, hadamard);
+ return fails;
+}
+
+// ------------------------------------------------------------------------------------------------ the benchmark
+
+double now_us() {
+ return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+
+struct BenchExpert {
+ std::vector<uint8_t> gu, dn; // gate rows then up rows; down rows
+ size_t gu_row, d_row;
+};
+
+// one variant: compute one expert (gate+up rows, then down rows) for `nt` tokens
+using ExpertFn = std::function<void(const BenchExpert&, int nt)>;
+
+// the core clock right now: a chain of dependent adds runs at one per cycle (GCC / Clang on x86-64; 0 elsewhere)
+double core_ghz() {
+#if defined(__GNUC__) && defined(__x86_64__)
+ uint64_t x = 0;
+ const long n = 1000000;
+ const double t0 = now_us();
+ for (long i = 0; i < n; ++i) __asm__ volatile("add $1, %0\n\tadd $1, %0\n\tadd $1, %0\n\tadd $1, %0" : "+r"(x));
+ return 4.0 * n / ((now_us() - t0) * 1e3);
+#else
+ return 0.0;
+#endif
+}
+
+constexpr double kBlocksPerExpert = 2.0 * FF * (H / 128) + (double) H * (FF / 128); // 128-weight blocks
+
+// one slice of `ms` milliseconds cycling over the experts: ns per expert call and the clock around it (negated when
+// the clock moved by more than 5% during the slice: such samples count only when a variant has no other)
+std::pair<double, double> time_slice(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int nt, double ms,
+ size_t& next) {
+ const double g0 = core_ghz();
+ int it = 0;
+ const double t0 = now_us();
+ double t1 = t0;
+ do {
+ fn(ex[next++ % ex.size()], nt);
+ ++it;
+ t1 = now_us();
+ } while (t1 - t0 < ms * 1000.0 || it < 2);
+ const double g1 = core_ghz();
+ const double ghz = 0.5 * (g0 + g1);
+ return {(t1 - t0) * 1e3 / it, ghz > 0 && std::fabs(g1 - g0) > 0.05 * g0 ? -ghz : ghz};
+}
+
+double median(std::vector<double> v) {
+ if (v.empty()) return 0.0;
+ std::sort(v.begin(), v.end());
+ return v[v.size() / 2];
+}
+
+// `threads` threads, each cycling over its own experts: aggregate weight GB/s
+double time_parallel(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int threads, double ms) {
+ std::atomic<long> done{0};
+ std::atomic<bool> stop{false};
+ std::vector<std::thread> th;
+ const double t0 = now_us();
+ for (int k = 0; k < threads; ++k)
+ th.emplace_back([&, k] {
+ long c = 0;
+ for (size_t i = (size_t) k; !stop.load(std::memory_order_relaxed); i += (size_t) threads, ++c)
+ fn(ex[i % ex.size()], 1);
+ done += c;
+ });
+ std::this_thread::sleep_for(std::chrono::milliseconds((int) ms));
+ stop = true;
+ for (auto& t : th) t.join();
+ const double us = now_us() - t0;
+ const double bytes = (double) (ex[0].gu_row * 2 * FF + ex[0].dn.size()) * (double) done.load();
+ return bytes / (us * 1e3);
+}
+
+int bench(bool quick, const std::string& only) {
+ const int ncold = quick ? 48 : 96; // ~50-130 MB of weights: past the last-level cache
+ std::printf("CPU: %s | AVX-512 kernels: %s, AVX2: %s | ggml-cpu built for this CPU (-march=native unless "
+ "STRATA_PORTABLE)\n", cpu_name().c_str(), cpu_avx512_ok() ? "yes" : "no", cpu_avx2_ok() ? "yes" : "no");
+ std::printf("expert = gate+up %d x %d and down %d x %d of one type, %d distinct experts cycled (their weights come "
+ "from DRAM, as a cache miss's do)\n\n", FF, H, H, FF, ncold);
+ struct Variant { std::string name; int type; ExpertFn fn; bool ok; };
+ std::vector<Variant> vars;
+ auto add_tq = [&](int type) {
+ for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {
+ vars.push_back({std::string("strata ") + tq_isa_name(isa), type,
+ [type, isa](const BenchExpert& e, int nt) {
+ thread_local std::vector<uint8_t> a, h;
+ thread_local std::vector<float> ff, out;
+ a.resize(ggml_row_size(GGML_TYPE_Q8_0, H) * kTqMaxTokens);
+ h.resize(ggml_row_size(GGML_TYPE_Q8_0, FF) * kTqMaxTokens);
+ ff.resize((size_t) FF * kTqMaxTokens);
+ out.resize((size_t) H * kTqMaxTokens);
+ const void* ap[kTqMaxTokens];
+ const void* hp[kTqMaxTokens];
+ float* fp[kTqMaxTokens];
+ float* op[kTqMaxTokens];
+ for (int t = 0; t < nt; ++t) {
+ ap[t] = a.data() + t * ggml_row_size(GGML_TYPE_Q8_0, H);
+ hp[t] = h.data() + t * ggml_row_size(GGML_TYPE_Q8_0, FF);
+ fp[t] = ff.data() + (size_t) t * FF;
+ op[t] = out.data() + (size_t) t * H;
+ }
+ tq_gu_rows(isa, type, e.gu.data(), e.gu_row, e.gu_row * FF, H, ap, nt, fp, 0, FF);
+ tq_rows(isa, type, e.dn.data(), e.d_row, FF, hp, nt, op, 0, H);
+ },
+ tq_isa_ok(isa)});
+ }
+ vars.push_back({"ggml-cpu vec_dot", type,
+ [type](const BenchExpert& e, int nt) {
+ const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);
+ thread_local std::vector<uint8_t> a(ggml_row_size(GGML_TYPE_Q8_0, H)),
+ h(ggml_row_size(GGML_TYPE_Q8_0, FF));
+ thread_local std::vector<float> ff(FF), out(H);
+ for (int t = 0; t < nt; ++t) {
+ for (int r = 0; r < FF; ++r) {
+ float g, u;
+ tc->vec_dot(H, &g, 0, e.gu.data() + (size_t) r * e.gu_row, 0, a.data(), 0, 1);
+ tc->vec_dot(H, &u, 0, e.gu.data() + (size_t) (FF + r) * e.gu_row, 0, a.data(), 0, 1);
+ ff[r] = (g / (1.f + std::exp(-g))) * u;
+ }
+ for (int r = 0; r < H; ++r)
+ tc->vec_dot(FF, &out[r], 0, e.dn.data() + (size_t) r * e.d_row, 0, h.data(), 0, 1);
+ }
+ },
+ true});
+ };
+ auto add_q2 = [&](bool avx512) {
+ vars.push_back({avx512 ? "strata Q2_0 avx512" : "strata Q2_0 avx2", 42,
+ [avx512](const BenchExpert& e, int nt) {
+ thread_local ActQ a1[MAXT], a2[MAXT];
+ thread_local bool init = false;
+ if (!init) { // real scales and codes (the kernel reads every field)
+ const auto x = make_x(H, 3), h = make_x(FF, 4);
+ for (int t = 0; t < MAXT; ++t) { act_quant_q8_1_avx2(x.data(), H, a1[t]); act_quant_q8_1_avx2(h.data(), FF, a2[t]); }
+ init = true;
+ }
+ thread_local float g[MAXT][FF], u[MAXT][FF], out[MAXT][H];
+ const ActQ* ap[MAXT];
+ const ActQ* hp[MAXT];
+ float* gp[MAXT];
+ float* up[MAXT];
+ float* op[MAXT];
+ for (int t = 0; t < nt; ++t) { ap[t] = &a1[t]; hp[t] = &a2[t]; gp[t] = g[t]; up[t] = u[t]; op[t] = out[t]; }
+ auto rows = avx512 ? q2_0_gguf_rows_multi : q2_0_gguf_rows_multi_avx2;
+ rows(e.gu.data(), e.gu_row, H / 64, ap, nt, gp, 0, FF);
+ rows(e.gu.data() + e.gu_row * FF, e.gu_row, H / 64, ap, nt, up, 0, FF);
+ for (int t = 0; t < nt; ++t)
+ for (int r = 0; r < FF; ++r) g[t][r] = (g[t][r] / (1.f + std::exp(-g[t][r]))) * u[t][r];
+ rows(e.dn.data(), e.d_row, FF / 64, hp, nt, op, 0, H);
+ },
+ avx512 ? cpu_avx512_ok() : cpu_avx2_ok()});
+ };
+ for (int type : {145, 146, 144}) add_tq(type);
+ add_q2(false);
+ add_q2(true);
+ // the Q2_0 pack's own AVX-512 kernel (the repacked 1,382,400-byte blob that setup makes on AVX-512 CPUs)
+ vars.push_back({"strata Q2_0 avx512 pack", 42,
+ [](const BenchExpert& e, int nt) {
+ thread_local ActQ a1, a2;
+ thread_local bool init = false;
+ if (!init) {
+ const auto x = make_x(H, 3), h = make_x(FF, 4);
+ act_quant_q8_1(x.data(), H, a1);
+ act_quant_q8_1(h.data(), FF, a2);
+ init = true;
+ }
+ thread_local float ff[FF], out[H];
+ for (int t = 0; t < nt; ++t) { // its multi-token kernels are the same per token
+ s2_expert_gu_rows(e.gu.data(), a1, ff, 0, FF);
+ s2_expert_down_rows(e.gu.data(), a2, out, 0, H);
+ }
+ },
+ cpu_avx512_ok()});
+
+ auto make_experts = [&](int type, int count) {
+ std::vector<BenchExpert> v((size_t) count);
+ for (int k = 0; k < count; ++k) {
+ BenchExpert& e = v[(size_t) k];
+ e.gu_row = ggml_row_size((ggml_type) type, H);
+ e.d_row = ggml_row_size((ggml_type) type, FF);
+ // (Q2_0: room for one repacked blob too, which the pack kernel reads from the same buffer)
+ e.gu.resize(type == 42 ? std::max(e.gu_row * 2 * FF, (size_t) BLOB) : e.gu_row * 2 * FF);
+ e.dn.resize(e.d_row * H);
+ std::mt19937 rng(77 + k);
+ for (auto& b : e.gu) b = (uint8_t) rng();
+ for (auto& b : e.dn) b = (uint8_t) rng();
+ // sane fp16 scales at every block start (Q2_0: 18-byte blocks, TQ: 26/30/34)
+ const size_t bsz = type == 42 ? 18 : ggml_row_size((ggml_type) type, 128);
+ const uint16_t d = f2h(0.01f);
+ for (size_t o = 0; o + bsz <= e.gu.size(); o += bsz) std::memcpy(e.gu.data() + o, &d, 2);
+ for (size_t o = 0; o + bsz <= e.dn.size(); o += bsz) std::memcpy(e.dn.data() + o, &d, 2);
+ if (type == 42) { // the same bytes also hold one repacked Q2_0 blob (codes, then fp16 scales)
+ for (size_t o = O_GU_SCALES; o + 2 <= BLOB; o += 2) std::memcpy(e.gu.data() + o, &d, 2);
+ }
+ }
+ return v;
+ };
+ // Single thread, the variants of a type interleaved in short slices over several rounds (this machine's clock
+ // moves; a slice whose clock moved is dropped), medians reported: ns and core cycles per 128-weight block, us
+ // per expert, weight GB/s; and per token in a group of three (a verify window's tokens routed to one expert).
+ // The activations are zero-filled (the kernels do not branch on values) except Q2_0's, which are real.
+ const int rounds = quick ? 5 : 15;
+ const double slice_ms = quick ? 40 : 60;
+ std::printf("%-6s %-24s %9s %8s %8s %8s | %10s %8s\n", "type", "kernel", "us/expert", "ns/blk", "cyc/blk", "GB/s",
+ "x3: us/tok", "cyc/blk");
+ for (int type : {145, 146, 144, 42}) {
+ const auto cold = make_experts(type, ncold);
+ const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size()); // weight bytes per expert
+ std::vector<const Variant*> vs;
+ for (const auto& v : vars)
+ if (v.type == type && v.name.find(only) != std::string::npos) {
+ if (v.ok) vs.push_back(&v);
+ else std::printf("%-6s %-24s (not on this CPU)\n", ggml_type_name((ggml_type) type), v.name.c_str());
+ }
+ std::vector<std::vector<double>> ns1(vs.size()), cy1(vs.size()), ns3(vs.size()), cy3(vs.size());
+ std::vector<std::vector<double>> ns1m(vs.size()), cy1m(vs.size()), ns3m(vs.size()), cy3m(vs.size());
+ size_t next = 0;
+ for (auto* v : vs) v->fn(cold[0], 3); // warm-up: tables, buffers
+ for (int r = 0; r < rounds; ++r)
+ for (size_t k = 0; k < vs.size(); ++k) {
+ const bool slow = vs[k]->name.find("scalar") != std::string::npos;
+ const auto a = time_slice(cold, vs[k]->fn, 1, slow ? slice_ms / 2 : slice_ms, next);
+ const auto b = time_slice(cold, vs[k]->fn, 3, slow ? slice_ms / 2 : slice_ms, next);
+ if (a.second >= 0) { ns1[k].push_back(a.first); cy1[k].push_back(a.first * a.second); }
+ else { ns1m[k].push_back(a.first); cy1m[k].push_back(-a.first * a.second); }
+ if (b.second >= 0) { ns3[k].push_back(b.first); cy3[k].push_back(b.first * b.second); }
+ else { ns3m[k].push_back(b.first); cy3m[k].push_back(-b.first * b.second); }
+ }
+ std::vector<size_t> steady(vs.size());
+ for (size_t k = 0; k < vs.size(); ++k) {
+ steady[k] = ns1[k].size();
+ if (ns1[k].empty()) { ns1[k] = ns1m[k]; cy1[k] = cy1m[k]; }
+ if (ns3[k].empty()) { ns3[k] = ns3m[k]; cy3[k] = cy3m[k]; }
+ }
+ for (size_t k = 0; k < vs.size(); ++k) {
+ const double n1 = median(ns1[k]), c1 = median(cy1[k]), n3 = median(ns3[k]), c3 = median(cy3[k]);
+ std::printf("%-6s %-24s %9.1f %8.1f %8.1f %8.2f | %10.1f %8.1f (%zu/%d steady)\n",
+ ggml_type_name((ggml_type) type), vs[k]->name.c_str(), n1 / 1e3, n1 / kBlocksPerExpert,
+ c1 / kBlocksPerExpert, eb / n1, n3 / 3e3, c3 / kBlocksPerExpert, steady[k], rounds);
+ }
+ std::fflush(stdout);
+ }
+ // several workers at once (as the pool runs a layer's misses): aggregate GB/s on the cold set
+ const int nthr = (int) std::max(1u, std::min(4u, std::thread::hardware_concurrency()));
+ std::printf("\n%d threads at once, cold experts, one token: aggregate weight GB/s (us per expert per thread)\n", nthr);
+ for (int type : {145, 146, 144, 42}) {
+ const auto cold = make_experts(type, ncold);
+ const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size());
+ std::vector<const Variant*> vs;
+ for (const auto& v : vars)
+ if (v.type == type && v.ok && v.name.find("scalar") == std::string::npos && v.name.find(only) != std::string::npos)
+ vs.push_back(&v);
+ std::vector<std::vector<double>> g(vs.size());
+ for (int r = 0; r < (quick ? 2 : 4); ++r)
+ for (size_t k = 0; k < vs.size(); ++k) g[k].push_back(time_parallel(cold, vs[k]->fn, nthr, quick ? 150 : 250));
+ for (size_t k = 0; k < vs.size(); ++k) {
+ const double gbs = median(g[k]);
+ std::printf("%-6s %-24s %8.2f GB/s (%7.1f us per expert per thread)\n", ggml_type_name((ggml_type) type),
+ vs[k]->name.c_str(), gbs, eb / (gbs * 1e3) * nthr);
+ }
+ std::fflush(stdout);
+ }
+ // the per-token Hadamard rotation of one layer's activation (once per token per layer, shared by its experts)
+ {
+ std::vector<float> x = make_x(H, 5), s(H, 1.f);
+ const int it = 20000;
+ const double t0 = now_us();
+ for (int i = 0; i < it; ++i) hadamard_rotate(x.data(), H, 128, s.data());
+ std::printf("\nHadamard rotation of one %d-wide activation (block 128): %.2f us\n", H, (now_us() - t0) / it);
+ }
+ return 0;
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ ggml_cpu_init();
+ if (GGML_TYPE_COUNT <= 146) {
+ std::fprintf(stderr, "this ggml has no trellis types (build with STRATA_GGML_DIR = the agention llama.cpp fork)\n");
+ return 1;
+ }
+ if (argc >= 2 && std::strcmp(argv[1], "--bench") == 0) { // --bench [--quick] [--only <kernel name part>]
+ bool quick = false;
+ std::string only;
+ for (int i = 2; i < argc; ++i) {
+ if (std::strcmp(argv[i], "--quick") == 0) quick = true;
+ else if (std::strcmp(argv[i], "--only") == 0 && i + 1 < argc) only = argv[++i];
+ }
+ return bench(quick, only);
+ }
+ if (argc >= 3 && (std::strcmp(argv[1], "--synthetic") == 0 || std::strcmp(argv[1], "--synthetic-hadamard") == 0)) {
+ const std::string pair = argv[2];
+ const size_t sl = pair.find('/');
+ const int gu = type_by_name(pair.substr(0, sl)), dn = sl == std::string::npos ? gu : type_by_name(pair.substr(sl + 1));
+ if (!tq_supported(gu) || !tq_supported(dn)) { std::fprintf(stderr, "not a trellis pair: %s\n", argv[2]); return 2; }
+ const int f = synthetic(gu, dn, std::strcmp(argv[1], "--synthetic-hadamard") == 0);
+ std::printf("%d failures\n", f);
+ return f ? 1 : 0;
+ }
+ int f = 0;
+ for (bool had : {false, true}) {
+ f += synthetic(145, 146, had); // Gyro-S
+ f += synthetic(144, 144, had); // Gyro-M
+ }
+ std::printf("%d failures\n", f);
+ return f ? 1 : 0;
+}
diff --git a/src/kernels/cpu/tq_states.hpp b/src/kernels/cpu/tq_states.hpp
new file mode 100644
index 000000000..bb4eaa7a1
--- /dev/null
+++ b/src/kernels/cpu/tq_states.hpp
@@ -0,0 +1,88 @@
+// src/kernels/cpu/tq_states.hpp - the 32 trellis states of one TQ2_T / TQK6 / TQK7 block in plain C++ (all three
+// kernels: scalar code beat byte-shuffle versions on AVX2 and AVX-512). Internal to src/kernels/cpu/tq_*.cpp.
+//
+// TQ2_T (K = 8): state t = qs[(t + 31) % 32] << 8 | qs[t], masked to 15 bits.
+// TQK (K = 6, 7): the 16 stream bits from bit (31 - t)*K, LSB first, circular modulo 32*K bits, masked to 15 bits.
+// Every window is one unaligned 32-bit little-endian read inside the block's 4*K bytes, except the windows that run
+// past the end (t = 0 and 1), which read a 64-bit view of the last four bytes followed by the first four.
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+namespace strata::kernels::cpu::tq_detail {
+
+inline constexpr uint32_t kTqStateMask = 0x7fffu;
+
+inline uint32_t tq_ld32(const uint8_t* p) {
+ uint32_t v;
+ std::memcpy(&v, p, 4);
+ return v; // x86 (and every target Strata builds for) is little-endian
+}
+
+/// State t alone (t a constant after unrolling); `wrap` = tq_wrap<KB>(qs) for the TQK windows past the end.
+template <int KB>
+inline uint64_t tq_wrap(const uint8_t* qs) {
+ if constexpr (KB == 8) return 0;
+ else return (uint64_t) tq_ld32(qs + 4 * KB - 4) | ((uint64_t) tq_ld32(qs) << 32);
+}
+template <int KB>
+inline uint32_t tq_state(const uint8_t* qs, uint64_t wrap, int t) {
+ if constexpr (KB == 8) {
+ return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & kTqStateMask;
+ } else {
+ constexpr int NB = 4 * KB;
+ const int off = (31 - t) * KB, b = off >> 3;
+ const uint32_t v = b + 4 <= NB ? tq_ld32(qs + b) >> (off & 7) : (uint32_t) (wrap >> (off - 8 * (NB - 4)));
+ return v & kTqStateMask;
+ }
+}
+
+template <int KB>
+inline void tq_states(const uint8_t* qs, uint32_t* st) {
+ if constexpr (KB == 8) {
+ st[0] = (((uint32_t) qs[31] << 8) | qs[0]) & kTqStateMask;
+#if defined(__GNUC__)
+#pragma GCC unroll 31
+#endif
+ for (int t = 1; t < 32; ++t) st[t] = (((uint32_t) qs[t - 1] << 8) | qs[t]) & kTqStateMask;
+ } else {
+ constexpr int NB = 4 * KB;
+ const uint64_t wrap = (uint64_t) tq_ld32(qs + NB - 4) | ((uint64_t) tq_ld32(qs) << 32);
+#if defined(__GNUC__)
+#pragma GCC unroll 32
+#endif
+ for (int t = 0; t < 32; ++t) {
+ const int off = (31 - t) * KB, b = off >> 3;
+ const uint32_t v = b + 4 <= NB ? tq_ld32(qs + b) >> (off & 7) : (uint32_t) (wrap >> (off - 8 * (NB - 4)));
+ st[t] = v & kTqStateMask;
+ }
+ }
+}
+
+/// fp16 bits -> float, exact (the block scales and the Q8_0 activation scales).
+inline float tq_half_to_float(uint16_t h) {
+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;
+ uint32_t exp = (h >> 10) & 0x1fu, man = h & 0x3ffu, bits;
+ if (exp == 0x1fu) {
+ bits = sign | 0x7f800000u | (man << 13);
+ } else if (exp != 0) {
+ bits = sign | ((exp + 112u) << 23) | (man << 13);
+ } else if (man == 0) {
+ bits = sign;
+ } else { // subnormal: man * 2^-24, exact in float
+ const float m = (float) man * (1.0f / 16777216.0f);
+ return sign ? -m : m;
+ }
+ float f;
+ std::memcpy(&f, &bits, 4);
+ return f;
+}
+
+inline uint16_t tq_ld16(const uint8_t* p) {
+ uint16_t v;
+ std::memcpy(&v, p, 2);
+ return v;
+}
+
+} // namespace strata::kernels::cpu::tq_detail
diff --git a/third_party/ggml/ggml-common-tq.h b/third_party/ggml/ggml-common-tq.h
index a1cf9d853..1ce3773bf 100644
--- a/third_party/ggml/ggml-common-tq.h
+++ b/third_party/ggml/ggml-common-tq.h
@@ -1,7 +1,7 @@
// third_party/ggml/ggml-common-tq.h - the trellis (APR) expert formats of Agention's llama.cpp fork
// (agentionai/llama.cpp 073bc9e68, ggml/src/ggml-common.h; MIT, the same license as the rest of third_party/ggml).
-// The block structs and the 2048-point "Hyb" codebook, unchanged, for Strata's device kernels. ggml-cpu takes
-// them from the fork itself (-DSTRATA_GGML_DIR).
+// The block structs and the 2048-point "Hyb" codebook, unchanged, for Strata's device kernels and its CPU trellis
+// kernels (src/kernels/cpu/tq_*.cpp). ggml-cpu takes them from the fork itself (-DSTRATA_GGML_DIR).
//
// TQ2_T / TQK6 / TQK7: 128 weights per block, {fp16 d; qs}. qs is 32 trellis steps of K bits (K = 8 / 6 / 7;
// TQ2_T byte-aligned, TQK bit-packed LSB-first, circular, step t's 16-bit window at bit (31 - t)*K; TQ2_T's state
@@ -32,9 +32,14 @@ typedef struct {
} block_tqk7;
static_assert(sizeof(block_tqk7) == 30, "wrong tqk7 block size/padding");
+// device memory in a CUDA / HIP translation unit, ordinary host data everywhere else
#if defined(__CUDACC__) || defined(__HIPCC__)
+#define STRATA_TQ_LUT_SPACE __device__
+#else
+#define STRATA_TQ_LUT_SPACE
+#endif
// point i = (tq_lut_f16[2i], tq_lut_f16[2i+1]) as fp16 bit patterns; 16-byte aligned so a point reads as one u32
-alignas(16) static const __device__ uint16_t tq_lut_f16[4096] = {
+alignas(16) static const STRATA_TQ_LUT_SPACE uint16_t tq_lut_f16[4096] = {
0xbd4d, 0xb8f7, 0xb6ce, 0x3cd5, 0xbd9e, 0x4368, 0x3dce, 0xb8a6, 0x395e, 0xb430, 0x3bc1, 0x3dbc, 0xb92b, 0x3c9c, 0xbca2, 0x3aaa,
0x3cdb, 0xb8e1, 0xbb33, 0x3dfb, 0x3a8d, 0xbb18, 0xbd64, 0x3da0, 0xbf1a, 0x3b83, 0xb389, 0x3659, 0xb40f, 0xabd7, 0x40ee, 0x35a2,
0xc057, 0x3972, 0xbe17, 0x3996, 0xb926, 0xb9b5, 0x393b, 0x3c75, 0x4151, 0xbdef, 0xb5a3, 0xbd09, 0xb408, 0x4012, 0x3b7f, 0x3ece,
@@ -292,4 +297,3 @@ alignas(16) static const __device__ uint16_t tq_lut_f16[4096] = {
0xa11a, 0xc068, 0x35bb, 0xb82e, 0x3f77, 0xbbb9, 0xb386, 0x3b46, 0xb675, 0x22f3, 0x3e8b, 0xbcb2, 0xb8ba, 0xba29, 0x4235, 0xbd08,
0x1038, 0x3f9f, 0xade8, 0x399f, 0x3363, 0xb9da, 0x3909, 0x37c8, 0x3c82, 0xbc4c, 0xbc3a, 0xb788, 0xa897, 0x39da, 0xb6d2, 0x3bee,
};
-#endif