Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
CPU trellis kernels: AVX2 / AVX-512 decode-and-dot for TQ2_T / TQK6 / TQK7
Laurent Zuijdwijkd0a02fe4264c
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 3e253561d..44b6ec4e9 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1353,7 +1353,8 @@ if(STRATA_NATIVE_EXPERTS)target_sources(strata_kernels_cpu PRIVATE src/kernels/cpu/native_expert.cpp src/kernels/cpu/iq_avx512.cppsrc/kernels/cpu/iq_avx2.cpp src/kernels/cpu/kq_avx2.cppsrc/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp src/kernels/cpu/iq_avx1.cpp- src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp)+ src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp+ src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)if(MSVC)set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp@@ -1361,6 +1362,8 @@ if(STRATA_NATIVE_EXPERTS)set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")+ set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")+ set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")# the router lookahead's AVX1 dot (older CPUs; called behind cpu_avx1_ok): no FMA, no 256-bit integer opsset_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")# the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): 128-bit integer lanes,@@ -1374,6 +1377,10 @@ if(STRATA_NATIVE_EXPERTS)set_source_files_properties(src/kernels/cpu/iq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")set_source_files_properties(src/kernels/cpu/kq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")+ # the trellis kernels: AVX2 + FMA, and AVX-512 F/BW/VL/VBMI/VNNI (exactly what cpu_avx512_ok() checks)+ set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")+ set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS+ "-mavx512f;-mavx512bw;-mavx512vl;-mavx512vnni;-mavx512vbmi;-mavx2;-mfma;-mf16c")set_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "-mavx")# the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): SSE4.2 + AVX with# AVX2/FMA/F16C explicitly off, so a -march=native build host cannot leak AVX2 code into the file that@@ -1816,6 +1823,13 @@ if(STRATA_BUILD_TESTS)target_link_libraries(native_hadamard_test PRIVATE strata_kernels_cpu strata_artifact)add_test(NAME native_hadamard_test COMMAND native_hadamard_test)set_tests_properties(native_hadamard_test PROPERTIES TIMEOUT 300)+ # the CPU trellis kernels (tq_cpu.hpp): AVX2 / AVX-512 vs the scalar kernel bit for bit, vs ggml-cpu's vec_dot+ # and to_float, plain and with the Hadamard rotation; `tq_cpu_parity --bench` is the micro-benchmark+ add_executable(tq_cpu_parity src/kernels/cpu/tq_cpu_parity.cpp)+ target_link_libraries(tq_cpu_parity PRIVATE strata_kernels_cpu)+ add_test(NAME tq_cpu_parity COMMAND tq_cpu_parity)+ add_test(NAME tq_cpu_parity_avx2 COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_AVX2=1 $<TARGET_FILE:tq_cpu_parity>)+ set_tests_properties(tq_cpu_parity tq_cpu_parity_avx2 PROPERTIES TIMEOUT 300)endif()add_executable(expert_parity src/kernels/cpu/expert_parity.cpp)diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppnew file mode 100644index 000000000..6f5f83e09--- /dev/null+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -0,0 +1,80 @@+// include/strata/kernels/cpu/tq_cpu.hpp - the trellis expert formats (TQ2_T / TQK6 / TQK7, ggml types 144-146 of the+// agention llama.cpp fork) on the CPU: decode-and-dot against Q8_0 activations (ggml's block_q8_0), one or several+// tokens per call, on AVX-512, AVX2 or plain C++.+//+// THE FORMAT (third_party/ggml/ggml-common-tq.h): a 128-weight block is {fp16 d; 32 steps of K bits} (K = 8 / 6 / 7).+// Step t has a 15-bit state; the state's hash picks two points (pairs) of a 2048-point fp16 codebook, which are+// weights 4t..4t+3, times d.+//+// THE DECODE: a 32768 x 4 int16 table maps a state straight to its four weights at scale 2^13 (256 KiB, the+// approach of the fork's ggml-cpu vec_dot): one 8-byte lookup per four weights, no hash. The int16 rounding is+// exact for codebook values >= 2^-3 in magnitude and within 2^-14 below that; it is the same table, rounded the+// same way, as the fork's ggml-cpu kernels.+//+// THE ARITHMETIC, identical on every ISA so every path gives the same bits (and so does any token-group size):+// S[i][k] = sum_j w16[i][32k + j] * q8[i][k][j] exact in int32 (|w16| < 2^15, |q8| <= 127)+// acc[k] = fma((float) S[i][k], d_w[i] * d_a[i][k], acc[k]) for blocks i in order, k = 0..3+// dot = ((acc[0] + acc[1]) + (acc[2] + acc[3])) * 2^-13+// i.e. four float accumulators, one per Q8_0 sub-block position, each a fixed sequential chain. It differs from+// ggml-cpu's vec_dot (same integers, other float summation order) by float rounding only.+//+// THE KERNELS: `tq_isa()` picks AVX-512 (F/BW/VL/VBMI/VNNI, cpu_avx512_ok) or AVX2 (cpu_avx2_ok) at run time; the+// scalar kernel is the reference (on a CPU without AVX2 the engine keeps ggml-cpu's vec_dot, see tq_kernels_enabled).+// STRATA_TQ_ISA=scalar|avx2|avx512 caps the choice (tests, A/B); STRATA_NO_TQ_KERNELS=1 makes native_expert.cpp use+// ggml-cpu's vec_dot instead (the previous path).+#pragma once++#include <cstddef>+#include <cstdint>++namespace strata::kernels::cpu {++enum class TqIsa : int { scalar = 0, avx2 = 1, avx512 = 2 };++/// TQ2_T (144), TQK6 (145), TQK7 (146).+bool tq_supported(int ggml_type) noexcept;+/// Bits per step: 8, 6, 7 (0 for any other type).+int tq_kbits(int ggml_type) noexcept;+/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).+TqIsa tq_isa();+/// Whether this CPU can run `isa` at all (scalar: always).+bool tq_isa_ok(TqIsa isa);+/// Whether the engine's native expert rows take these kernels for the trellis types: on an AVX2 or AVX-512 CPU+/// unless STRATA_NO_TQ_KERNELS=1; the scalar kernel only when STRATA_TQ_ISA=scalar asks for it (otherwise a CPU+/// without AVX2 keeps ggml-cpu's vec_dot, compiled for the build's floor).+bool tq_kernels_enabled();+const char* tq_isa_name(TqIsa isa);++/// The 32768 x 4 int16 state table (filled on first use; thread-safe).+const int16_t* tq_state_table();++/// Rows [r0, r1) of a trellis matrix (`row_bytes` per row, `n` weights per row) against `nt` (<= 8) Q8_0+/// activations of n values: out[t][r] = w_r . a[t].+void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,+ float* const* out, int r0, int r1);+/// Gate and up rows [r0, r1) (gate at blob, up at blob + up_off): ff[t][r] = silu(g) * u, the SwiGLU expression+/// of native_gu_rows.+void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,+ const void* const* act, int nt, float* const* ff, int r0, int r1);++/// Largest token group per call and widest row the kernels take.+inline constexpr int kTqMaxTokens = 8;+inline constexpr int kTqMaxN = 4096;++namespace tq_detail {+/// One token's Q8_0 activation unpacked for the kernels: the int8 values widened to int16 and the scales as floats.+struct TqAct {+ alignas(64) int16_t q[kTqMaxN];+ alignas(64) float d[kTqMaxN / 32];+};+void unpack_act(const void* q8_0, int n, TqAct& a);+// the per-ISA row kernels (dot products only; `out[t][r]` for rows [r0, r1))+void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1);+void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1);+void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1);+} // namespace tq_detail++} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cppindex 7ee0e36aa..e3072b934 100644--- a/src/kernels/cpu/native_expert.cpp+++ b/src/kernels/cpu/native_expert.cpp@@ -11,6 +11,7 @@#include "strata/kernels/cpu/nvfp4_avx512.hpp"#include "strata/kernels/cpu/expert_layout.hpp"#include "strata/kernels/cpu/hadamard.hpp"+#include "strata/kernels/cpu/tq_cpu.hpp"#include "ggml.h"#include "ggml-cpu.h"@@ -187,6 +188,12 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);return;}+ // The trellis types (TQ2_T / TQK6 / TQK7): Strata's own kernels (tq_cpu.hpp), for every group size - they decode+ // a block once for all the tokens and give each token the same bits whatever the group (no #152 rule needed).+ if (tq_supported(f.gu_type) && tq_kernels_enabled()) {+ tq_gu_rows(tq_isa(), f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);+ return;+ }// A format with only an AVX-2 kernel (IQ4_XS, #415) takes it on AVX-2 CPUs only: an AVX-512 CPU keeps ggml-cpu for// it, as before (its rows would round differently). Each kernel only for the formats it implements: falling// through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.@@ -254,6 +261,10 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const// Both multi-token kernels below are /arch:AVX2 translation units (kq_avx2.cpp and iq_avx2.cpp),// so a CPU without AVX2 has to reach ggml-cpu's vec_dot instead - same reasoning as the gate/up// rows above, where `avx512` tested cpu_avx512_ok() and `avx2` did not.+ if (tq_supported(f.d_type) && tq_kernels_enabled()) { // the trellis types, as in native_gu_rows+ tq_rows(tq_isa(), f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);+ return;+ }if (cpu_avx2_ok() && kq && nt >= 2 && (f.d_type == 7 || f.d_type == 8)) { // Q5_1 / Q8_0 down: bit-exact, any group sizekq256_rows(f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);return;diff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cppnew file mode 100644index 000000000..00918edf6--- /dev/null+++ b/src/kernels/cpu/tq_avx2.cpp@@ -0,0 +1,87 @@+// src/kernels/cpu/tq_avx2.cpp - the trellis expert rows (TQ2_T / TQK6 / TQK7) on AVX2 + FMA (tq_cpu.hpp).+// Compiled for AVX2 only and reached behind cpu_avx2_ok(), so nothing here can fault on an older CPU.+//+// Per 128-weight block: the 32 states in scalar code (tq_states.hpp: one 32-bit read, a shift and a mask each, used+// as table indices straight away), four 8-byte state-table rows per YMM register as scalar loads, then per token+// VPMADDWD against the unpacked int16 activations and a VPHADDD tree that leaves the four exact sub-block sums+// S[k] in one XMM, so the float part is the scalar kernel's, lane for lane. The weights are decoded once per block+// for all the tokens of the group.+//+// Measured on Zen 5 (cycles per block, one token, AVX2): ~48 with this scheme; computing the states with byte+// shuffles instead (VPSHUFB / VPSRLVD per 8 states) and moving them to GPRs or through memory was slower (60-80),+// and ggml-cpu's AVX2 vec_dot for these types (agentionai llama.cpp, the same state table) takes about 2.3x as long.+#include "strata/kernels/cpu/tq_cpu.hpp"++#include "tq_states.hpp"++#include <immintrin.h>++namespace strata::kernels::cpu::tq_detail {+namespace {++// four state-table rows (8 bytes each) into one YMM: a load and three broadcast loads blended in. Broadcast loads+// and VPBLENDD avoid the shuffle port that VPINSRQ / VINSERTI128 need (the two measure the same on Zen 5).+inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2, uint32_t s3) {+ __m256i v = _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i*) (tab + s0)));+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s1]), 0x0C);+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s2]), 0x30);+ return _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s3]), 0xC0);+}++template <int KB>+void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,+ int r1) {+ const int nb = n / 128;+ constexpr size_t bsz = 2 + 4 * KB;+ const long long* tab = (const long long*) tq_state_table();+ for (int r = r0; r < r1; ++r) {+ const uint8_t* x = w + (size_t) r * row_bytes;+ __m128 acc[kTqMaxTokens];+ for (int t = 0; t < nt; ++t) acc[t] = _mm_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = x + (size_t) i * bsz;+ const uint8_t* qs = blk + 2;+ const uint64_t wr = tq_wrap<KB>(qs);+ __m256i wv[8]; // wv[m] = weights 16m .. 16m+15 (states 4m .. 4m+3)+#if defined(__GNUC__)+#pragma GCC unroll 8+#endif+ for (int m = 0; m < 8; ++m)+ wv[m] = rows4(tab, tq_state<KB>(qs, wr, 4 * m), tq_state<KB>(qs, wr, 4 * m + 1),+ tq_state<KB>(qs, wr, 4 * m + 2), tq_state<KB>(qs, wr, 4 * m + 3));+ const __m128 dx = _mm_set1_ps(tq_half_to_float(tq_ld16(blk)));+ for (int t = 0; t < nt; ++t) {+ const __m256i* q = (const __m256i*) (a[t]->q + 128 * i);+ const __m256i p0 = _mm256_add_epi32(_mm256_madd_epi16(wv[0], _mm256_load_si256(q + 0)),+ _mm256_madd_epi16(wv[1], _mm256_load_si256(q + 1)));+ const __m256i p1 = _mm256_add_epi32(_mm256_madd_epi16(wv[2], _mm256_load_si256(q + 2)),+ _mm256_madd_epi16(wv[3], _mm256_load_si256(q + 3)));+ const __m256i p2 = _mm256_add_epi32(_mm256_madd_epi16(wv[4], _mm256_load_si256(q + 4)),+ _mm256_madd_epi16(wv[5], _mm256_load_si256(q + 5)));+ const __m256i p3 = _mm256_add_epi32(_mm256_madd_epi16(wv[6], _mm256_load_si256(q + 6)),+ _mm256_madd_epi16(wv[7], _mm256_load_si256(q + 7)));+ // per 128-bit lane [p0, p1, p2, p3] partial sums, then the two lanes added: S[0..3], exact+ const __m256i h = _mm256_hadd_epi32(_mm256_hadd_epi32(p0, p1), _mm256_hadd_epi32(p2, p3));+ const __m128i s = _mm_add_epi32(_mm256_castsi256_si128(h), _mm256_extracti128_si256(h, 1));+ const __m128 sc = _mm_mul_ps(dx, _mm_loadu_ps(a[t]->d + 4 * i));+ acc[t] = _mm_fmadd_ps(_mm_cvtepi32_ps(s), sc, acc[t]);+ }+ }+ for (int t = 0; t < nt; ++t) {+ alignas(16) float v[4];+ _mm_store_ps(v, acc[t]);+ out[t][r] = ((v[0] + v[1]) + (v[2] + v[3])) * (1.0f / 8192.0f);+ }+ }+}++} // namespace++void tq_rows_avx2(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+}++} // namespace strata::kernels::cpu::tq_detaildiff --git a/src/kernels/cpu/tq_avx512.cpp b/src/kernels/cpu/tq_avx512.cppnew file mode 100644index 000000000..b45118844--- /dev/null+++ b/src/kernels/cpu/tq_avx512.cpp@@ -0,0 +1,95 @@+// src/kernels/cpu/tq_avx512.cpp - the trellis expert rows (TQ2_T / TQK6 / TQK7) on AVX-512 F/BW/VL/VNNI+// (tq_cpu.hpp). Reached behind cpu_avx512_ok() only.+//+// Per 128-weight block: the AVX2 kernel's decode (scalar states, scalar 8-byte state-table loads, one ZMM of 32+// int16 weights per Q8_0 sub-block); per token four VPDPWSSD against the unpacked int16 activations, an unpack/add+// tree to the four exact sub-block sums S[k] in one XMM, and the scalar kernel's float steps lane for lane. The+// weights are decoded once per block for all the tokens of the group.+//+// Measured on Zen 5 (cycles per block): the decode bounds it, so one token is no faster than AVX2 (~48); each extra+// token of a group costs ~10% less than on AVX2. ggml-cpu's AVX-512 decode for these types - all states from two+// VPERMB and the table rows by VPGATHERQQ - was 20-30% slower here: Zen gathers are slow, and so are Intel ones+// under the Gather Data Sampling microcode mitigation.+#include "strata/kernels/cpu/tq_cpu.hpp"++#include "tq_states.hpp"++#include <immintrin.h>++namespace strata::kernels::cpu::tq_detail {+namespace {++// four ZMM of 16 int32 partial sums -> [S0, S1, S2, S3] (exact integer adds)+inline __m128i reduce4(__m512i p0, __m512i p1, __m512i p2, __m512i p3) {+ const __m512i a = _mm512_add_epi32(_mm512_unpacklo_epi32(p0, p1), _mm512_unpackhi_epi32(p0, p1));+ const __m512i b = _mm512_add_epi32(_mm512_unpacklo_epi32(p2, p3), _mm512_unpackhi_epi32(p2, p3));+ const __m512i c = _mm512_add_epi32(_mm512_unpacklo_epi64(a, b), _mm512_unpackhi_epi64(a, b));+ // c: per 128-bit lane [p0, p1, p2, p3] partial sums; add the four lanes+ const __m256i d = _mm256_add_epi32(_mm512_castsi512_si256(c), _mm512_extracti64x4_epi64(c, 1));+ return _mm_add_epi32(_mm256_castsi256_si128(d), _mm256_extracti128_si256(d, 1));+}++// four state-table rows (8 bytes each) into one YMM: scalar loads, broadcast and blended in; two make a ZMM+inline __m256i rows4(const long long* tab, uint32_t s0, uint32_t s1, uint32_t s2, uint32_t s3) {+ __m256i v = _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i*) (tab + s0)));+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s1]), 0x0C);+ v = _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s2]), 0x30);+ return _mm256_blend_epi32(v, _mm256_set1_epi64x(tab[s3]), 0xC0);+}++template <int KB>+inline __m512i rows8(const long long* tab, const uint8_t* qs, uint64_t wr, int t0) {+ const __m256i lo = rows4(tab, tq_state<KB>(qs, wr, t0), tq_state<KB>(qs, wr, t0 + 1), tq_state<KB>(qs, wr, t0 + 2),+ tq_state<KB>(qs, wr, t0 + 3));+ const __m256i hi = rows4(tab, tq_state<KB>(qs, wr, t0 + 4), tq_state<KB>(qs, wr, t0 + 5),+ tq_state<KB>(qs, wr, t0 + 6), tq_state<KB>(qs, wr, t0 + 7));+ return _mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1);+}++template <int KB>+void rows_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,+ int r1) {+ const int nb = n / 128;+ constexpr size_t bsz = 2 + 4 * KB;+ const long long* tab = (const long long*) tq_state_table();+ for (int r = r0; r < r1; ++r) {+ const uint8_t* x = w + (size_t) r * row_bytes;+ __m128 acc[kTqMaxTokens];+ for (int t = 0; t < nt; ++t) acc[t] = _mm_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = x + (size_t) i * bsz;+ const uint8_t* qs = blk + 2;+ const uint64_t wr = tq_wrap<KB>(qs);+ // states 8k .. 8k+7 -> weights 32k .. 32k+31 = Q8_0 sub-block k+ const __m512i w0 = rows8<KB>(tab, qs, wr, 0), w1 = rows8<KB>(tab, qs, wr, 8);+ const __m512i w2 = rows8<KB>(tab, qs, wr, 16), w3 = rows8<KB>(tab, qs, wr, 24);+ const __m128 dx = _mm_set1_ps(tq_half_to_float(tq_ld16(blk)));+ for (int t = 0; t < nt; ++t) {+ const __m512i* q = (const __m512i*) (a[t]->q + 128 * i);+ const __m512i z = _mm512_setzero_si512();+ const __m128i s = reduce4(_mm512_dpwssd_epi32(z, w0, _mm512_load_si512(q + 0)),+ _mm512_dpwssd_epi32(z, w1, _mm512_load_si512(q + 1)),+ _mm512_dpwssd_epi32(z, w2, _mm512_load_si512(q + 2)),+ _mm512_dpwssd_epi32(z, w3, _mm512_load_si512(q + 3)));+ const __m128 sc = _mm_mul_ps(dx, _mm_loadu_ps(a[t]->d + 4 * i));+ acc[t] = _mm_fmadd_ps(_mm_cvtepi32_ps(s), sc, acc[t]);+ }+ }+ for (int t = 0; t < nt; ++t) {+ alignas(16) float v[4];+ _mm_store_ps(v, acc[t]);+ out[t][r] = ((v[0] + v[1]) + (v[2] + v[3])) * (1.0f / 8192.0f);+ }+ }+}++} // namespace++void tq_rows_avx512(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ if (kb == 8) rows_t<8>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else rows_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+}++} // namespace strata::kernels::cpu::tq_detaildiff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cppnew file mode 100644index 000000000..94536b36d--- /dev/null+++ b/src/kernels/cpu/tq_cpu.cpp@@ -0,0 +1,201 @@+// src/kernels/cpu/tq_cpu.cpp - see include/strata/kernels/cpu/tq_cpu.hpp: the state table, the scalar reference+// kernel, the activation unpacking and the run-time ISA choice. Compiled for the baseline x86-64 (no AVX), so+// everything here runs on any CPU; the AVX2 / AVX-512 kernels are in their own translation units.+#include "strata/kernels/cpu/tq_cpu.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"++#include "tq_states.hpp"+#include "ggml-common-tq.h"++#include <cmath>+#include <cstdlib>+#include <cstring>+#include <memory>+#include <mutex>+#include <string>+#include <vector>++namespace strata::kernels::cpu {+namespace {++alignas(64) int16_t g_table[(tq_detail::kTqStateMask + 1) * 4];++void fill_table() {+ for (uint32_t s = 0; s <= tq_detail::kTqStateMask; ++s) {+ const uint32_t x = s * 0x9e3779b1u;+ const uint32_t pts[2] = {x >> 21, (x >> 10) & 2047u}; // the hash's two codebook points+ for (int h = 0; h < 2; ++h)+ for (int j = 0; j < 2; ++j) {+ // fp16 * 2^13 is exact in float; round half away from zero, as the fork's ggml-cpu table does+ const float v = tq_detail::tq_half_to_float(tq_lut_f16[2 * pts[h] + j]) * 8192.0f;+ g_table[4 * s + 2 * h + j] = (int16_t) (v < 0.0f ? v - 0.5f : v + 0.5f);+ }+ }+}++int env_isa_cap() {+ const char* e = std::getenv("STRATA_TQ_ISA");+ if (e == nullptr || *e == 0) return 2;+ const std::string v(e);+ if (v == "scalar") return 0;+ if (v == "avx2") return 1;+ return 2;+}++template <int KB>+void rows_scalar_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ const int nb = n / 128;+ constexpr size_t bsz = 2 + 4 * KB;+ const int16_t* tab = tq_state_table();+ for (int r = r0; r < r1; ++r) {+ const uint8_t* x = w + (size_t) r * row_bytes;+ float acc[kTqMaxTokens][4] = {};+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = x + (size_t) i * bsz;+ const float dx = tq_detail::tq_half_to_float(tq_detail::tq_ld16(blk));+ uint32_t st[32];+ tq_detail::tq_states<KB>(blk + 2, st);+ for (int t = 0; t < nt; ++t) {+ const int16_t* q = a[t]->q + 128 * i;+ for (int k = 0; k < 4; ++k) {+ int32_t s = 0;+ for (int j = 0; j < 8; ++j) {+ const int16_t* wv = tab + 4 * st[8 * k + j];+ const int16_t* qv = q + 32 * k + 4 * j;+ s += wv[0] * qv[0] + wv[1] * qv[1] + wv[2] * qv[2] + wv[3] * qv[3];+ }+ const float sc = dx * a[t]->d[4 * i + k];+ acc[t][k] = std::fma((float) s, sc, acc[t][k]);+ }+ }+ }+ for (int t = 0; t < nt; ++t)+ out[t][r] = ((acc[t][0] + acc[t][1]) + (acc[t][2] + acc[t][3])) * (1.0f / 8192.0f);+ }+}++} // namespace++const int16_t* tq_state_table() {+ static std::once_flag once;+ std::call_once(once, fill_table);+ return g_table;+}++bool tq_supported(int t) noexcept { return t == 144 || t == 145 || t == 146; }++int tq_kbits(int t) noexcept { return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : 0; }++bool tq_isa_ok(TqIsa isa) {+ switch (isa) {+ case TqIsa::avx512: return cpu_avx512_ok();+ case TqIsa::avx2: return cpu_avx2_ok();+ default: return true;+ }+}++TqIsa tq_isa() {+ static const TqIsa isa = [] {+ const int cap = env_isa_cap();+ if (cap >= 2 && tq_isa_ok(TqIsa::avx512)) return TqIsa::avx512;+ if (cap >= 1 && tq_isa_ok(TqIsa::avx2)) return TqIsa::avx2;+ return TqIsa::scalar;+ }();+ return isa;+}++bool tq_kernels_enabled() {+ static const bool on = [] {+ if (const char* e = std::getenv("STRATA_NO_TQ_KERNELS"); e != nullptr && std::atoi(e) != 0) return false;+ return tq_isa() != TqIsa::scalar || env_isa_cap() == 0;+ }();+ return on;+}++const char* tq_isa_name(TqIsa isa) {+ return isa == TqIsa::avx512 ? "avx512" : isa == TqIsa::avx2 ? "avx2" : "scalar";+}++namespace tq_detail {++void unpack_act(const void* q8_0, int n, TqAct& a) {+ const uint8_t* p = (const uint8_t*) q8_0;+ for (int b = 0; b < n / 32; ++b, p += 34) { // block_q8_0: fp16 d, 32 x int8+ a.d[b] = tq_half_to_float(tq_ld16(p));+ const int8_t* q = (const int8_t*) (p + 2);+ for (int j = 0; j < 32; ++j) a.q[32 * b + j] = q[j];+ }+}++void tq_rows_scalar(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ if (kb == 8) rows_scalar_t<8>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_scalar_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else rows_scalar_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+}++} // namespace tq_detail++namespace {++// the activations of one call, unpacked once for all its rows+struct ActSet {+ tq_detail::TqAct act[kTqMaxTokens];+ const tq_detail::TqAct* ptr[kTqMaxTokens];+};++ActSet& unpack_all(const void* const* act, int n, int nt) {+ thread_local std::unique_ptr<ActSet> set;+ if (!set) set.reset(new ActSet);+ for (int t = 0; t < nt; ++t) {+ tq_detail::unpack_act(act[t], n, set->act[t]);+ set->ptr[t] = &set->act[t];+ }+ return *set;+}++void rows_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ switch (isa) {+ case TqIsa::avx512: tq_detail::tq_rows_avx512(kb, w, row_bytes, n, a, nt, out, r0, r1); break;+ case TqIsa::avx2: tq_detail::tq_rows_avx2(kb, w, row_bytes, n, a, nt, out, r0, r1); break;+ default: tq_detail::tq_rows_scalar(kb, w, row_bytes, n, a, nt, out, r0, r1); break;+ }+}++} // namespace++void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,+ float* const* out, int r0, int r1) {+ if (!tq_isa_ok(isa)) std::abort(); // a caller bug: asked for an ISA this CPU does not have+ tq_state_table();+ const ActSet& s = unpack_all(act, n, nt);+ rows_on(isa, tq_kbits(ggml_type), w, row_bytes, n, s.ptr, nt, out, r0, r1);+}++void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,+ const void* const* act, int nt, float* const* ff, int r0, int r1) {+ if (!tq_isa_ok(isa)) std::abort();+ tq_state_table();+ const ActSet& s = unpack_all(act, n, nt);+ thread_local std::vector<float> gbuf, ubuf;+ const size_t nr = (size_t) r1; // row r of token t at [t * r1 + r]+ if (gbuf.size() < nr * kTqMaxTokens) { gbuf.resize(nr * kTqMaxTokens); ubuf.resize(nr * kTqMaxTokens); }+ float* gp[kTqMaxTokens];+ float* up[kTqMaxTokens];+ for (int t = 0; t < nt; ++t) {+ gp[t] = gbuf.data() + (size_t) t * nr;+ up[t] = ubuf.data() + (size_t) t * nr;+ }+ const int kb = tq_kbits(ggml_type);+ rows_on(isa, kb, blob, gu_row, n, s.ptr, nt, gp, r0, r1);+ rows_on(isa, kb, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);+ for (int t = 0; t < nt; ++t)+ for (int r = r0; r < r1; ++r) {+ const float g = gp[t][r], u = up[t][r];+ ff[t][r] = (g / (1.f + std::exp(-g))) * u; // native_gu_rows' SwiGLU expression+ }+}++} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/tq_cpu_parity.cpp b/src/kernels/cpu/tq_cpu_parity.cppnew file mode 100644index 000000000..2f860ee66--- /dev/null+++ b/src/kernels/cpu/tq_cpu_parity.cpp@@ -0,0 +1,565 @@+// src/kernels/cpu/tq_cpu_parity.cpp - the CPU trellis kernels (tq_cpu.hpp: TQ2_T / TQK6 / TQK7) against their scalar+// reference, against ggml-cpu's own vec_dot and to_float, and their speed. CPU only (no GPU runtime needed).+//+// tq_cpu_parity every check below for tqk6/tqk7 and tq2_t/tq2_t, plain and Hadamard+// tq_cpu_parity --synthetic GU/DOWN one expert pair (types by ggml name, e.g. tqk6/tqk7)+// tq_cpu_parity --synthetic-hadamard GU/DOWN the same with the block-128 Hadamard rotation of the activations+// tq_cpu_parity --bench [--quick] [--only K] the micro-benchmark (run it pinned: taskset -c <cores>);+// --only: the kernels whose name contains K+//+// Checks, per matrix (random stream bytes and scales, so every state is reached, and ggml-quantized Gaussian rows):+// (1) AVX2 and AVX-512 kernels == the scalar kernel, bit for bit, for groups of 1..8 tokens+// (2) a token's row in a group of k == the same token alone, bit for bit (no group-size dependence)+// (3) vs ggml-cpu's vec_dot on the same Q8_0 activations: the same integers summed in another float order, so+// ~1e-7 relative+// (4) vs ggml's to_float weights . the dequantized Q8_0 activations in double: the int16 state table's rounding+// (exact for |codebook| >= 2^-3, within 2^-14 below) plus float rounding, ~1e-6 relative+// and per expert (native_gu_rows -> native_quant_h -> native_down_rows, the engine's entry points, which now take+// these kernels) against the same expert computed with ggml-cpu's vec_dot: float rounding plus the odd Q8_0+// rounding flip of the intermediate.+#include "strata/kernels/cpu/tq_cpu.hpp"+#include "strata/kernels/cpu/native_expert.hpp"+#include "strata/kernels/cpu/expert.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"++#include "ggml.h"+#include "ggml-cpu.h"++#include <algorithm>+#include <atomic>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <functional>+#include <random>+#include <string>+#include <thread>+#include <vector>++using namespace strata::kernels::cpu;++namespace {++constexpr int H = 2560, FF = 640;++int type_by_name(const std::string& s) {+ for (int t = 0; t < GGML_TYPE_COUNT; ++t) {+ const char* n = ggml_type_name((ggml_type) t);+ if (n != nullptr && s == n) return t;+ }+ return -1;+}++uint16_t f2h(float f) { return ggml_fp32_to_fp16(f); }++// rows x n weights of `type`: random stream bytes and scales (every state reachable, both scale signs), and the+// last rows (up to 32, ggml's trellis encoder is slow) ggml's own encoding of Gaussian rows+std::vector<uint8_t> make_matrix(int type, int rows, int n, uint32_t seed) {+ const size_t rb = ggml_row_size((ggml_type) type, n);+ std::vector<uint8_t> m(rb * rows);+ std::mt19937 rng(seed);+ std::uniform_int_distribution<int> byte(0, 255);+ std::uniform_real_distribution<float> scale(0.002f, 0.05f);+ const size_t bsz = rb / (size_t) (n / 128);+ const int half = rows - std::min(rows / 2, 32);+ for (int r = 0; r < half; ++r)+ for (int b = 0; b < n / 128; ++b) {+ uint8_t* blk = m.data() + (size_t) r * rb + (size_t) b * bsz;+ const uint16_t d = f2h((byte(rng) & 1 ? -1.f : 1.f) * scale(rng));+ std::memcpy(blk, &d, 2);+ for (size_t j = 2; j < bsz; ++j) blk[j] = (uint8_t) byte(rng);+ }+ std::normal_distribution<float> nd(0.f, 0.02f);+ std::vector<float> wf((size_t) (rows - half) * n);+ for (auto& v : wf) v = nd(rng);+ ggml_quantize_chunk((ggml_type) type, wf.data(), m.data() + (size_t) half * rb, 0, rows - half, n, nullptr);+ return m;+}++std::vector<float> make_x(int n, uint32_t seed) {+ std::mt19937 rng(seed);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> x(n);+ for (auto& v : x) v = nd(rng);+ x[(size_t) (seed % (uint32_t) n)] = 40.f; // a massive channel, as the residual stream has+ return x;+}++std::vector<uint8_t> q8_0(const std::vector<float>& x) {+ std::vector<uint8_t> q(ggml_row_size(GGML_TYPE_Q8_0, (int64_t) x.size()));+ ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float(x.data(), q.data(), (int64_t) x.size());+ return q;+}++bool same_bits(const float* a, const float* b, size_t n) { return std::memcmp(a, b, n * sizeof(float)) == 0; }++// one matrix: checks (1) - (4); returns failures+int check_matrix(int type, int rows, int n, uint32_t seed, bool hadamard) {+ int fails = 0;+ const size_t rb = ggml_row_size((ggml_type) type, n);+ const auto m = make_matrix(type, rows, n, seed);+ constexpr int NT = kTqMaxTokens;+ std::vector<std::vector<uint8_t>> act(NT);+ std::vector<std::vector<float>> xr(NT);+ std::vector<float> signs(n);+ for (int i = 0; i < n; ++i) signs[i] = ((i * 2654435761u) >> 7) & 1 ? -1.f : 1.f;+ const void* ap[NT];+ for (int t = 0; t < NT; ++t) {+ xr[t] = make_x(n, seed * 31 + t);+ if (hadamard) hadamard_rotate(xr[t].data(), n, 128, signs.data());+ act[t] = q8_0(xr[t]);+ ap[t] = act[t].data();+ }+ auto run = [&](TqIsa isa, int nt, const void* const* a, std::vector<float>& o) {+ o.assign((size_t) nt * rows, 0.f);+ float* op[NT];+ for (int t = 0; t < nt; ++t) op[t] = o.data() + (size_t) t * rows;+ // two row ranges, as the pool splits them+ tq_rows(isa, type, m.data(), rb, n, a, nt, op, 0, rows / 3);+ tq_rows(isa, type, m.data(), rb, n, a, nt, op, rows / 3, rows);+ };+ std::vector<float> ref, got;+ run(TqIsa::scalar, NT, ap, ref);+ // (1) every ISA this CPU has, every group size, == scalar bit for bit; (2) group-size independence+ for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {+ if (!tq_isa_ok(isa)) { std::printf(" %-6s not on this CPU: skipped\n", tq_isa_name(isa)); continue; }+ bool ok = true;+ for (int nt = 1; nt <= NT; ++nt)+ for (int t0 = 0; t0 + nt <= NT; t0 += nt) {+ run(isa, nt, ap + t0, got);+ ok = ok && same_bits(got.data(), ref.data() + (size_t) t0 * rows, (size_t) nt * rows);+ }+ std::printf(" %-6s groups of 1..%d tokens vs the scalar kernel's 8-token run: %s\n", tq_isa_name(isa), NT,+ ok ? "bitwise equal" : "DIFFERENT");+ fails += !ok;+ }+ // (3) ggml-cpu's vec_dot, (4) to_float in double+ const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);+ const auto* tt = ggml_get_type_traits((ggml_type) type);+ const auto* q8t = ggml_get_type_traits(GGML_TYPE_Q8_0);+ double num3 = 0, num4 = 0, den = 0, max3 = 0;+ std::vector<float> wf(n), af(n);+ for (int t = 0; t < 2; ++t) {+ q8t->to_float(act[t].data(), af.data(), n);+ for (int r = 0; r < rows; ++r) {+ float g = 0;+ tc->vec_dot(n, &g, 0, m.data() + (size_t) r * rb, 0, act[t].data(), 0, 1);+ tt->to_float(m.data() + (size_t) r * rb, wf.data(), n);+ double d = 0, mag = 0;+ for (int i = 0; i < n; ++i) { d += (double) wf[i] * af[i]; mag += std::fabs((double) wf[i] * af[i]); }+ const double ours = ref[(size_t) t * rows + r];+ num3 += (ours - g) * (ours - g);+ num4 += (ours - d) * (ours - d);+ den += d * d;+ max3 = std::max(max3, std::fabs(ours - g) / (mag + 1e-30));+ }+ }+ const double r3 = std::sqrt(num3 / den), r4 = std::sqrt(num4 / den);+ const bool ok3 = r3 < 1e-5 && max3 < 1e-5, ok4 = r4 < 1e-4;+ std::printf(" vs ggml-cpu vec_dot: rel %.2e (worst row %.2e of sum|w a|) %s | vs to_float in double: rel %.2e %s\n",+ r3, max3, ok3 ? "ok" : "FAILED", r4, ok4 ? "ok" : "FAILED");+ return fails + !ok3 + !ok4;+}++// a whole expert through the engine's entry points vs the same expert on ggml-cpu's vec_dot+int check_expert(int gu, int dn, bool hadamard) {+ NativeFmt f;+ std::string err;+ if (!native_fmt(gu, dn, H, FF, f, err)) { std::printf("native_fmt: %s\n", err.c_str()); return 1; }+ std::vector<float> sh(H), sf(FF);+ for (int i = 0; i < H; ++i) sh[i] = (i * 7 % 3) ? 1.f : -1.f;+ for (int i = 0; i < FF; ++i) sf[i] = (i * 5 % 3) ? -1.f : 1.f;+ if (hadamard) {+ f.had_block = 128;+ f.had_gu = f.had_d = true;+ f.gu_signs = sh.data();+ f.d_signs = sf.data();+ }+ std::vector<uint8_t> blob(f.bytes);+ const auto g = make_matrix(gu, FF, H, 101), u = make_matrix(gu, FF, H, 202), d = make_matrix(dn, H, FF, 303);+ std::memcpy(blob.data(), g.data(), g.size());+ std::memcpy(blob.data() + f.up_off, u.data(), u.size());+ std::memcpy(blob.data() + f.down_off, d.data(), d.size());+ constexpr int NT = 3;+ double worst = 0;+ bool bitwise_group = true;+ std::vector<std::vector<float>> outs(NT, std::vector<float>(H));+ std::vector<std::vector<uint8_t>> acts(NT, std::vector<uint8_t>(kNativeActBytes));+ std::vector<std::vector<float>> ffs(NT, std::vector<float>(FF));+ std::vector<std::vector<uint8_t>> hqs(NT, std::vector<uint8_t>(kNativeHBytes));+ const void* a[NT];+ float* ffp[NT];+ const void* hp[NT];+ float* op[NT];+ for (int t = 0; t < NT; ++t) {+ const auto x = make_x(H, 7 + t);+ native_quant_act(f, x.data(), acts[t].data());+ a[t] = acts[t].data();+ ffp[t] = ffs[t].data();+ hp[t] = hqs[t].data();+ op[t] = outs[t].data();+ }+ native_gu_rows(f, blob.data(), a, NT, ffp, 0, FF);+ for (int t = 0; t < NT; ++t) native_quant_h(f, ffs[t].data(), hqs[t].data());+ native_down_rows(f, blob.data(), hp, NT, op, 0, H);+ const auto* tg = ggml_get_type_traits_cpu((ggml_type) gu);+ const auto* td = ggml_get_type_traits_cpu((ggml_type) dn);+ for (int t = 0; t < NT; ++t) {+ std::vector<float> ff(FF), out(H);+ for (int r = 0; r < FF; ++r) {+ float gg = 0, uu = 0;+ tg->vec_dot(H, &gg, 0, blob.data() + (size_t) r * f.gu_row, 0, a[t], 0, 1);+ tg->vec_dot(H, &uu, 0, blob.data() + f.up_off + (size_t) r * f.gu_row, 0, a[t], 0, 1);+ ff[r] = (gg / (1.f + std::exp(-gg))) * uu;+ }+ std::vector<uint8_t> hq(kNativeHBytes);+ native_quant_h(f, ff.data(), hq.data());+ for (int r = 0; r < H; ++r)+ td->vec_dot(FF, &out[r], 0, blob.data() + f.down_off + (size_t) r * f.d_row, 0, hq.data(), 0, 1);+ double num = 0, den = 0;+ for (int r = 0; r < H; ++r) { num += std::pow((double) outs[t][r] - out[r], 2); den += (double) out[r] * out[r]; }+ worst = std::max(worst, std::sqrt(num / den));+ // the same token alone through the engine path: the same bits as in the group of three+ std::vector<float> ff1(FF), o1(H);+ float* f1[1] = {ff1.data()};+ float* o1p[1] = {o1.data()};+ const void* a1[1] = {a[t]};+ native_gu_rows(f, blob.data(), a1, 1, f1, 0, FF);+ std::vector<uint8_t> h1(kNativeHBytes);+ native_quant_h(f, ff1.data(), h1.data());+ const void* h1p[1] = {h1.data()};+ native_down_rows(f, blob.data(), h1p, 1, o1p, 0, H);+ bitwise_group = bitwise_group && same_bits(o1.data(), outs[t].data(), H);+ }+ const bool ok = worst < 2e-3 && bitwise_group;+ std::printf(" expert %s/%s%s through native_gu_rows/native_down_rows (%s, %s): vs ggml-cpu vec_dot rel %.2e, "+ "alone vs in a group of %d: %s %s\n",+ ggml_type_name((ggml_type) gu), ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "",+ tq_kernels_enabled() ? "tq kernels" : "ggml-cpu", tq_isa_name(tq_isa()), worst, NT,+ bitwise_group ? "bitwise equal" : "DIFFERENT", ok ? "ok" : "FAILED");+ return ok ? 0 : 1;+}++int synthetic(int gu, int dn, bool hadamard) {+ int fails = 0;+ std::printf("%s/%s%s (tq kernels: %s, best ISA here %s)\n", ggml_type_name((ggml_type) gu),+ ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "", tq_kernels_enabled() ? "on" : "off",+ tq_isa_name(tq_isa()));+ std::printf(" %s, 96 rows of %d (gate/up shape)\n", ggml_type_name((ggml_type) gu), H);+ fails += check_matrix(gu, 96, H, 1000 + gu, hadamard);+ std::printf(" %s, 256 rows of %d (down shape)\n", ggml_type_name((ggml_type) dn), FF);+ fails += check_matrix(dn, 256, FF, 2000 + dn, hadamard);+ fails += check_expert(gu, dn, hadamard);+ return fails;+}++// ------------------------------------------------------------------------------------------------ the benchmark++double now_us() {+ return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now().time_since_epoch()).count();+}++struct BenchExpert {+ std::vector<uint8_t> gu, dn; // gate rows then up rows; down rows+ size_t gu_row, d_row;+};++// one variant: compute one expert (gate+up rows, then down rows) for `nt` tokens+using ExpertFn = std::function<void(const BenchExpert&, int nt)>;++// the core clock right now: a chain of dependent adds runs at one per cycle (GCC / Clang on x86-64; 0 elsewhere)+double core_ghz() {+#if defined(__GNUC__) && defined(__x86_64__)+ uint64_t x = 0;+ const long n = 1000000;+ const double t0 = now_us();+ for (long i = 0; i < n; ++i) __asm__ volatile("add $1, %0\n\tadd $1, %0\n\tadd $1, %0\n\tadd $1, %0" : "+r"(x));+ return 4.0 * n / ((now_us() - t0) * 1e3);+#else+ return 0.0;+#endif+}++constexpr double kBlocksPerExpert = 2.0 * FF * (H / 128) + (double) H * (FF / 128); // 128-weight blocks++// one slice of `ms` milliseconds cycling over the experts: ns per expert call and the clock around it (negated when+// the clock moved by more than 5% during the slice: such samples count only when a variant has no other)+std::pair<double, double> time_slice(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int nt, double ms,+ size_t& next) {+ const double g0 = core_ghz();+ int it = 0;+ const double t0 = now_us();+ double t1 = t0;+ do {+ fn(ex[next++ % ex.size()], nt);+ ++it;+ t1 = now_us();+ } while (t1 - t0 < ms * 1000.0 || it < 2);+ const double g1 = core_ghz();+ const double ghz = 0.5 * (g0 + g1);+ return {(t1 - t0) * 1e3 / it, ghz > 0 && std::fabs(g1 - g0) > 0.05 * g0 ? -ghz : ghz};+}++double median(std::vector<double> v) {+ if (v.empty()) return 0.0;+ std::sort(v.begin(), v.end());+ return v[v.size() / 2];+}++// `threads` threads, each cycling over its own experts: aggregate weight GB/s+double time_parallel(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int threads, double ms) {+ std::atomic<long> done{0};+ std::atomic<bool> stop{false};+ std::vector<std::thread> th;+ const double t0 = now_us();+ for (int k = 0; k < threads; ++k)+ th.emplace_back([&, k] {+ long c = 0;+ for (size_t i = (size_t) k; !stop.load(std::memory_order_relaxed); i += (size_t) threads, ++c)+ fn(ex[i % ex.size()], 1);+ done += c;+ });+ std::this_thread::sleep_for(std::chrono::milliseconds((int) ms));+ stop = true;+ for (auto& t : th) t.join();+ const double us = now_us() - t0;+ const double bytes = (double) (ex[0].gu_row * 2 * FF + ex[0].dn.size()) * (double) done.load();+ return bytes / (us * 1e3);+}++int bench(bool quick, const std::string& only) {+ const int ncold = quick ? 48 : 96; // ~50-130 MB of weights: past the last-level cache+ std::printf("CPU: %s | AVX-512 kernels: %s, AVX2: %s | ggml-cpu built for this CPU (-march=native unless "+ "STRATA_PORTABLE)\n", cpu_name().c_str(), cpu_avx512_ok() ? "yes" : "no", cpu_avx2_ok() ? "yes" : "no");+ std::printf("expert = gate+up %d x %d and down %d x %d of one type, %d distinct experts cycled (their weights come "+ "from DRAM, as a cache miss's do)\n\n", FF, H, H, FF, ncold);+ struct Variant { std::string name; int type; ExpertFn fn; bool ok; };+ std::vector<Variant> vars;+ auto add_tq = [&](int type) {+ for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {+ vars.push_back({std::string("strata ") + tq_isa_name(isa), type,+ [type, isa](const BenchExpert& e, int nt) {+ thread_local std::vector<uint8_t> a, h;+ thread_local std::vector<float> ff, out;+ a.resize(ggml_row_size(GGML_TYPE_Q8_0, H) * kTqMaxTokens);+ h.resize(ggml_row_size(GGML_TYPE_Q8_0, FF) * kTqMaxTokens);+ ff.resize((size_t) FF * kTqMaxTokens);+ out.resize((size_t) H * kTqMaxTokens);+ const void* ap[kTqMaxTokens];+ const void* hp[kTqMaxTokens];+ float* fp[kTqMaxTokens];+ float* op[kTqMaxTokens];+ for (int t = 0; t < nt; ++t) {+ ap[t] = a.data() + t * ggml_row_size(GGML_TYPE_Q8_0, H);+ hp[t] = h.data() + t * ggml_row_size(GGML_TYPE_Q8_0, FF);+ fp[t] = ff.data() + (size_t) t * FF;+ op[t] = out.data() + (size_t) t * H;+ }+ tq_gu_rows(isa, type, e.gu.data(), e.gu_row, e.gu_row * FF, H, ap, nt, fp, 0, FF);+ tq_rows(isa, type, e.dn.data(), e.d_row, FF, hp, nt, op, 0, H);+ },+ tq_isa_ok(isa)});+ }+ vars.push_back({"ggml-cpu vec_dot", type,+ [type](const BenchExpert& e, int nt) {+ const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);+ thread_local std::vector<uint8_t> a(ggml_row_size(GGML_TYPE_Q8_0, H)),+ h(ggml_row_size(GGML_TYPE_Q8_0, FF));+ thread_local std::vector<float> ff(FF), out(H);+ for (int t = 0; t < nt; ++t) {+ for (int r = 0; r < FF; ++r) {+ float g, u;+ tc->vec_dot(H, &g, 0, e.gu.data() + (size_t) r * e.gu_row, 0, a.data(), 0, 1);+ tc->vec_dot(H, &u, 0, e.gu.data() + (size_t) (FF + r) * e.gu_row, 0, a.data(), 0, 1);+ ff[r] = (g / (1.f + std::exp(-g))) * u;+ }+ for (int r = 0; r < H; ++r)+ tc->vec_dot(FF, &out[r], 0, e.dn.data() + (size_t) r * e.d_row, 0, h.data(), 0, 1);+ }+ },+ true});+ };+ auto add_q2 = [&](bool avx512) {+ vars.push_back({avx512 ? "strata Q2_0 avx512" : "strata Q2_0 avx2", 42,+ [avx512](const BenchExpert& e, int nt) {+ thread_local ActQ a1[MAXT], a2[MAXT];+ thread_local bool init = false;+ if (!init) { // real scales and codes (the kernel reads every field)+ const auto x = make_x(H, 3), h = make_x(FF, 4);+ for (int t = 0; t < MAXT; ++t) { act_quant_q8_1_avx2(x.data(), H, a1[t]); act_quant_q8_1_avx2(h.data(), FF, a2[t]); }+ init = true;+ }+ thread_local float g[MAXT][FF], u[MAXT][FF], out[MAXT][H];+ const ActQ* ap[MAXT];+ const ActQ* hp[MAXT];+ float* gp[MAXT];+ float* up[MAXT];+ float* op[MAXT];+ for (int t = 0; t < nt; ++t) { ap[t] = &a1[t]; hp[t] = &a2[t]; gp[t] = g[t]; up[t] = u[t]; op[t] = out[t]; }+ auto rows = avx512 ? q2_0_gguf_rows_multi : q2_0_gguf_rows_multi_avx2;+ rows(e.gu.data(), e.gu_row, H / 64, ap, nt, gp, 0, FF);+ rows(e.gu.data() + e.gu_row * FF, e.gu_row, H / 64, ap, nt, up, 0, FF);+ for (int t = 0; t < nt; ++t)+ for (int r = 0; r < FF; ++r) g[t][r] = (g[t][r] / (1.f + std::exp(-g[t][r]))) * u[t][r];+ rows(e.dn.data(), e.d_row, FF / 64, hp, nt, op, 0, H);+ },+ avx512 ? cpu_avx512_ok() : cpu_avx2_ok()});+ };+ for (int type : {145, 146, 144}) add_tq(type);+ add_q2(false);+ add_q2(true);+ // the Q2_0 pack's own AVX-512 kernel (the repacked 1,382,400-byte blob that setup makes on AVX-512 CPUs)+ vars.push_back({"strata Q2_0 avx512 pack", 42,+ [](const BenchExpert& e, int nt) {+ thread_local ActQ a1, a2;+ thread_local bool init = false;+ if (!init) {+ const auto x = make_x(H, 3), h = make_x(FF, 4);+ act_quant_q8_1(x.data(), H, a1);+ act_quant_q8_1(h.data(), FF, a2);+ init = true;+ }+ thread_local float ff[FF], out[H];+ for (int t = 0; t < nt; ++t) { // its multi-token kernels are the same per token+ s2_expert_gu_rows(e.gu.data(), a1, ff, 0, FF);+ s2_expert_down_rows(e.gu.data(), a2, out, 0, H);+ }+ },+ cpu_avx512_ok()});++ auto make_experts = [&](int type, int count) {+ std::vector<BenchExpert> v((size_t) count);+ for (int k = 0; k < count; ++k) {+ BenchExpert& e = v[(size_t) k];+ e.gu_row = ggml_row_size((ggml_type) type, H);+ e.d_row = ggml_row_size((ggml_type) type, FF);+ // (Q2_0: room for one repacked blob too, which the pack kernel reads from the same buffer)+ e.gu.resize(type == 42 ? std::max(e.gu_row * 2 * FF, (size_t) BLOB) : e.gu_row * 2 * FF);+ e.dn.resize(e.d_row * H);+ std::mt19937 rng(77 + k);+ for (auto& b : e.gu) b = (uint8_t) rng();+ for (auto& b : e.dn) b = (uint8_t) rng();+ // sane fp16 scales at every block start (Q2_0: 18-byte blocks, TQ: 26/30/34)+ const size_t bsz = type == 42 ? 18 : ggml_row_size((ggml_type) type, 128);+ const uint16_t d = f2h(0.01f);+ for (size_t o = 0; o + bsz <= e.gu.size(); o += bsz) std::memcpy(e.gu.data() + o, &d, 2);+ for (size_t o = 0; o + bsz <= e.dn.size(); o += bsz) std::memcpy(e.dn.data() + o, &d, 2);+ if (type == 42) { // the same bytes also hold one repacked Q2_0 blob (codes, then fp16 scales)+ for (size_t o = O_GU_SCALES; o + 2 <= BLOB; o += 2) std::memcpy(e.gu.data() + o, &d, 2);+ }+ }+ return v;+ };+ // Single thread, the variants of a type interleaved in short slices over several rounds (this machine's clock+ // moves; a slice whose clock moved is dropped), medians reported: ns and core cycles per 128-weight block, us+ // per expert, weight GB/s; and per token in a group of three (a verify window's tokens routed to one expert).+ // The activations are zero-filled (the kernels do not branch on values) except Q2_0's, which are real.+ const int rounds = quick ? 5 : 15;+ const double slice_ms = quick ? 40 : 60;+ std::printf("%-6s %-24s %9s %8s %8s %8s | %10s %8s\n", "type", "kernel", "us/expert", "ns/blk", "cyc/blk", "GB/s",+ "x3: us/tok", "cyc/blk");+ for (int type : {145, 146, 144, 42}) {+ const auto cold = make_experts(type, ncold);+ const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size()); // weight bytes per expert+ std::vector<const Variant*> vs;+ for (const auto& v : vars)+ if (v.type == type && v.name.find(only) != std::string::npos) {+ if (v.ok) vs.push_back(&v);+ else std::printf("%-6s %-24s (not on this CPU)\n", ggml_type_name((ggml_type) type), v.name.c_str());+ }+ std::vector<std::vector<double>> ns1(vs.size()), cy1(vs.size()), ns3(vs.size()), cy3(vs.size());+ std::vector<std::vector<double>> ns1m(vs.size()), cy1m(vs.size()), ns3m(vs.size()), cy3m(vs.size());+ size_t next = 0;+ for (auto* v : vs) v->fn(cold[0], 3); // warm-up: tables, buffers+ for (int r = 0; r < rounds; ++r)+ for (size_t k = 0; k < vs.size(); ++k) {+ const bool slow = vs[k]->name.find("scalar") != std::string::npos;+ const auto a = time_slice(cold, vs[k]->fn, 1, slow ? slice_ms / 2 : slice_ms, next);+ const auto b = time_slice(cold, vs[k]->fn, 3, slow ? slice_ms / 2 : slice_ms, next);+ if (a.second >= 0) { ns1[k].push_back(a.first); cy1[k].push_back(a.first * a.second); }+ else { ns1m[k].push_back(a.first); cy1m[k].push_back(-a.first * a.second); }+ if (b.second >= 0) { ns3[k].push_back(b.first); cy3[k].push_back(b.first * b.second); }+ else { ns3m[k].push_back(b.first); cy3m[k].push_back(-b.first * b.second); }+ }+ std::vector<size_t> steady(vs.size());+ for (size_t k = 0; k < vs.size(); ++k) {+ steady[k] = ns1[k].size();+ if (ns1[k].empty()) { ns1[k] = ns1m[k]; cy1[k] = cy1m[k]; }+ if (ns3[k].empty()) { ns3[k] = ns3m[k]; cy3[k] = cy3m[k]; }+ }+ for (size_t k = 0; k < vs.size(); ++k) {+ const double n1 = median(ns1[k]), c1 = median(cy1[k]), n3 = median(ns3[k]), c3 = median(cy3[k]);+ std::printf("%-6s %-24s %9.1f %8.1f %8.1f %8.2f | %10.1f %8.1f (%zu/%d steady)\n",+ ggml_type_name((ggml_type) type), vs[k]->name.c_str(), n1 / 1e3, n1 / kBlocksPerExpert,+ c1 / kBlocksPerExpert, eb / n1, n3 / 3e3, c3 / kBlocksPerExpert, steady[k], rounds);+ }+ std::fflush(stdout);+ }+ // several workers at once (as the pool runs a layer's misses): aggregate GB/s on the cold set+ const int nthr = (int) std::max(1u, std::min(4u, std::thread::hardware_concurrency()));+ std::printf("\n%d threads at once, cold experts, one token: aggregate weight GB/s (us per expert per thread)\n", nthr);+ for (int type : {145, 146, 144, 42}) {+ const auto cold = make_experts(type, ncold);+ const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size());+ std::vector<const Variant*> vs;+ for (const auto& v : vars)+ if (v.type == type && v.ok && v.name.find("scalar") == std::string::npos && v.name.find(only) != std::string::npos)+ vs.push_back(&v);+ std::vector<std::vector<double>> g(vs.size());+ for (int r = 0; r < (quick ? 2 : 4); ++r)+ for (size_t k = 0; k < vs.size(); ++k) g[k].push_back(time_parallel(cold, vs[k]->fn, nthr, quick ? 150 : 250));+ for (size_t k = 0; k < vs.size(); ++k) {+ const double gbs = median(g[k]);+ std::printf("%-6s %-24s %8.2f GB/s (%7.1f us per expert per thread)\n", ggml_type_name((ggml_type) type),+ vs[k]->name.c_str(), gbs, eb / (gbs * 1e3) * nthr);+ }+ std::fflush(stdout);+ }+ // the per-token Hadamard rotation of one layer's activation (once per token per layer, shared by its experts)+ {+ std::vector<float> x = make_x(H, 5), s(H, 1.f);+ const int it = 20000;+ const double t0 = now_us();+ for (int i = 0; i < it; ++i) hadamard_rotate(x.data(), H, 128, s.data());+ std::printf("\nHadamard rotation of one %d-wide activation (block 128): %.2f us\n", H, (now_us() - t0) / it);+ }+ return 0;+}++} // namespace++int main(int argc, char** argv) {+ ggml_cpu_init();+ if (GGML_TYPE_COUNT <= 146) {+ std::fprintf(stderr, "this ggml has no trellis types (build with STRATA_GGML_DIR = the agention llama.cpp fork)\n");+ return 1;+ }+ if (argc >= 2 && std::strcmp(argv[1], "--bench") == 0) { // --bench [--quick] [--only <kernel name part>]+ bool quick = false;+ std::string only;+ for (int i = 2; i < argc; ++i) {+ if (std::strcmp(argv[i], "--quick") == 0) quick = true;+ else if (std::strcmp(argv[i], "--only") == 0 && i + 1 < argc) only = argv[++i];+ }+ return bench(quick, only);+ }+ if (argc >= 3 && (std::strcmp(argv[1], "--synthetic") == 0 || std::strcmp(argv[1], "--synthetic-hadamard") == 0)) {+ const std::string pair = argv[2];+ const size_t sl = pair.find('/');+ const int gu = type_by_name(pair.substr(0, sl)), dn = sl == std::string::npos ? gu : type_by_name(pair.substr(sl + 1));+ if (!tq_supported(gu) || !tq_supported(dn)) { std::fprintf(stderr, "not a trellis pair: %s\n", argv[2]); return 2; }+ const int f = synthetic(gu, dn, std::strcmp(argv[1], "--synthetic-hadamard") == 0);+ std::printf("%d failures\n", f);+ return f ? 1 : 0;+ }+ int f = 0;+ for (bool had : {false, true}) {+ f += synthetic(145, 146, had); // Gyro-S+ f += synthetic(144, 144, had); // Gyro-M+ }+ std::printf("%d failures\n", f);+ return f ? 1 : 0;+}diff --git a/src/kernels/cpu/tq_states.hpp b/src/kernels/cpu/tq_states.hppnew file mode 100644index 000000000..bb4eaa7a1--- /dev/null+++ b/src/kernels/cpu/tq_states.hpp@@ -0,0 +1,88 @@+// src/kernels/cpu/tq_states.hpp - the 32 trellis states of one TQ2_T / TQK6 / TQK7 block in plain C++ (all three+// kernels: scalar code beat byte-shuffle versions on AVX2 and AVX-512). Internal to src/kernels/cpu/tq_*.cpp.+//+// TQ2_T (K = 8): state t = qs[(t + 31) % 32] << 8 | qs[t], masked to 15 bits.+// TQK (K = 6, 7): the 16 stream bits from bit (31 - t)*K, LSB first, circular modulo 32*K bits, masked to 15 bits.+// Every window is one unaligned 32-bit little-endian read inside the block's 4*K bytes, except the windows that run+// past the end (t = 0 and 1), which read a 64-bit view of the last four bytes followed by the first four.+#pragma once++#include <cstdint>+#include <cstring>++namespace strata::kernels::cpu::tq_detail {++inline constexpr uint32_t kTqStateMask = 0x7fffu;++inline uint32_t tq_ld32(const uint8_t* p) {+ uint32_t v;+ std::memcpy(&v, p, 4);+ return v; // x86 (and every target Strata builds for) is little-endian+}++/// State t alone (t a constant after unrolling); `wrap` = tq_wrap<KB>(qs) for the TQK windows past the end.+template <int KB>+inline uint64_t tq_wrap(const uint8_t* qs) {+ if constexpr (KB == 8) return 0;+ else return (uint64_t) tq_ld32(qs + 4 * KB - 4) | ((uint64_t) tq_ld32(qs) << 32);+}+template <int KB>+inline uint32_t tq_state(const uint8_t* qs, uint64_t wrap, int t) {+ if constexpr (KB == 8) {+ return (((uint32_t) qs[(t + 31) & 31] << 8) | qs[t]) & kTqStateMask;+ } else {+ constexpr int NB = 4 * KB;+ const int off = (31 - t) * KB, b = off >> 3;+ const uint32_t v = b + 4 <= NB ? tq_ld32(qs + b) >> (off & 7) : (uint32_t) (wrap >> (off - 8 * (NB - 4)));+ return v & kTqStateMask;+ }+}++template <int KB>+inline void tq_states(const uint8_t* qs, uint32_t* st) {+ if constexpr (KB == 8) {+ st[0] = (((uint32_t) qs[31] << 8) | qs[0]) & kTqStateMask;+#if defined(__GNUC__)+#pragma GCC unroll 31+#endif+ for (int t = 1; t < 32; ++t) st[t] = (((uint32_t) qs[t - 1] << 8) | qs[t]) & kTqStateMask;+ } else {+ constexpr int NB = 4 * KB;+ const uint64_t wrap = (uint64_t) tq_ld32(qs + NB - 4) | ((uint64_t) tq_ld32(qs) << 32);+#if defined(__GNUC__)+#pragma GCC unroll 32+#endif+ for (int t = 0; t < 32; ++t) {+ const int off = (31 - t) * KB, b = off >> 3;+ const uint32_t v = b + 4 <= NB ? tq_ld32(qs + b) >> (off & 7) : (uint32_t) (wrap >> (off - 8 * (NB - 4)));+ st[t] = v & kTqStateMask;+ }+ }+}++/// fp16 bits -> float, exact (the block scales and the Q8_0 activation scales).+inline float tq_half_to_float(uint16_t h) {+ const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;+ uint32_t exp = (h >> 10) & 0x1fu, man = h & 0x3ffu, bits;+ if (exp == 0x1fu) {+ bits = sign | 0x7f800000u | (man << 13);+ } else if (exp != 0) {+ bits = sign | ((exp + 112u) << 23) | (man << 13);+ } else if (man == 0) {+ bits = sign;+ } else { // subnormal: man * 2^-24, exact in float+ const float m = (float) man * (1.0f / 16777216.0f);+ return sign ? -m : m;+ }+ float f;+ std::memcpy(&f, &bits, 4);+ return f;+}++inline uint16_t tq_ld16(const uint8_t* p) {+ uint16_t v;+ std::memcpy(&v, p, 2);+ return v;+}++} // namespace strata::kernels::cpu::tq_detaildiff --git a/third_party/ggml/ggml-common-tq.h b/third_party/ggml/ggml-common-tq.hindex a1cf9d853..1ce3773bf 100644--- a/third_party/ggml/ggml-common-tq.h+++ b/third_party/ggml/ggml-common-tq.h@@ -1,7 +1,7 @@// third_party/ggml/ggml-common-tq.h - the trellis (APR) expert formats of Agention's llama.cpp fork// (agentionai/llama.cpp 073bc9e68, ggml/src/ggml-common.h; MIT, the same license as the rest of third_party/ggml).-// The block structs and the 2048-point "Hyb" codebook, unchanged, for Strata's device kernels. ggml-cpu takes-// them from the fork itself (-DSTRATA_GGML_DIR).+// The block structs and the 2048-point "Hyb" codebook, unchanged, for Strata's device kernels and its CPU trellis+// kernels (src/kernels/cpu/tq_*.cpp). ggml-cpu takes them from the fork itself (-DSTRATA_GGML_DIR).//// TQ2_T / TQK6 / TQK7: 128 weights per block, {fp16 d; qs}. qs is 32 trellis steps of K bits (K = 8 / 6 / 7;// TQ2_T byte-aligned, TQK bit-packed LSB-first, circular, step t's 16-bit window at bit (31 - t)*K; TQ2_T's state@@ -32,9 +32,14 @@ typedef struct {} block_tqk7;static_assert(sizeof(block_tqk7) == 30, "wrong tqk7 block size/padding");+// device memory in a CUDA / HIP translation unit, ordinary host data everywhere else#if defined(__CUDACC__) || defined(__HIPCC__)+#define STRATA_TQ_LUT_SPACE __device__+#else+#define STRATA_TQ_LUT_SPACE+#endif// point i = (tq_lut_f16[2i], tq_lut_f16[2i+1]) as fp16 bit patterns; 16-byte aligned so a point reads as one u32-alignas(16) static const __device__ uint16_t tq_lut_f16[4096] = {+alignas(16) static const STRATA_TQ_LUT_SPACE uint16_t tq_lut_f16[4096] = {0xbd4d, 0xb8f7, 0xb6ce, 0x3cd5, 0xbd9e, 0x4368, 0x3dce, 0xb8a6, 0x395e, 0xb430, 0x3bc1, 0x3dbc, 0xb92b, 0x3c9c, 0xbca2, 0x3aaa,0x3cdb, 0xb8e1, 0xbb33, 0x3dfb, 0x3a8d, 0xbb18, 0xbd64, 0x3da0, 0xbf1a, 0x3b83, 0xb389, 0x3659, 0xb40f, 0xabd7, 0x40ee, 0x35a2,0xc057, 0x3972, 0xbe17, 0x3996, 0xb926, 0xb9b5, 0x393b, 0x3c75, 0x4151, 0xbdef, 0xb5a3, 0xbd09, 0xb408, 0x4012, 0x3b7f, 0x3ece,@@ -292,4 +297,3 @@ alignas(16) static const __device__ uint16_t tq_lut_f16[4096] = {0xa11a, 0xc068, 0x35bb, 0xb82e, 0x3f77, 0xbbb9, 0xb386, 0x3b46, 0xb675, 0x22f3, 0x3e8b, 0xbcb2, 0xb8ba, 0xba29, 0x4235, 0xbd08,0x1038, 0x3f9f, 0xade8, 0x399f, 0x3363, 0xb9da, 0x3909, 0x37c8, 0x3c82, 0xbc4c, 0xbc3a, 0xb788, 0xa897, 0x39da, 0xb6d2, 0x3bee,};-#endif