Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
native experts: window-invariant CPU arithmetic for STRATA_SPEC_PROB decode, Default dispatch unchanged (#3416)
Keyan Hu499e963593f4
diff --git a/CMakeLists.txt b/CMakeLists.txtindex a5fb2b6f8..008d8b505 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1834,6 +1834,21 @@ if(STRATA_NATIVE_EXPERTS)add_executable(router_dot_parity src/kernels/cpu/router_dot_parity.cpp)target_link_libraries(router_dot_parity PRIVATE strata_kernels_cpu)if(STRATA_BUILD_TESTS)+ # a token's native expert rows are the same bits at every number of tokens that share its expert (window-invariant arithmetic)+ add_executable(native_arithmetic_parity src/kernels/cpu/native_arithmetic_parity.cpp)+ target_link_libraries(native_arithmetic_parity PRIVATE strata_kernels_cpu)+ add_test(NAME native_arithmetic_parity COMMAND native_arithmetic_parity)+ add_test(NAME native_arithmetic_parity_avx COMMAND ${CMAKE_COMMAND} -E env+ STRATA_FORCE_ISA=avx $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_kq256 COMMAND ${CMAKE_COMMAND} -E env+ STRATA_KQ256=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_MT_MIN=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_no512 COMMAND ${CMAKE_COMMAND} -E env STRATA_NO_IQ512=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_ggml COMMAND ${CMAKE_COMMAND} -E env STRATA_NO_IQ512=1 STRATA_NO_IQ256=1 STRATA_NO_IQ4NL=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_nvfp4_avx2 COMMAND ${CMAKE_COMMAND} -E env+ STRATA_NO_NVFP4_512=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_nvfp4_ggml COMMAND ${CMAKE_COMMAND} -E env+ STRATA_NO_NVFP4_512=1 STRATA_NO_NVFP4_256=1 $<TARGET_FILE:native_arithmetic_parity>)add_executable(pool_tasks_test src/kernels/cpu/pool_tasks_test.cpp)target_link_libraries(pool_tasks_test PRIVATE strata_kernels_cpu)add_test(NAME pool_tasks_test COMMAND pool_tasks_test)diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex 32c37e30f..ef2e2ec7c 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -518,6 +518,12 @@ applies at the next start. Network, key, MCP and program keys are not editable tcorrectness change, not a speed change. `spec_admission_test` enumerates the rule exactly for a two-token vocabulary,`gated_proposal_test` checks the proposal packet and `spec_proposal_parity` checks the GPU verifier against a CPUreference. A model-level distribution run is not published yet.+- With `STRATA_SPEC_PROB=1` the CPU experts of a native (IQ) pack compute a token's rows the same way whatever the window+ width: the grouped kernels that round differently from the one-token result are replaced by the one-token arithmetic,+ and the grouped kernels that already match it stay. Prefill and every other decode keep their arithmetic.+ `native_arithmetic_parity` checks every group width from 1 to 8 for the supported formats, bit for bit. Comparing other+ decode modes against each other still needs one target evaluator; `STRATA_IQ_MT_MIN=1` selects the grouped IQ arithmetic+ at every width and also changes prefill, so keep it the same in both arms.- **Non-finite logits end the request and drop the cached state (on by default).** If a verify window's logits are NaN, thewindow emits no tokens, the request ends, and the next request drops the live session, checkpoints and parkedconversations and reads its prompt from token 0. The server retries a request that had sent nothing once, transparently.diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex 3ab77dc2a..256f6bd8a 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -671,7 +671,7 @@ int tail_skip_rank();void set_tail_skip_rank(int r);void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k,- float* out);+ float* out, strata::kernels::cpu::NativeArithmetic arithmetic = strata::kernels::cpu::NativeArithmetic::Default);/// **THE HITS, LAUNCHED AFTER THE MISSES ARE STAGED AND BEFORE `post[l]`.** Same shape as `PoolFn` and for the/// same reason: `session_loop` owns the ORDER and this owns the work, so the loop needs to know nothing aboutdiff --git a/include/strata/kernels/cpu/iq_avx2.hpp b/include/strata/kernels/cpu/iq_avx2.hppindex ca5fa2584..59731bec8 100644--- a/include/strata/kernels/cpu/iq_avx2.hpp+++ b/include/strata/kernels/cpu/iq_avx2.hpp@@ -56,4 +56,8 @@ void iq4nl256_down_rows(const uint8_t* w, size_t row_bytes, int n, const void* cvoid iq4nl256_down_rows_v(int variant, const uint8_t* w, size_t row_bytes, int n, const void* const* hq, int nt,float* const* out, int r0, int r1);+// Singleton-equivalent IQ4_NL down rows; caller checks the linked ggml AVX2/FMA path.+void iq4nl_reference_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* hq,+ int nt, float* const* out, int r0, int r1);+} // namespace strata::kernels::cpudiff --git a/include/strata/kernels/cpu/iq_avx512.hpp b/include/strata/kernels/cpu/iq_avx512.hppindex 3f51b9dee..df505f0a9 100644--- a/include/strata/kernels/cpu/iq_avx512.hpp+++ b/include/strata/kernels/cpu/iq_avx512.hpp@@ -25,4 +25,9 @@ bool iq512x_supported(int ggml_type) noexcept;void iq512x_rows(int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,float* const* out, int r0, int r1);+/// Singleton-equivalent accumulation for types 18, 21 and 22 (the linked ggml AVX2 order, one token or many); the caller+/// checks the linked ggml ISA. Used only for window-invariant native arithmetic.+void iq512_reference_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,+ const void* const* act, int nt, float* const* ff, int r0, int r1);+} // namespace strata::kernels::cpudiff --git a/include/strata/kernels/cpu/native_expert.hpp b/include/strata/kernels/cpu/native_expert.hppindex e0cad9065..2d2e43e7c 100644--- a/include/strata/kernels/cpu/native_expert.hpp+++ b/include/strata/kernels/cpu/native_expert.hpp@@ -2,9 +2,9 @@//// The IQ2_XS / IQ3_XXS model files keep their experts in i-quant formats (IQ1_M, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS,// IQ3_S gate/up; Q2_0 or IQ4_NL down) whose values cannot be re-expressed in the Q2_0 pack form. A native expert-// blob is the three GGUF slices back to back, [gate rows | up rows | down rows], and the arithmetic is ggml-cpu's-// own (`ggml_get_type_traits_cpu`): the activation is quantized with the weight type's `vec_dot_type` and each row-// is one `vec_dot`, exactly what llama.cpp's CPU backend computes for the same tensor.+// blob is the three GGUF slices back to back, [gate rows | up rows | down rows]. Activations use the weight type's+// ggml `vec_dot_type`; dispatch selects a format-specific kernel or ggml's per-token dot. The arithmetic mode+// controls whether grouping tokens may change the selected row arithmetic.#pragma once#include <cstddef>@@ -18,6 +18,9 @@ inline constexpr size_t kNativeActBytes = 4096;/// Bytes of the largest quantized down activation (640 values as Q8_0: 20 x 34, or Q8_K 3 x 292).inline constexpr size_t kNativeHBytes = 1024;+/// Arithmetic fixed for one pool batch. WindowInvariant preserves the legacy singleton result at every width.+enum class NativeArithmetic { Default, WindowInvariant };+/// One layer's native expert geometry.struct NativeFmt {int gu_type = -1, d_type = -1; ///< ggml types of gate/up and of down@@ -63,4 +66,11 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt, float* const* out,int r0, int r1);+/// Same per-token result as the corresponding legacy nt=1 call on this build and CPU.+/// Kernels that already satisfy this contract retain their grouped execution.+void native_gu_rows_invariant(const NativeFmt& f, const uint8_t* blob, const void* const* act, int nt,+ float* const* ff, int r0, int r1);+void native_down_rows_invariant(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt,+ float* const* out, int r0, int r1);+} // namespace strata::kernels::cpudiff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hppindex 6c21d2d21..64c4a6cd2 100644--- a/include/strata/kernels/cpu/pool.hpp+++ b/include/strata/kernels/cpu/pool.hpp@@ -247,7 +247,8 @@ public:/// read once for all of its tokens.void run_split_multi(ExpertJobMulti* jobs, int n);/// Plan v0.3 P6: the same for a native pack's layer (ggml-cpu arithmetic, `nact` activations).- void run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs, int n);+ void run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs, int n,+ NativeArithmetic arithmetic = NativeArithmetic::Default);static constexpr int kMaxSplitMulti = 96;/// run_split_multi's phases, accumulated ms: gate/up rows, the intermediate quantization, down rows.double ms_multi_gu = 0, ms_multi_q = 0, ms_multi_down = 0;@@ -366,6 +367,7 @@ private:alignas(64) uint8_t hq[MAXT][kNativeHBytes]; // plan v0.3 P6: native down activations};const NativeFmt* nfmt_ = nullptr;+ NativeArithmetic narithmetic_ = NativeArithmetic::Default;std::vector<SplitBufMulti> split_multi_;// STRATA_NUMA_SPLIT=1 (opt-in, Linux, exactly 2 nodes): in the native row phases (modes 5/6) every expert's rows// are cut into one slice per NUMA node (slice k = rows [per*k/2, per*(k+1)/2) of the gate/up or down matrix, thediff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex b8ad8fb42..d4ff377be 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -4777,7 +4777,7 @@ int tail_skip_rank() {void set_tail_skip_rank(int r) { g_tail_skip_rank.store(r > 0 && r < 10 ? r : 0, std::memory_order_relaxed); }void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k,- float* out) {+ float* out, strata::kernels::cpu::NativeArithmetic arithmetic) {using namespace strata::kernels::cpu;if (d.failed) return;if (n_tok < 1 || n_tok > MAXT) {@@ -5130,7 +5130,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32{const auto b0 = std::chrono::steady_clock::now();if (njobs > 0) {- if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);+ if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs, arithmetic);else d.pool->run_split_multi(d.jobs_multi.data(), njobs);++d.cold_batches;}@@ -5175,7 +5175,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32if (d.plan != nullptr) d.plan->cpu_jobs = njobs;pt("run", njobs);if (njobs > 0 && npend == 0) {- if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);+ if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs, arithmetic);else d.pool->run_split_multi(d.jobs_multi.data(), njobs);}if (d.remote_count > 0) {diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cppindex 9d296c976..ee1730b6d 100644--- a/src/kernels/cpu/expert_layout.cpp+++ b/src/kernels/cpu/expert_layout.cpp@@ -383,6 +383,8 @@ void native_quant_h(const NativeFmt&, const float*, void*) { std::abort(); }int native_gu_mt_min(int) { return 2; }void native_gu_rows(const NativeFmt&, const uint8_t*, const void* const*, int, float* const*, int, int) { std::abort(); }void native_down_rows(const NativeFmt&, const uint8_t*, const void* const*, int, float* const*, int, int) { std::abort(); }+void native_gu_rows_invariant(const NativeFmt&, const uint8_t*, const void* const*, int, float* const*, int, int) { std::abort(); }+void native_down_rows_invariant(const NativeFmt&, const uint8_t*, const void* const*, int, float* const*, int, int) { std::abort(); }#endifbool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err) {diff --git a/src/kernels/cpu/iq_avx2.cpp b/src/kernels/cpu/iq_avx2.cppindex 0ab33d211..9aad0725b 100644--- a/src/kernels/cpu/iq_avx2.cpp+++ b/src/kernels/cpu/iq_avx2.cpp@@ -571,4 +571,93 @@ void iq4nl256_down_rows(const uint8_t* w, size_t row_bytes, int n, const void* ciq4nl256_down_rows_v(vnni_on() ? kIq256Vnni : 0, w, row_bytes, n, hq, nt, out, r0, r1);}-} // namespace strata::kernels::cpu++// Singleton-equivalent down accumulation; legacy/VNNI kernels above keep their order.+namespace {+namespace singleton_iq4nl {++float h2f(uint16_t x) {+ return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)));+}+float hsum(__m256 x) {+ __m128 s = _mm256_extractf128_ps(x, 1);+ s = _mm_add_ps(s, _mm256_castps256_ps128(x));+ s = _mm_add_ps(s, _mm_movehl_ps(s, s));+ s = _mm_add_ss(s, _mm_movehdup_ps(s));+ return _mm_cvtss_f32(s);+}+template <int NT>+void rows(const uint8_t* w, size_t row_bytes, int n, const void* const* hq, float* const* out, int r0, int r1) {+ const __m128i values = _mm_loadu_si128((const __m128i*)kvalues_iq4nl), mask = _mm_set1_epi8(15);+ const __m256i ones = _mm256_set1_epi16(1);+ const int nb = n / QK4_NL;+ for (int r = r0; r < r1; ++r) {+ auto* x = (const block_iq4_nl*)(w + r * row_bytes);+ __m256 even[NT], odd[NT];+ for (int t = 0; t < NT; ++t) {+ even[t] = _mm256_setzero_ps();+ odd[t] = _mm256_setzero_ps();+ }+ int ib = 0;+ for (; ib + 1 < nb; ib += 2) {+ const __m128i b0 = _mm_loadu_si128((const __m128i*)x[ib].qs),+ b1 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);+ const __m256i w0 =+ _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(values, _mm_and_si128(b0, mask))),+ _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(b0, 4), mask)), 1);+ const __m256i w1 =+ _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(values, _mm_and_si128(b1, mask))),+ _mm_shuffle_epi8(values, _mm_and_si128(_mm_srli_epi16(b1, 4), mask)), 1);+ const __m256i a0 = _mm256_sign_epi8(w0, w0), a1 = _mm256_sign_epi8(w1, w1);+ const float d0 = h2f(x[ib].d), d1 = h2f(x[ib + 1].d);+ for (int t = 0; t < NT; ++t) {+ auto* y = (const block_q8_0*)hq[t];+ auto p0 = _mm256_madd_epi16(+ _mm256_maddubs_epi16(a0, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)y[ib].qs), w0)), ones);+ auto p1 = _mm256_madd_epi16(+ _mm256_maddubs_epi16(a1, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*)y[ib + 1].qs), w1)),+ ones);+ even[t] = _mm256_fmadd_ps(_mm256_set1_ps(h2f(y[ib].d) * d0), _mm256_cvtepi32_ps(p0), even[t]);+ odd[t] = _mm256_fmadd_ps(_mm256_set1_ps(h2f(y[ib + 1].d) * d1), _mm256_cvtepi32_ps(p1), odd[t]);+ }+ }+ for (int t = 0; t < NT; ++t) {+ float s = hsum(_mm256_add_ps(even[t], odd[t]));+ if (ib < nb) {+ auto* y = (const block_q8_0*)hq[t];+ int a = 0, b = 0;+ for (int j = 0; j < QK4_NL / 2; ++j) {+ a += kvalues_iq4nl[x[ib].qs[j] & 15] * y[ib].qs[j];+ b += kvalues_iq4nl[x[ib].qs[j] >> 4] * y[ib].qs[j + QK4_NL / 2];+ }+ const float d = h2f(y[ib].d) * h2f(x[ib].d);+ s += d * (a + b);+ }+ out[t][r] = s;+ }+ }+}+} // namespace singleton_iq4nl+} // namespace+void iq4nl_reference_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* hq, int nt, float* const* out,+ int r0, int r1) {+ for (int at = 0; at < nt; at += 4) {+ int k = nt - at < 4 ? nt - at : 4;+ switch (k) {+ case 1:+ singleton_iq4nl::rows<1>(w, row_bytes, n, hq + at, out + at, r0, r1);+ break;+ case 2:+ singleton_iq4nl::rows<2>(w, row_bytes, n, hq + at, out + at, r0, r1);+ break;+ case 3:+ singleton_iq4nl::rows<3>(w, row_bytes, n, hq + at, out + at, r0, r1);+ break;+ case 4:+ singleton_iq4nl::rows<4>(w, row_bytes, n, hq + at, out + at, r0, r1);+ break;+ }+ }+}++} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cppindex 444c64d28..9ea01bf7c 100644--- a/src/kernels/cpu/iq_avx512.cpp+++ b/src/kernels/cpu/iq_avx512.cpp@@ -173,10 +173,19 @@ template <> struct Fmt<21> { // IQ3_S: d, qs[64], qh[8], signs[32], scales[4]}};-template <int TY, int NT>+template <bool Singleton> struct RowAccumulator;+template <> struct RowAccumulator<false> { using Type = __m512; };+template <> struct RowAccumulator<true> { using Type = __m256; };++template <int TY, int NT, bool Singleton = false>inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y, float* res) {- __m512 accf[NT];- for (int t = 0; t < NT; ++t) accf[t] = _mm512_setzero_ps();+ // Decode weights once for the group; select the accumulation order at compile time.+ using Acc = typename RowAccumulator<Singleton>::Type;+ Acc accf[NT];+ for (int t = 0; t < NT; ++t) {+ if constexpr (Singleton) accf[t] = _mm256_setzero_ps();+ else accf[t] = _mm512_setzero_ps();+ }const __m512i zero = _mm512_setzero_si512();for (int i = 0; i < nblocks; ++i) {const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;@@ -199,14 +208,35 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,acci[t] = _mm512_add_epi32(acci[t], _mm512_madd_epi16(_mm512_maddubs_epi16(g, ys), sc));}}- const float dx = h2f(u16(blk)) * Fmt<TY>::K;- for (int t = 0; t < NT; ++t)- accf[t] = _mm512_fmadd_ps(_mm512_set1_ps(dx * y[t][i].d), _mm512_cvtepi32_ps(acci[t]), accf[t]);+ if constexpr (Singleton) {+ const float dx = h2f(u16(blk));+ for (int t = 0; t < NT; ++t) {+ // Preserve ggml AVX2's eight integer lanes BEFORE floating conversion.+ const __m256i folded = _mm256_add_epi32(_mm512_castsi512_si256(acci[t]),+ _mm512_extracti64x4_epi64(acci[t], 1));+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d),+ _mm256_cvtepi32_ps(folded), accf[t]);+ }+ } else {+ const float dx = h2f(u16(blk)) * Fmt<TY>::K;+ for (int t = 0; t < NT; ++t)+ accf[t] = _mm512_fmadd_ps(_mm512_set1_ps(dx * y[t][i].d), _mm512_cvtepi32_ps(acci[t]), accf[t]);+ }+ }+ if constexpr (Singleton) {+ for (int t = 0; t < NT; ++t) {+ __m128 sum = _mm256_extractf128_ps(accf[t], 1);+ sum = _mm_add_ps(sum, _mm256_castps256_ps128(accf[t]));+ sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum));+ sum = _mm_add_ss(sum, _mm_movehdup_ps(sum));+ res[t] = Fmt<TY>::K * _mm_cvtss_f32(sum);+ }+ } else {+ for (int t = 0; t < NT; ++t) res[t] = _mm512_reduce_add_ps(accf[t]);}- for (int t = 0; t < NT; ++t) res[t] = _mm512_reduce_add_ps(accf[t]);}-template <int TY, int NT>+template <int TY, int NT, bool Singleton = false>void gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act, float* const* ff,int r0, int r1) {const block_q8_K* y[NT];@@ -214,8 +244,8 @@ void gu_rows(const uint8_t* blob, size_t gu_row, size_t up_off, int n, const voiconst int nb = n / QK_K;float g[NT], u[NT];for (int r = r0; r < r1; ++r) {- row_dot<TY, NT>(blob + (size_t) r * gu_row, nb, y, g);- row_dot<TY, NT>(blob + up_off + (size_t) r * gu_row, nb, y, u);+ row_dot<TY, NT, Singleton>(blob + (size_t) r * gu_row, nb, y, g);+ row_dot<TY, NT, Singleton>(blob + up_off + (size_t) r * gu_row, nb, y, u);for (int t = 0; t < NT; ++t) ff[t][r] = (g[t] / (1.f + std::exp(-g[t]))) * u[t];}}@@ -231,18 +261,18 @@ void dot_rows(const uint8_t* w, size_t row_bytes, int n, const void* const* act,}}-template <int TY>+template <int TY, bool Singleton = false>void gu_rows_nt(int nt, const uint8_t* blob, size_t gu_row, size_t up_off, int n, const void* const* act,float* const* ff, int r0, int r1) {switch (nt) {- case 1: gu_rows<TY, 1>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 2: gu_rows<TY, 2>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 3: gu_rows<TY, 3>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 4: gu_rows<TY, 4>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 5: gu_rows<TY, 5>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 6: gu_rows<TY, 6>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- case 7: gu_rows<TY, 7>(blob, gu_row, up_off, n, act, ff, r0, r1); break;- default: gu_rows<TY, 8>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 1: gu_rows<TY, 1, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 2: gu_rows<TY, 2, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 3: gu_rows<TY, 3, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 4: gu_rows<TY, 4, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 5: gu_rows<TY, 5, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 6: gu_rows<TY, 6, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 7: gu_rows<TY, 7, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;+ default: gu_rows<TY, 8, Singleton>(blob, gu_row, up_off, n, act, ff, r0, r1); break;}}@@ -571,6 +601,16 @@ void iq512_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off,}}+// The linked ggml AVX2 singleton order, for the formats validated against its production dot.+void iq512_reference_gu_rows(int type, const uint8_t* blob, size_t gu_row, size_t up_off, int n,+ const void* const* act, int nt, float* const* ff, int r0, int r1) {+ switch (type) {+ case 18: gu_rows_nt<18, true>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 21: gu_rows_nt<21, true>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;+ case 22: gu_rows_nt<22, true>(nt, blob, gu_row, up_off, n, act, ff, r0, r1); break;+ }+}+void iq512_rows(int type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt, float* const* out,int r0, int r1) {switch (type) {diff --git a/src/kernels/cpu/native_arithmetic_parity.cpp b/src/kernels/cpu/native_arithmetic_parity.cppnew file mode 100644index 000000000..04a30216f--- /dev/null+++ b/src/kernels/cpu/native_arithmetic_parity.cpp@@ -0,0 +1,128 @@+// A token's native expert result must not depend on how many other tokens share its group.+// Compare the invariant entry points with the production singleton dispatch, including SwiGLU.+// Synthetic packed blocks only: no GPU, model download or private fixtures.+#include "ggml.h"+#include "strata/kernels/cpu/native_expert.hpp"+#include "strata/kernels/cpu/tq_cpu.hpp"+#include <cmath>+#include <cstdio>+#include <cstring>+#include <random>+#include <vector>+#include <utility>++namespace cpu = strata::kernels::cpu;+namespace {+constexpr int width = 8, hidden = 512;+size_t checked = 0;+bool equal(const std::vector<float> &actual, const std::vector<float> &expected) {+ for (size_t i = 0; i < actual.size(); ++i) {+ ++checked;+ if (!std::isfinite(actual[i]) || std::memcmp(&actual[i], &expected[i], sizeof(float)))+ return false;+ }+ return true;+}++bool check_format(int gu, int down, int ff, bool folded = false) {+ cpu::NativeFmt fmt;+ std::string error;+ if (!cpu::native_fmt(gu, down, hidden, ff, fmt, error)) {+ std::fprintf(stderr, "%s\n", error.c_str());+ return false;+ }+ std::vector<float> gu_signs(hidden), down_signs(ff);+ if (folded) {+ for (int i = 0; i < hidden; ++i) gu_signs[i] = i % 3 ? 1.f : -1.f;+ for (int i = 0; i < ff; ++i) down_signs[i] = i % 5 ? 1.f : -1.f;+ fmt.had_block = 128;+ fmt.had_gu = fmt.had_d = true;+ fmt.gu_signs = gu_signs.data();+ fmt.d_signs = down_signs.data();+ }+ std::mt19937 rng(20261016 + gu * 100 + down + ff);+ std::vector<uint8_t> blob(fmt.bytes);+ for (auto &byte : blob)+ byte = static_cast<uint8_t>(rng());+ // Use finite block scales and random packed values. NVFP4 has four UE4M3 sub-block scales.+ const uint16_t scale = 0x2400; // 2^-6; exclude NaN/Inf scales from the random bytes.+ auto scales = [&](size_t first, size_t last, int type) {+ const size_t block = cpu::tq_supported(type) ? cpu::tq_block_bytes(type) : ggml_type_size(static_cast<ggml_type>(type));+ for (size_t at = first; at < last; at += block) {+ if (type == cpu::kNvfp4Type) std::memset(blob.data() + at, 0x28, 4); // finite UE4M3 sub-block scales+ else {+ std::memcpy(blob.data() + at, &scale, sizeof(scale));+ if (type == 12 || type == 7) std::memcpy(blob.data() + at + 2, &scale, sizeof(scale));+ }+ }+ };+ scales(0, fmt.down_off, gu);+ scales(fmt.down_off, fmt.tail_off ? fmt.tail_off : blob.size(), down);+ if (fmt.tail_off) {+ const float tail[4] = {gu == cpu::kNvfp4Type ? .5f : 1.f, gu == cpu::kNvfp4Type ? .75f : 1.f,+ down == cpu::kNvfp4Type ? 1.25f : 1.f, 0.f};+ std::memcpy(blob.data() + fmt.tail_off, tail, sizeof(tail));+ }++ std::vector<std::vector<uint8_t>> act(width, std::vector<uint8_t>(fmt.act_bytes));+ std::vector<std::vector<uint8_t>> hq(width, std::vector<uint8_t>(fmt.h_bytes));+ std::vector<std::vector<float>> gates(width, std::vector<float>(ff)), downs(width, std::vector<float>(hidden));+ auto solo_gates = gates, solo_downs = downs;+ const void *ap[width], *hp[width];+ float *gp[width], *dp[width], *sg[width], *sd[width];+ for (int t = 0; t < width; ++t) {+ ap[t] = act[t].data();+ hp[t] = hq[t].data();+ gp[t] = gates[t].data();+ dp[t] = downs[t].data();+ sg[t] = solo_gates[t].data();+ sd[t] = solo_downs[t].data();+ }+ for (float amplitude : {0.f, .01f, .3f, 3.f}) {+ std::vector<float> x(hidden), h(ff);+ for (int t = 0; t < width; ++t) {+ for (auto &v : x)+ v = (static_cast<int>(rng() % 2049) - 1024) * amplitude / 1024.f;+ for (auto &v : h)+ v = (static_cast<int>(rng() % 2049) - 1024) * amplitude / 1024.f;+ cpu::native_quant_act(fmt, x.data(), act[t].data());+ cpu::native_quant_h(fmt, h.data(), hq[t].data());+ cpu::native_gu_rows(fmt, blob.data(), ap + t, 1, sg + t, 0, ff);+ cpu::native_down_rows(fmt, blob.data(), hp + t, 1, sd + t, 0, hidden);+ }+ for (int nt = 1; nt <= width; ++nt) {+ cpu::native_gu_rows_invariant(fmt, blob.data(), ap, nt, gp, 0, ff);+ cpu::native_down_rows_invariant(fmt, blob.data(), hp, nt, dp, 0, hidden);+ for (int t = 0; t < nt; ++t) {+ if (!equal(gates[t], solo_gates[t]) || !equal(downs[t], solo_downs[t])) {+ std::fprintf(stderr, "native arithmetic mismatch: gu=%d down=%d ff=%d nt=%d token=%d scale=%g\n",+ gu, down, ff, nt, t, amplitude);+ return false;+ }+ }+ }+ }+ return true;+}+} // namespace++int main() {+ // 16/17 and IQ4_XS exercise the per-token legacy fallback as well as optimized 18/21/22.+ for (int gu : {16, 17, 18, 21, 22, 23}) {+ if (!check_format(gu, 20, 64) || !check_format(gu, 20, 96) || !check_format(gu, 42, 64))+ return 1;+ }+ // NVFP4's kernels already preserve singleton arithmetic. Mixed formats also carry scale tails.+ for (auto pair : {std::pair{40, 40}, {40, 20}, {40, 42}, {22, 40}, {23, 40}})+ if (!check_format(pair.first, pair.second, 64)) return 1;+ // STRATA_KQ256 selects grouped Q4_K/Q5_1/Q8_0 kernels only when their target arithmetic matches.+ if (!check_format(12, 7, 64) || !check_format(12, 8, 64)) return 1;+ // 0.2 adds Trellis formats and folded expert inputs. Exercise both their+ // production singleton dispatch and every grouped width through the new API.+ for (int gu : {144, 145, 146})+ for (int down : {20, 144, 145, 146})+ for (bool folded : {false, true})+ if (!check_format(gu, down, 256, folded)) return 1;+ if (!check_format(22, 20, 256, true)) return 1;+ std::printf("native_arithmetic_parity: %zu finite bitwise comparisons passed\n", checked);+}diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cppindex ec1a68704..c8d72f48a 100644--- a/src/kernels/cpu/native_expert.cpp+++ b/src/kernels/cpu/native_expert.cpp@@ -1,6 +1,6 @@-// src/kernels/cpu/native_expert.cpp - plan v0.3 P6: native (GGUF-form) experts on the CPU through ggml-cpu.-// See the header. Nothing here is Strata arithmetic: the activation quantizers and the row dot products are-// ggml-cpu's, so an IQ expert computes what llama.cpp's CPU backend computes for it.+// Native (GGUF-form) experts: ggml activation quantizers and format-specific CPU row kernels.+// Default keeps the existing dispatch. WindowInvariant preserves that dispatch's single-token arithmetic+// while retaining shared weight decoding where the grouped kernel gives the same result.#include "strata/kernels/cpu/native_expert.hpp"#include "strata/kernels/cpu/expert.hpp"#include "strata/kernels/cpu/iq_avx512.hpp"@@ -293,7 +293,8 @@ int native_gu_mt_min(int gu_type) {return iq3s_one && gu_type == 21 ? 1 : mt_min;}-void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const* act, int nt, float* const* ff,+template <NativeArithmetic Arithmetic>+void native_gu_rows_impl(const NativeFmt& f, const uint8_t* blob, const void* const* act, int nt, float* const* ff,int r0, int r1) {// the multi-token kernels decode the weights once for all tokens: 2.0-2.4x ggml-cpu at three tokens, no faster// at one (all are bound by the codebook lookups, ~5 GB/s per core), measured by native_expert_parity. AVX-512@@ -353,11 +354,14 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*return;}const int mt_min = native_gu_mt_min(f.gu_type); // #152+ const bool singleton_iq = Arithmetic == NativeArithmetic::WindowInvariant && mt_min > 1;// Unsloth UD-Q4_K_XL's Q4_K gate/up: the multi-token kernel is bit-exact against ggml's per-token dot (any group// size, no #152 rule). Opt-in, STRATA_KQ256=1: measured no faster in the engine (a window's expert groups hold// ~1.4 tokens and the weights stay in L1 across ggml's per-token calls; 1.01-1.13x in native_expert_parity).static const bool kq = [] { const char* v = std::getenv("STRATA_KQ256"); return cpu_avx2_ok() && v != nullptr && std::atoi(v) != 0; }();- if (kq && f.gu_type == 12 && nt >= 2) { // one token: ggml's own dot below (the same bits, less overhead)+ // KQ matches the ggml AVX2/FMA dot; other builds keep their own singleton arithmetic.+ if (kq && f.gu_type == 12 && nt >= 2 &&+ (Arithmetic == NativeArithmetic::Default || (ggml_cpu_has_avx2() && ggml_cpu_has_fma()))) { // one token: ggml's own dot below (the same bits, less overhead)kq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);return;}@@ -372,16 +376,27 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*// through an empty switch would leave ff unwritten instead of falling back to ggml-cpu.static const bool cpu512 = cpu_avx512_ok();if (nt >= mt_min && (iq512_supported(f.gu_type) || (!cpu512 && iq256_supported(f.gu_type)))) {- if (avx512 && iq512_supported(f.gu_type)) {- iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);- return;- }- if (avx2 && iq256_supported(f.gu_type)) {- iq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);- return;+ if (singleton_iq) {+#if defined(__x86_64__) || defined(_M_X64)+ if (avx512 && ggml_cpu_has_avx2() && ggml_cpu_has_fma() &&+ (f.gu_type == 18 || f.gu_type == 21 || f.gu_type == 22)) {+ iq512_reference_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);+ return;+ }+#endif+ // Otherwise use the singleton dot below, keeping its row-major weight reuse across tokens.+ } else {+ if (avx512 && iq512_supported(f.gu_type)) {+ iq512_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);+ return;+ }+ if (avx2 && iq256_supported(f.gu_type)) {+ iq256_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);+ return;+ }}}- if (avx1 && nt >= mt_min && iq128_supported(f.gu_type)) {+ if (!singleton_iq && avx1 && nt >= mt_min && iq128_supported(f.gu_type)) {iq128_gu_rows(f.gu_type, blob, f.gu_row, f.up_off, (int) f.n_embd, act, nt, ff, r0, r1);return;}@@ -424,7 +439,8 @@ void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const*}}-void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt, float* const* out,+template <NativeArithmetic Arithmetic>+void native_down_rows_impl(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt, float* const* out,int r0, int r1) {// IQ4_NL down rows: the AVX-2 multi-token kernel decodes the nibbles and absolutises the weights once per// block instead of once per token; ggml-cpu's dot is single-token. STRATA_NO_IQ4NL falls back to it.@@ -438,13 +454,22 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* consttq_rows(tq_isa(), f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);return;}- if (cpu_avx2_ok() && kq && nt >= 2 && (f.d_type == 7 || f.d_type == 8)) { // Q5_1 / Q8_0 down: bit-exact, any group size+ if (cpu_avx2_ok() && kq && nt >= 2 && (f.d_type == 7 || f.d_type == 8) &&+ (Arithmetic == NativeArithmetic::Default || (ggml_cpu_has_avx2() && ggml_cpu_has_fma()))) { // Q5_1 / Q8_0 down: bit-exact, any group sizekq256_rows(f.d_type, blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);return;}if (cpu_avx2_ok() && nt >= mt_min && f.d_type == 20 && iq4nl_mt) { // #152: the same rule as the gate/up rows- iq4nl256_down_rows(blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);- return;+ if (Arithmetic == NativeArithmetic::Default || mt_min <= 1) {+ iq4nl256_down_rows(blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);+ return;+ }+#if defined(__x86_64__) || defined(_M_X64)+ if (ggml_cpu_has_avx2() && ggml_cpu_has_fma()) {+ iq4nl_reference_rows(blob + f.down_off, f.d_row, (int) f.n_ff, hq, nt, out, r0, r1);+ return;+ }+#endif}// the older-CPU tier: IQ4_NL and Q2_0 down rows in 128-bit (ggml-cpu's dots for both are scalar here)static const bool avx1 = !cpu_avx2_ok() && cpu_avx1_ok() && std::getenv("STRATA_NO_IQ128") == nullptr;@@ -476,4 +501,26 @@ void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const}}+// Keep one dispatch implementation: only width-sensitive IQ kernels need a different arithmetic path.+// Already invariant kernels (including NVFP4) retain their shared decoding and scale handling.+void native_gu_rows(const NativeFmt& f, const uint8_t* blob, const void* const* act, int nt,+ float* const* ff, int r0, int r1) {+ native_gu_rows_impl<NativeArithmetic::Default>(f, blob, act, nt, ff, r0, r1);+}++void native_down_rows(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt,+ float* const* out, int r0, int r1) {+ native_down_rows_impl<NativeArithmetic::Default>(f, blob, hq, nt, out, r0, r1);+}++void native_gu_rows_invariant(const NativeFmt& f, const uint8_t* blob, const void* const* act, int nt,+ float* const* ff, int r0, int r1) {+ native_gu_rows_impl<NativeArithmetic::WindowInvariant>(f, blob, act, nt, ff, r0, r1);+}++void native_down_rows_invariant(const NativeFmt& f, const uint8_t* blob, const void* const* hq, int nt,+ float* const* out, int r0, int r1) {+ native_down_rows_impl<NativeArithmetic::WindowInvariant>(f, blob, hq, nt, out, r0, r1);+}+} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex cdcc8d792..d73557215 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -1231,7 +1231,10 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {} else if (mode_ == 5) {float* ff[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) ff[t] = sb.ff[t];- native_gu_rows(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);+ if (narithmetic_ == NativeArithmetic::WindowInvariant)+ native_gu_rows_invariant(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);+ else+ native_gu_rows(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);} else if (q2_native_kernels(nfmt->d_type)) {// Q2_0 down (most IQ layers): the AVX-512 kernel, ggml-cpu has only a scalar one on x86const ActQ* a2[MAXT];@@ -1241,7 +1244,10 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {} else {const void* hq[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];- native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);+ if (narithmetic_ == NativeArithmetic::WindowInvariant)+ native_down_rows_invariant(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);+ else+ native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);}g += l1 - l0;}@@ -1356,7 +1362,7 @@ void ExpertPool::run_split_multi(ExpertJobMulti* jobs, int n) {ms_drain_ += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();}-void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs, int n) {+void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs, int n, NativeArithmetic arithmetic) {if (n <= 0) return;const auto t0 = std::chrono::steady_clock::now();// more distinct experts than buffers: run them in batches@@ -1364,6 +1370,7 @@ void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobsconst int nb = (std::min)(kMaxSplitMulti, n - b0);mjobs_ = jobs + b0;nfmt_ = &f;+ narithmetic_ = arithmetic;mrows_ = (int64_t) nb * f.n_ff;mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_ff, nb) : phase_tasks(mrows_);const auto a = std::chrono::steady_clock::now();diff --git a/src/program/generate.cpp b/src/program/generate.cppindex cd3cb12e3..71ce02a18 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -1166,12 +1166,12 @@ void drive_pool(void* user, const float* x_f, const int32_t* ids, const float* w}/// Plan v0.3 P6: the pool for a verify window.-void drive_pool_multi(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out,- int64_t layer) {+void drive_pool_multi_impl(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out,+ int64_t layer, strata::kernels::cpu::NativeArithmetic arithmetic) {Drive* t = (Drive*) user;t->d.layers = layer;const Clock::time_point a = Clock::now();- strata::core::expert_pool_dispatch_multi(t->d, x_f, ids, n_tok, k, out);+ strata::core::expert_pool_dispatch_multi(t->d, x_f, ids, n_tok, k, out, arithmetic);t->cpu_ms += std::chrono::duration<double, std::milli>(Clock::now() - a).count();++t->calls;// the routing trace for the serve path: the same record format drive_pool writes (layer, k, ids, weights),@@ -1189,6 +1189,15 @@ void drive_pool_multi(void* user, const float* x_f, const int32_t* ids, int64_t}}+/// The verify window's pool as every decode has always run it (Default arithmetic), and the same with the window-invariant native+/// arithmetic that a probabilistic (STRATA_SPEC_PROB) decode uses, so a row's target distribution does not depend on the window.+void drive_pool_multi(void* u, const float* x, const int32_t* ids, int64_t n, int64_t k, float* out, int64_t l) {+ drive_pool_multi_impl(u, x, ids, n, k, out, l, strata::kernels::cpu::NativeArithmetic::Default);+}+void drive_pool_multi_invariant(void* u, const float* x, const int32_t* ids, int64_t n, int64_t k, float* out, int64_t l) {+ drive_pool_multi_impl(u, x, ids, n, k, out, l, strata::kernels::cpu::NativeArithmetic::WindowInvariant);+}+/// Layer split: every verify stage shares one Drive (its counters, usage and failure flags); the GPU plan, the expert/// cache and the PCIe share the pool uses for a layer are those of the stage that runs it.struct SplitDrive {@@ -1201,8 +1210,8 @@ struct SplitDrive {const uint64_t* cache_slot_off[kMax] = {};int pcie_num[kMax] = {};};-void drive_pool_split(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out,- int64_t layer) {+void drive_pool_split_impl(void* user, const float* x_f, const int32_t* ids, int64_t n_tok, int64_t k, float* out,+ int64_t layer, strata::kernels::cpu::NativeArithmetic arithmetic) {SplitDrive* s = (SplitDrive*) user;int st = 0;while (st + 1 < s->n && layer >= s->end[st]) ++st;@@ -1211,7 +1220,13 @@ void drive_pool_split(void* user, const float* x_f, const int32_t* ids, int64_td.d.cache_base = s->cache_base[st];d.d.cache_slot_off = s->cache_slot_off[st];d.d.pcie_num = s->pcie_num[st];- drive_pool_multi(s->base, x_f, ids, n_tok, k, out, layer);+ drive_pool_multi_impl(s->base, x_f, ids, n_tok, k, out, layer, arithmetic);+}+void drive_pool_split(void* u, const float* x, const int32_t* ids, int64_t n, int64_t k, float* out, int64_t l) {+ drive_pool_split_impl(u, x, ids, n, k, out, l, strata::kernels::cpu::NativeArithmetic::Default);+}+void drive_pool_split_invariant(void* u, const float* x, const int32_t* ids, int64_t n, int64_t k, float* out, int64_t l) {+ drive_pool_split_impl(u, x, ids, n, k, out, l, strata::kernels::cpu::NativeArithmetic::WindowInvariant);}/// Layer split across GPUs: a later stage on its own device, with its own copy of the dense weights, a session, an@@ -9442,6 +9457,7 @@ int main(int argc, char** argv) {}// the pool the verify windows call: with a layer split, the wrapper that routes each layer to its stageconst strata::core::PoolMultiFn win_pool_fn = n_stages > 1 ? &drive_pool_split : &drive_pool_multi;+ const strata::core::PoolMultiFn invariant_win_pool_fn = n_stages > 1 ? &drive_pool_split_invariant : &drive_pool_multi_invariant;void* const win_pool_user = n_stages > 1 ? (void*) &split_drive : (void*) &drive;mem_mark("the verifier and the drafter's binding");ver.set_split(o.spec_split);@@ -15384,7 +15400,8 @@ int main(int argc, char** argv) {// (never on a first window: a merged one holds prompt tokens, not drafts)if (gated_proposal) ver.set_spec_proposal(proposal.n_q ? proposal.mu.data() : nullptr, proposal.n_q);else if (use_mtp && mtp.prob() && !from_sfx && !first_window && T_mtp > 1) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);- if (!ver.run(T, window.data(), p, win_pool_fn, win_pool_user, outv.data(), err) || drive.d.failed) {+ const auto decode_pool_fn = use_mtp && mtp.prob() ? invariant_win_pool_fn : win_pool_fn;+ if (!ver.run(T, window.data(), p, decode_pool_fn, win_pool_user, outv.data(), err) || drive.d.failed) {std::printf("ERR %s\n", drive.d.failed && drive.d.fail ? drive.d.fail : err.c_str());return 1;}@@ -16962,7 +16979,8 @@ int main(int argc, char** argv) {else if (pending.empty() || ++pending_age >= adapt_lag()) apply_pending(true);if (gated_proposal) ver.set_spec_proposal(proposal.n_q ? proposal.mu.data() : nullptr, proposal.n_q);else if (use_mtp && mtp.prob() && !from_sfx && T_mtp > 1 && o.spec_corrupt <= 0) ver.set_spec_q(mtp.spec_q(), T_mtp - 1);- if (!ver.run(T, window.data(), p, &drive_pool_multi, &drive, outv.data(), err)) {+ const auto decode_pool_fn = use_mtp && mtp.prob() ? &drive_pool_multi_invariant : &drive_pool_multi;+ if (!ver.run(T, window.data(), p, decode_pool_fn, &drive, outv.data(), err)) {// The graph of a window size is captured the first time it is used, and the draft source took VRAM: when the capture runs// out of memory the window gets one row narrower (the widths below it are already captured) instead of the run failing.if (T > 2 && !first_window && !gated_proposal && err.find("out of memory") != std::string::npos) {diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txtindex 3e070cc20..6e817a76e 100644--- a/sycl/CMakeLists.txt+++ b/sycl/CMakeLists.txt@@ -289,6 +289,23 @@ if(STRATA_SYCL_PARITY)add_test(NAME spec_admission_pick COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=pick $<TARGET_FILE:spec_admission_test> pick)add_test(NAME spec_admission_top COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=top $<TARGET_FILE:spec_admission_test> top)add_test(NAME spec_admission_exact_top_only COMMAND ${CMAKE_COMMAND} -E env STRATA_SPEC_PROB_GATE=TOP $<TARGET_FILE:spec_admission_test> pick)+ if(STRATA_NATIVE_EXPERTS)+ # a token's native expert rows are the same bits at every number of tokens that share its expert (host CPU kernels)+ add_executable(native_arithmetic_parity ${STRATA_ROOT}/src/kernels/cpu/native_arithmetic_parity.cpp)+ target_link_libraries(native_arithmetic_parity PRIVATE strata_kernels_cpu)+ add_test(NAME native_arithmetic_parity COMMAND native_arithmetic_parity)+ add_test(NAME native_arithmetic_parity_avx COMMAND ${CMAKE_COMMAND} -E env+ STRATA_FORCE_ISA=avx $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_kq256 COMMAND ${CMAKE_COMMAND} -E env+ STRATA_KQ256=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_mt1 COMMAND ${CMAKE_COMMAND} -E env STRATA_IQ_MT_MIN=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_no512 COMMAND ${CMAKE_COMMAND} -E env STRATA_NO_IQ512=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_ggml COMMAND ${CMAKE_COMMAND} -E env STRATA_NO_IQ512=1 STRATA_NO_IQ256=1 STRATA_NO_IQ4NL=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_nvfp4_avx2 COMMAND ${CMAKE_COMMAND} -E env+ STRATA_NO_NVFP4_512=1 $<TARGET_FILE:native_arithmetic_parity>)+ add_test(NAME native_arithmetic_parity_nvfp4_ggml COMMAND ${CMAKE_COMMAND} -E env+ STRATA_NO_NVFP4_512=1 STRATA_NO_NVFP4_256=1 $<TARGET_FILE:native_arithmetic_parity>)+ endif()foreach(p IN ITEMS dequant_s2 s2_gemv shared_expert gr gdn s2_gemv_q8 sampler rope quantize_act router_top10s_gemv elementwise bf16_gemv s_gemv_q8k qsa kv_q8 kv_stream kv_q4 cvec)strata_resolve(_src src/kernels/${p}_parity.cpp)