Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
APR port: decode-once verify kernels and MMQ prompt path for TQ types
Laurent Zuijdwijkf7f21bc823b7
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 4b9532bbf..1964ee7bc 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1437,6 +1437,12 @@ if(STRATA_NATIVE_EXPERTS)add_test(NAME q8k_quant_parity_avx COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_ISA=avx $<TARGET_FILE:q8k_quant_parity>)endif()set(_strata_mmq_types q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)+ # APR: the agention llama.cpp fork has MMQ instances for its trellis expert types (f62fdead0); any other ggml+ # tree leaves them to the FP16 prompt path+ if(STRATA_GGML_DIR AND EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqk6.cu")+ list(APPEND _strata_mmq_types tq2_t tqk6 tqk7)+ set(_strata_mmq_tq ON)+ endif()# #296: OrcaRouter Q4_K_S's Q5_0 experts (Q4_K and Q5_1 come with STRATA_MMQ_KQUANTS on CUDA, S5's branch)if(STRATA_ORCA_Q4KS_MMQ)list(APPEND _strata_mmq_types q5_0)@@ -1576,6 +1582,9 @@ if(STRATA_NATIVE_EXPERTS)if(STRATA_ORCA_Q4KS_MMQ)target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)endif()+ if(_strata_mmq_tq)+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)+ endif()target_link_libraries(strata_prefill PUBLIC strata_mmq)target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)if(STRATA_BUILD_TESTS)@@ -1656,6 +1665,9 @@ if(STRATA_NATIVE_EXPERTS)if(STRATA_ORCA_Q4KS_MMQ)target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)endif()+ if(_strata_mmq_tq)+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)+ endif()target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)target_link_libraries(strata_prefill PUBLIC strata_mmq)target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex 4d6ef38ab..45e78edc0 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -647,26 +647,46 @@ __device__ __forceinline__ void tq_states4(const uint8_t* qs, int l, uint32_t s[for (int i = 0; i < 4; ++i) s[i] = (uint32_t) (v >> (r + (3 - i) * K)) & 0xFFFFu;}}-// lane iqs/4 of block kbx against the q8_1 blocks aligned with the block's start (4 per trellis block)+// lane iqs/4 of block kbx against the q8_1 blocks aligned with the block's start (4 per trellis block). Split in+// two, like the IQ types' decode-once kernels below: tq_load decodes the lane's 16 codebook values (weight side only),+// tq_apply does the activation side. vec_dot_tq_q8_1 is apply(load()), so the per-entry and the decode-once kernels+// do the same float operations in the same order: bitwise equal.+struct TqLane { float v[16]; float d; };template<int TY>-__device__ __forceinline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,- const int& kbx, const int& iqs) {+__device__ __forceinline__ TqLane tq_load(const void* __restrict__ vbq, int kbx, int iqs) {const auto* b = (const typename TqInfo<TY>::block_t*) vbq + kbx;- const int l = iqs / 4;uint32_t s[4];- tq_states4<TY>(b->qs, l, s);+ tq_states4<TY>(b->qs, iqs / 4, s);+ TqLane r;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ float2 p0, p1;+ tq_step(s[i], p0, p1);+ r.v[4 * i + 0] = p0.x;+ r.v[4 * i + 1] = p0.y;+ r.v[4 * i + 2] = p1.x;+ r.v[4 * i + 3] = p1.y;+ }+ r.d = tq_h2f(b->d);+ return r;+}+__device__ __forceinline__ float tq_apply(const TqLane& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ const int l = iqs / 4;const block_q8_1* b8 = bq8_1 + l / 2;const int* q8 = (const int*) b8->qs + 4 * (l & 1);float sum = 0.0f;#pragma unrollfor (int i = 0; i < 4; ++i) {- float2 p0, p1;- tq_step(s[i], p0, p1);const int v = q8[i];- sum += p0.x * (float) (int8_t) (v) + p0.y * (float) (int8_t) (v >> 8) + p1.x * (float) (int8_t) (v >> 16) +- p1.y * (float) (int8_t) (v >> 24);+ sum += r.v[4 * i + 0] * (float) (int8_t) (v) + r.v[4 * i + 1] * (float) (int8_t) (v >> 8) ++ r.v[4 * i + 2] * (float) (int8_t) (v >> 16) + r.v[4 * i + 3] * (float) (int8_t) (v >> 24);}- return tq_h2f(b->d) * __low2float(b8->ds) * sum;+ return r.d * __low2float(b8->ds) * sum;+}+template<int TY>+__device__ __forceinline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,+ const int& kbx, const int& iqs) {+ return tq_apply(tq_load<TY>(vbq, kbx, iqs), bq8_1, iqs);}// ---------------------------------------------------------------- the formats@@ -1111,6 +1131,18 @@ template<> struct Split<40> { // NVFP4: vec_dot_nvfp4_q8_1's two 16-value sub-return sum;}};+// APR trellis types: the lane's 16 decoded codebook values (tq_load / tq_apply above)+template<> inline constexpr bool kSplit<144> = true;+template<> inline constexpr bool kSplit<145> = true;+template<> inline constexpr bool kSplit<146> = true;+template<int TY> struct SplitTq {+ using W = TqLane;+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) { return tq_load<TY>(vbq, kbx, iqs); }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }+};+template<> struct Split<144> : SplitTq<144> {};+template<> struct Split<145> : SplitTq<145> {};+template<> struct Split<146> : SplitTq<146> {};// One row against the n <= NC activations x + off[0..n) (n >= 1, warp-uniform; offsets in q8_1 blocks, 32-bit to// spare registers), the whole warp. Per activation this is row_dot: the same calls k, lane-strided the same way,diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cuindex 9b78fcc9f..21a9c2e72 100644--- a/src/prefill/moe_mmq.cu+++ b/src/prefill/moe_mmq.cu@@ -263,6 +263,9 @@ bool supported(int t) {case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S:case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS:case GGML_TYPE_Q8_0: // the draft layer's dense matrices (E-9)+#ifdef STRATA_MMQ_TQ+ case GGML_TYPE_TQ2_T: case GGML_TYPE_TQK6: case GGML_TYPE_TQK7: // APR experts (agention fork's MMQ)+#endifreturn true;#ifdef STRATA_MMQ_KQUANTS// Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF projections (STRATA_DENSE_MMQ)@@ -431,6 +434,11 @@ void Context::run(const Product& p, void* stream) {case GGML_TYPE_IQ4_NL: mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, a, s); break;case GGML_TYPE_IQ4_XS: mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, a, s); break;case GGML_TYPE_Q8_0: mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, a, s); break;+#ifdef STRATA_MMQ_TQ+ case GGML_TYPE_TQ2_T: mul_mat_q_case<GGML_TYPE_TQ2_T>(ctx, a, s); break;+ case GGML_TYPE_TQK6: mul_mat_q_case<GGML_TYPE_TQK6>(ctx, a, s); break;+ case GGML_TYPE_TQK7: mul_mat_q_case<GGML_TYPE_TQK7>(ctx, a, s); break;+#endif#if defined(STRATA_MMQ_KQUANTS) && defined(STRATA_KQ_DSO)case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_1: {const int e = kq::run((int) t, &a, s);diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex c8c7fa72f..3f6ec14db 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -23,6 +23,7 @@#include "strata/kernels/native_ple_postops.hpp"#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include "strata/kernels/qsa.hpp"#include "strata/kernels/kv_stream.hpp"#include "strata/kernels/cvec.hpp"@@ -873,6 +874,7 @@ struct Prefill::Impl {*shared = nullptr, *sg = nullptr;int32_t *ids = nullptr, *slot_dev = nullptr, *src_dev = nullptr;uint16_t *Xs = nullptr, *Hh = nullptr, *sh_h = nullptr;+ float* Xr = nullptr; // APR: Hadamard-rotated copy of `mixed` for the MMQ path// step 2b (MMQ): the activations quantized per layer, H in FP32 and its group's quantized rows, the identity// row map, the group bounds, the group buffers of gathered expertsvoid *Xq = nullptr, *Hq = nullptr;@@ -1123,8 +1125,6 @@ const MmqPlan& mmq_plan() {if (on && lay.native && fused::native_supported(gt, dt)) { p.fo[(size_t) l] = 1; p.any = true; }continue;}- // APR (prism.hadamard): folded layers rotate their activations on the FP16 path only- if (lay.native && (lay.fmt[(size_t) l].had_gu || lay.fmt[(size_t) l].had_d)) { p.fallback = true; continue; }p.layer[(size_t) l] = 1;p.any = true;p.gu_max = std::max(p.gu_max, mmq::matrix_bytes(gt, 1280, N));@@ -1443,6 +1443,8 @@ bool Prefill::carve(size_t T, void* alloc) {m.Xscale = c.take<float>(T * K, ok); m.Hscale = c.take<float>(T * K, ok);m.grp_tail = c.take<float>(MMQ_GROUP * 4, ok);m.row_sd = c.take<float>(T * K, ok);+ // APR: the rotated expert input (only for Hadamard-folded packs)+ m.Xr = strata::kernels::cpu::hadamard_spec().any() ? c.take<float>(T * N, ok) : nullptr;}if (base == nullptr) ok = false;}@@ -1458,6 +1460,13 @@ bool Prefill::carve(size_t T, void* alloc) {if (!m.mmq_ctx) m.mmq_ctx = std::make_unique<mmq::Context>();}m.ring = ring_slots(T);+ { // APR: the device sign vectors now (cudaMalloc), not on first use inside a run+ const auto& lay = strata::kernels::cpu::expert_layout();+ for (size_t l = 0; lay.native && l < lay.fmt.size(); ++l) {+ if (lay.fmt[l].had_gu) strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);+ if (lay.fmt[l].had_d) strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);+ }+ }if (o.base == nullptr && m.ring > 0) {// OWNED buffers: the ring in ONE allocation. 384 separate 2.7 MiB cudaMallocs each round up to a 2 MiB page// (~1.3 MiB a slot, ~0.5 GiB in all) that no count ever saw. A borrowed region keeps its per-slot layout (and@@ -1727,6 +1736,8 @@ bool Prefill::set_stage_helper(Prefill* helper, std::string& err) {if (helper == nullptr || helper == this || !split_help_env()) return true;const Impl& h = *helper->impl_;if (m.device < 0 || h.device < 0 || h.device == m.device || !mmq_plan().any) return true;+ // APR: the helper's MMQ share does not rotate the activations of Hadamard-folded experts; not used for those packs+ if (strata::kernels::cpu::hadamard_spec().any()) return true;auto pp = std::make_unique<PeerPrefill>();pp->peer = nullptr; // stream-only: it holds no experts, it streams a share of this stage'spp->compact = true;@@ -1839,6 +1850,11 @@ bool Prefill::set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& efor (const auto& f : lay.fmt)if (lay.native && f.tail_off) { err = "prefill peer: not for NVFP4 experts yet"; return false; }}+ if (strata::kernels::cpu::hadamard_spec().any()) { // APR: the peer's MMQ share does not rotate; declined+ std::fprintf(stderr, "strata: prefill peer: not used for a Hadamard-folded (APR) pack\n");+ m.pp.reset();+ return true;+ }auto pp = std::make_unique<PeerPrefill>();pp->peer = peer;pp->p2p = peer->p2p();@@ -3194,11 +3210,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri// keep MMQ; without the variable nothing here runs. A native pack's layer takes the native kernels// (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M).const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();- // APR (prism.hadamard): the rotation is applied on the FP16 path only, so a folded layer never- // takes MMQ (nor the fused kernels or the peer, which need MMQ)+ // APR (prism.hadamard): folded gate/up take H (s * x), folded down H (s * h), on the FP16 and MMQ+ // paths; the fused kernels do not rotate (and have no trellis types), and the multi-GPU prompt+ // paths are not set up for a folded pack (set_peer, set_stage_helper)const bool had_gu = lay.native && lay.fmt[(size_t) l].had_gu;const bool had_d = lay.native && lay.fmt[(size_t) l].had_d;- const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l]; // never for a folded layer+ const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];const int had_block = lay.native ? lay.fmt[(size_t) l].had_block : 0;const float* had_sx = had_gu ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].gu_signs, N) : nullptr;const float* had_sh = had_d ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].d_signs, 640) : nullptr;@@ -3208,7 +3225,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri// fused_ring() sized the ring and the buffers for itconst bool no_peer = !core::peer_portable();const bool fused_only = mmq_plan().any && mmq_plan().fo[(size_t) l];- const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && fused::native_supported(mmq_gt, mmq_dt);+ const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && !had_gu && !had_d &&+ fused::native_supported(mmq_gt, mmq_dt);const bool fused_l = (use_mmq && stream_all && no_peer && !lay.native && fused::enabled()) || fused_nat;// #583 / #954: the fused layout's GU/H/Xq hold the grouping tables and the int8 rows, sized for// stream_all_min() - 1 tokens of MMQ's rows, so a layer that takes MMQ or the FP16 path at the FULL@@ -3634,7 +3652,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stript.mark(kPfGather, cs);if (use_mmq) {// step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`- mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);+ // (APR: from a rotated copy of `mixed`; `mixed` itself feeds the shared expert)+ const float* xsrc = m.mixed;+ if (had_gu) {+ strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);+ xsrc = m.Xr;+ }+ mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down// reads the group's own quantized H)const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;@@ -4147,6 +4171,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strim.grp_tail, r0, m.cs);elsemmq::swiglu(m.GU + r0 * 1280, m.H + r0 * 640, nr, 640, !lay.native, m.cs);+ if (had_d)+ strata::kernels::hadamard_rows(m.H + r0 * 640, m.H + r0 * 640, nr, 640, had_block, had_sh, m.cs);pt.mark(kPfGemmD, cs);mmq::quantize(m.H + r0 * 640, nullptr, m.Hq, mmq_dt, 640, 640, nr, m.cs, m.Hscale);mmq::Product dn;diff --git a/tests/hip/prefill_mmq_parity.cpp b/tests/hip/prefill_mmq_parity.cppindex c477d5b3e..4a204781d 100644--- a/tests/hip/prefill_mmq_parity.cpp+++ b/tests/hip/prefill_mmq_parity.cpp@@ -10,6 +10,8 @@#include <algorithm>#include <cmath>#include <cstdint>+#include <cstring>+#include <cstdlib>#include <fstream>#include <iostream>#include <numeric>@@ -55,10 +57,10 @@ std::vector<float> decode_matrix(ggml_type type, const std::vector<uint8_t> & q,return out;}-std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial) {- const ggml_type type = GGML_TYPE_Q2_0;+std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial,+ ggml_type type = GGML_TYPE_Q2_0) {const auto * tr = ggml_get_type_traits(type);- if (!tr || !tr->from_float_ref) throw std::runtime_error("Q2_0 host quantizer unavailable");+ if (!tr || !tr->from_float_ref) throw std::runtime_error("host quantizer unavailable");const size_t rb = row_bytes(type, cols);std::vector<uint8_t> out((size_t) rows * rb);std::vector<float> row((size_t) cols);@@ -171,8 +173,17 @@ Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name,}}}+ {+ size_t bad = 0, first = got.size();+ uint32_t first_bits = 0;+ for (size_t i = 0; i < got.size(); ++i)+ if (!std::isfinite(got[i])) { if (!bad++) { first = i; std::memcpy(&first_bits, &got[i], 4); } }+ if (bad)+ throw std::runtime_error(name + ": " + std::to_string(bad) + " non-finite or unwritten MMQ outputs, first at row " ++ std::to_string(first / (size_t) out_rows) + " col " + std::to_string(first % (size_t) out_rows) ++ " bits " + std::to_string(first_bits));+ }for (size_t i = 0; i < got.size(); ++i) {- if (!std::isfinite(got[i])) throw std::runtime_error(name + ": non-finite or unwritten MMQ output");const double d = (double) got[i] - ref[i];err2 += d * d; ref2 += (double) ref[i] * ref[i]; max_abs = std::max(max_abs, (float) std::fabs(d));}@@ -199,9 +210,10 @@ Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name,return m;}-std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial) {+std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial,+ ggml_type type = GGML_TYPE_Q2_0) {std::vector<std::vector<uint8_t>> w;- for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial));+ for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial, type));return w;}@@ -311,6 +323,24 @@ int main(int argc, char ** argv) {1280, 2560, synthetic_of_type(type, 1, 1280, 2560, 5), counts,make_activations(rows, 2560, 77), src, dst);}+ // APR trellis experts (ggml types 144-146), when the build has their MMQ instances (agention fork): gate/up+ // 1280 x 2560 and down 2560 x 640 (5 blocks per row: the last 256-slice reads past the row, against zero+ // activations)+ for (const ggml_type t : {(ggml_type) 145, (ggml_type) 146, (ggml_type) 144}) {+ if (!supported((int) t)) { std::cout << "type " << (int) t << ": no MMQ instance in this build\n"; continue; }+ const std::vector<int> counts{2, 3};+ const auto src = permutation(5, 1, false), dst = permutation(5, 2, true);+ const std::string tn = ggml_type_name(t);+ const char* rep = std::getenv("MMQ_PARITY_REPEAT"); // repeat each product (flakiness screen)+ const int reps = rep ? std::max(1, std::atoi(rep)) : 1;+ const auto wgu = synthetic_q2(2, 1280, 2560, 5, t), wd = synthetic_q2(2, 2560, 640, 6, t);+ for (int r = 0; r < reps; ++r) {+ run_product(ctx, stream, "synthetic-" + tn + "-GU", t, 1280, 2560, wgu, counts,+ make_activations(5, 2560, 5), src, dst);+ run_product(ctx, stream, "synthetic-" + tn + "-down", t, 2560, 640, wd, counts,+ make_activations(5, 640, 6), src, dst);+ }+ }if (argc == 2) run_real_iq3_first_expert(ctx, stream, argv[1]);}hip_check(hipStreamSynchronize(stream), "final synchronize");