Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

APR port: decode-once verify kernels and MMQ prompt path for TQ types

Laurent Zuijdwijkf7f21bc823b7
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4b9532bbf..1964ee7bc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1437,6 +1437,12 @@ if(STRATA_NATIVE_EXPERTS)
add_test(NAME q8k_quant_parity_avx COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_ISA=avx $<TARGET_FILE:q8k_quant_parity>)
endif()
set(_strata_mmq_types q2_0 iq2_xxs iq2_xs iq2_s iq3_xxs iq3_s iq4_nl iq4_xs q8_0)
+ # APR: the agention llama.cpp fork has MMQ instances for its trellis expert types (f62fdead0); any other ggml
+ # tree leaves them to the FP16 prompt path
+ if(STRATA_GGML_DIR AND EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqk6.cu")
+ list(APPEND _strata_mmq_types tq2_t tqk6 tqk7)
+ set(_strata_mmq_tq ON)
+ endif()
# #296: OrcaRouter Q4_K_S's Q5_0 experts (Q4_K and Q5_1 come with STRATA_MMQ_KQUANTS on CUDA, S5's branch)
if(STRATA_ORCA_Q4KS_MMQ)
list(APPEND _strata_mmq_types q5_0)
@@ -1576,6 +1582,9 @@ if(STRATA_NATIVE_EXPERTS)
if(STRATA_ORCA_Q4KS_MMQ)
target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)
endif()
+ if(_strata_mmq_tq)
+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)
+ endif()
target_link_libraries(strata_prefill PUBLIC strata_mmq)
target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)
if(STRATA_BUILD_TESTS)
@@ -1656,6 +1665,9 @@ if(STRATA_NATIVE_EXPERTS)
if(STRATA_ORCA_Q4KS_MMQ)
target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)
endif()
+ if(_strata_mmq_tq)
+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)
+ endif()
target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)
target_link_libraries(strata_prefill PUBLIC strata_mmq)
target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index 4d6ef38ab..45e78edc0 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -647,26 +647,46 @@ __device__ __forceinline__ void tq_states4(const uint8_t* qs, int l, uint32_t s[
for (int i = 0; i < 4; ++i) s[i] = (uint32_t) (v >> (r + (3 - i) * K)) & 0xFFFFu;
}
}
-// lane iqs/4 of block kbx against the q8_1 blocks aligned with the block's start (4 per trellis block)
+// lane iqs/4 of block kbx against the q8_1 blocks aligned with the block's start (4 per trellis block). Split in
+// two, like the IQ types' decode-once kernels below: tq_load decodes the lane's 16 codebook values (weight side only),
+// tq_apply does the activation side. vec_dot_tq_q8_1 is apply(load()), so the per-entry and the decode-once kernels
+// do the same float operations in the same order: bitwise equal.
+struct TqLane { float v[16]; float d; };
template<int TY>
-__device__ __forceinline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
- const int& kbx, const int& iqs) {
+__device__ __forceinline__ TqLane tq_load(const void* __restrict__ vbq, int kbx, int iqs) {
const auto* b = (const typename TqInfo<TY>::block_t*) vbq + kbx;
- const int l = iqs / 4;
uint32_t s[4];
- tq_states4<TY>(b->qs, l, s);
+ tq_states4<TY>(b->qs, iqs / 4, s);
+ TqLane r;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ float2 p0, p1;
+ tq_step(s[i], p0, p1);
+ r.v[4 * i + 0] = p0.x;
+ r.v[4 * i + 1] = p0.y;
+ r.v[4 * i + 2] = p1.x;
+ r.v[4 * i + 3] = p1.y;
+ }
+ r.d = tq_h2f(b->d);
+ return r;
+}
+__device__ __forceinline__ float tq_apply(const TqLane& r, const block_q8_1* __restrict__ bq8_1, int iqs) {
+ const int l = iqs / 4;
const block_q8_1* b8 = bq8_1 + l / 2;
const int* q8 = (const int*) b8->qs + 4 * (l & 1);
float sum = 0.0f;
#pragma unroll
for (int i = 0; i < 4; ++i) {
- float2 p0, p1;
- tq_step(s[i], p0, p1);
const int v = q8[i];
- sum += p0.x * (float) (int8_t) (v) + p0.y * (float) (int8_t) (v >> 8) + p1.x * (float) (int8_t) (v >> 16) +
- p1.y * (float) (int8_t) (v >> 24);
+ sum += r.v[4 * i + 0] * (float) (int8_t) (v) + r.v[4 * i + 1] * (float) (int8_t) (v >> 8) +
+ r.v[4 * i + 2] * (float) (int8_t) (v >> 16) + r.v[4 * i + 3] * (float) (int8_t) (v >> 24);
}
- return tq_h2f(b->d) * __low2float(b8->ds) * sum;
+ return r.d * __low2float(b8->ds) * sum;
+}
+template<int TY>
+__device__ __forceinline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,
+ const int& kbx, const int& iqs) {
+ return tq_apply(tq_load<TY>(vbq, kbx, iqs), bq8_1, iqs);
}
// ---------------------------------------------------------------- the formats
@@ -1111,6 +1131,18 @@ template<> struct Split<40> { // NVFP4: vec_dot_nvfp4_q8_1's two 16-value sub-
return sum;
}
};
+// APR trellis types: the lane's 16 decoded codebook values (tq_load / tq_apply above)
+template<> inline constexpr bool kSplit<144> = true;
+template<> inline constexpr bool kSplit<145> = true;
+template<> inline constexpr bool kSplit<146> = true;
+template<int TY> struct SplitTq {
+ using W = TqLane;
+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) { return tq_load<TY>(vbq, kbx, iqs); }
+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }
+};
+template<> struct Split<144> : SplitTq<144> {};
+template<> struct Split<145> : SplitTq<145> {};
+template<> struct Split<146> : SplitTq<146> {};
// One row against the n <= NC activations x + off[0..n) (n >= 1, warp-uniform; offsets in q8_1 blocks, 32-bit to
// spare registers), the whole warp. Per activation this is row_dot: the same calls k, lane-strided the same way,
diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cu
index 9b78fcc9f..21a9c2e72 100644
--- a/src/prefill/moe_mmq.cu
+++ b/src/prefill/moe_mmq.cu
@@ -263,6 +263,9 @@ bool supported(int t) {
case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS:
case GGML_TYPE_Q8_0: // the draft layer's dense matrices (E-9)
+#ifdef STRATA_MMQ_TQ
+ case GGML_TYPE_TQ2_T: case GGML_TYPE_TQK6: case GGML_TYPE_TQK7: // APR experts (agention fork's MMQ)
+#endif
return true;
#ifdef STRATA_MMQ_KQUANTS
// Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF projections (STRATA_DENSE_MMQ)
@@ -431,6 +434,11 @@ void Context::run(const Product& p, void* stream) {
case GGML_TYPE_IQ4_NL: mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, a, s); break;
case GGML_TYPE_IQ4_XS: mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, a, s); break;
case GGML_TYPE_Q8_0: mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, a, s); break;
+#ifdef STRATA_MMQ_TQ
+ case GGML_TYPE_TQ2_T: mul_mat_q_case<GGML_TYPE_TQ2_T>(ctx, a, s); break;
+ case GGML_TYPE_TQK6: mul_mat_q_case<GGML_TYPE_TQK6>(ctx, a, s); break;
+ case GGML_TYPE_TQK7: mul_mat_q_case<GGML_TYPE_TQK7>(ctx, a, s); break;
+#endif
#if defined(STRATA_MMQ_KQUANTS) && defined(STRATA_KQ_DSO)
case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_1: {
const int e = kq::run((int) t, &a, s);
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index c8c7fa72f..3f6ec14db 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -23,6 +23,7 @@
#include "strata/kernels/native_ple_postops.hpp"
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "strata/kernels/qsa.hpp"
#include "strata/kernels/kv_stream.hpp"
#include "strata/kernels/cvec.hpp"
@@ -873,6 +874,7 @@ struct Prefill::Impl {
*shared = nullptr, *sg = nullptr;
int32_t *ids = nullptr, *slot_dev = nullptr, *src_dev = nullptr;
uint16_t *Xs = nullptr, *Hh = nullptr, *sh_h = nullptr;
+ float* Xr = nullptr; // APR: Hadamard-rotated copy of `mixed` for the MMQ path
// step 2b (MMQ): the activations quantized per layer, H in FP32 and its group's quantized rows, the identity
// row map, the group bounds, the group buffers of gathered experts
void *Xq = nullptr, *Hq = nullptr;
@@ -1123,8 +1125,6 @@ const MmqPlan& mmq_plan() {
if (on && lay.native && fused::native_supported(gt, dt)) { p.fo[(size_t) l] = 1; p.any = true; }
continue;
}
- // APR (prism.hadamard): folded layers rotate their activations on the FP16 path only
- if (lay.native && (lay.fmt[(size_t) l].had_gu || lay.fmt[(size_t) l].had_d)) { p.fallback = true; continue; }
p.layer[(size_t) l] = 1;
p.any = true;
p.gu_max = std::max(p.gu_max, mmq::matrix_bytes(gt, 1280, N));
@@ -1443,6 +1443,8 @@ bool Prefill::carve(size_t T, void* alloc) {
m.Xscale = c.take<float>(T * K, ok); m.Hscale = c.take<float>(T * K, ok);
m.grp_tail = c.take<float>(MMQ_GROUP * 4, ok);
m.row_sd = c.take<float>(T * K, ok);
+ // APR: the rotated expert input (only for Hadamard-folded packs)
+ m.Xr = strata::kernels::cpu::hadamard_spec().any() ? c.take<float>(T * N, ok) : nullptr;
}
if (base == nullptr) ok = false;
}
@@ -1458,6 +1460,13 @@ bool Prefill::carve(size_t T, void* alloc) {
if (!m.mmq_ctx) m.mmq_ctx = std::make_unique<mmq::Context>();
}
m.ring = ring_slots(T);
+ { // APR: the device sign vectors now (cudaMalloc), not on first use inside a run
+ const auto& lay = strata::kernels::cpu::expert_layout();
+ for (size_t l = 0; lay.native && l < lay.fmt.size(); ++l) {
+ if (lay.fmt[l].had_gu) strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);
+ if (lay.fmt[l].had_d) strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);
+ }
+ }
if (o.base == nullptr && m.ring > 0) {
// OWNED buffers: the ring in ONE allocation. 384 separate 2.7 MiB cudaMallocs each round up to a 2 MiB page
// (~1.3 MiB a slot, ~0.5 GiB in all) that no count ever saw. A borrowed region keeps its per-slot layout (and
@@ -1727,6 +1736,8 @@ bool Prefill::set_stage_helper(Prefill* helper, std::string& err) {
if (helper == nullptr || helper == this || !split_help_env()) return true;
const Impl& h = *helper->impl_;
if (m.device < 0 || h.device < 0 || h.device == m.device || !mmq_plan().any) return true;
+ // APR: the helper's MMQ share does not rotate the activations of Hadamard-folded experts; not used for those packs
+ if (strata::kernels::cpu::hadamard_spec().any()) return true;
auto pp = std::make_unique<PeerPrefill>();
pp->peer = nullptr; // stream-only: it holds no experts, it streams a share of this stage's
pp->compact = true;
@@ -1839,6 +1850,11 @@ bool Prefill::set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& e
for (const auto& f : lay.fmt)
if (lay.native && f.tail_off) { err = "prefill peer: not for NVFP4 experts yet"; return false; }
}
+ if (strata::kernels::cpu::hadamard_spec().any()) { // APR: the peer's MMQ share does not rotate; declined
+ std::fprintf(stderr, "strata: prefill peer: not used for a Hadamard-folded (APR) pack\n");
+ m.pp.reset();
+ return true;
+ }
auto pp = std::make_unique<PeerPrefill>();
pp->peer = peer;
pp->p2p = peer->p2p();
@@ -3194,11 +3210,12 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// keep MMQ; without the variable nothing here runs. A native pack's layer takes the native kernels
// (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M).
const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
- // APR (prism.hadamard): the rotation is applied on the FP16 path only, so a folded layer never
- // takes MMQ (nor the fused kernels or the peer, which need MMQ)
+ // APR (prism.hadamard): folded gate/up take H (s * x), folded down H (s * h), on the FP16 and MMQ
+ // paths; the fused kernels do not rotate (and have no trellis types), and the multi-GPU prompt
+ // paths are not set up for a folded pack (set_peer, set_stage_helper)
const bool had_gu = lay.native && lay.fmt[(size_t) l].had_gu;
const bool had_d = lay.native && lay.fmt[(size_t) l].had_d;
- const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l]; // never for a folded layer
+ const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];
const int had_block = lay.native ? lay.fmt[(size_t) l].had_block : 0;
const float* had_sx = had_gu ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].gu_signs, N) : nullptr;
const float* had_sh = had_d ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].d_signs, 640) : nullptr;
@@ -3208,7 +3225,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// fused_ring() sized the ring and the buffers for it
const bool no_peer = !core::peer_portable();
const bool fused_only = mmq_plan().any && mmq_plan().fo[(size_t) l];
- const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && fused::native_supported(mmq_gt, mmq_dt);
+ const bool fused_nat = (use_mmq || fused_only) && stream_all && no_peer && lay.native && !had_gu && !had_d &&
+ fused::native_supported(mmq_gt, mmq_dt);
const bool fused_l = (use_mmq && stream_all && no_peer && !lay.native && fused::enabled()) || fused_nat;
// #583 / #954: the fused layout's GU/H/Xq hold the grouping tables and the int8 rows, sized for
// stream_all_min() - 1 tokens of MMQ's rows, so a layer that takes MMQ or the FP16 path at the FULL
@@ -3634,7 +3652,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
pt.mark(kPfGather, cs);
if (use_mmq) {
// step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`
- mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
+ // (APR: from a rotated copy of `mixed`; `mixed` itself feeds the shared expert)
+ const float* xsrc = m.mixed;
+ if (had_gu) {
+ strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);
+ xsrc = m.Xr;
+ }
+ mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down
// reads the group's own quantized H)
const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
@@ -4147,6 +4171,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
m.grp_tail, r0, m.cs);
else
mmq::swiglu(m.GU + r0 * 1280, m.H + r0 * 640, nr, 640, !lay.native, m.cs);
+ if (had_d)
+ strata::kernels::hadamard_rows(m.H + r0 * 640, m.H + r0 * 640, nr, 640, had_block, had_sh, m.cs);
pt.mark(kPfGemmD, cs);
mmq::quantize(m.H + r0 * 640, nullptr, m.Hq, mmq_dt, 640, 640, nr, m.cs, m.Hscale);
mmq::Product dn;
diff --git a/tests/hip/prefill_mmq_parity.cpp b/tests/hip/prefill_mmq_parity.cpp
index c477d5b3e..4a204781d 100644
--- a/tests/hip/prefill_mmq_parity.cpp
+++ b/tests/hip/prefill_mmq_parity.cpp
@@ -10,6 +10,8 @@
#include <algorithm>
#include <cmath>
#include <cstdint>
+#include <cstring>
+#include <cstdlib>
#include <fstream>
#include <iostream>
#include <numeric>
@@ -55,10 +57,10 @@ std::vector<float> decode_matrix(ggml_type type, const std::vector<uint8_t> & q,
return out;
}
-std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial) {
- const ggml_type type = GGML_TYPE_Q2_0;
+std::vector<uint8_t> make_q2_matrix(int64_t rows, int64_t cols, int expert, int trial,
+ ggml_type type = GGML_TYPE_Q2_0) {
const auto * tr = ggml_get_type_traits(type);
- if (!tr || !tr->from_float_ref) throw std::runtime_error("Q2_0 host quantizer unavailable");
+ if (!tr || !tr->from_float_ref) throw std::runtime_error("host quantizer unavailable");
const size_t rb = row_bytes(type, cols);
std::vector<uint8_t> out((size_t) rows * rb);
std::vector<float> row((size_t) cols);
@@ -171,8 +173,17 @@ Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name,
}
}
}
+ {
+ size_t bad = 0, first = got.size();
+ uint32_t first_bits = 0;
+ for (size_t i = 0; i < got.size(); ++i)
+ if (!std::isfinite(got[i])) { if (!bad++) { first = i; std::memcpy(&first_bits, &got[i], 4); } }
+ if (bad)
+ throw std::runtime_error(name + ": " + std::to_string(bad) + " non-finite or unwritten MMQ outputs, first at row " +
+ std::to_string(first / (size_t) out_rows) + " col " + std::to_string(first % (size_t) out_rows) +
+ " bits " + std::to_string(first_bits));
+ }
for (size_t i = 0; i < got.size(); ++i) {
- if (!std::isfinite(got[i])) throw std::runtime_error(name + ": non-finite or unwritten MMQ output");
const double d = (double) got[i] - ref[i];
err2 += d * d; ref2 += (double) ref[i] * ref[i]; max_abs = std::max(max_abs, (float) std::fabs(d));
}
@@ -199,9 +210,10 @@ Metrics run_product(Context & ctx, hipStream_t stream, const std::string & name,
return m;
}
-std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial) {
+std::vector<std::vector<uint8_t>> synthetic_q2(int experts, int64_t out_rows, int64_t cols, int trial,
+ ggml_type type = GGML_TYPE_Q2_0) {
std::vector<std::vector<uint8_t>> w;
- for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial));
+ for (int e = 0; e < experts; ++e) w.push_back(make_q2_matrix(out_rows, cols, e, trial, type));
return w;
}
@@ -311,6 +323,24 @@ int main(int argc, char ** argv) {
1280, 2560, synthetic_of_type(type, 1, 1280, 2560, 5), counts,
make_activations(rows, 2560, 77), src, dst);
}
+ // APR trellis experts (ggml types 144-146), when the build has their MMQ instances (agention fork): gate/up
+ // 1280 x 2560 and down 2560 x 640 (5 blocks per row: the last 256-slice reads past the row, against zero
+ // activations)
+ for (const ggml_type t : {(ggml_type) 145, (ggml_type) 146, (ggml_type) 144}) {
+ if (!supported((int) t)) { std::cout << "type " << (int) t << ": no MMQ instance in this build\n"; continue; }
+ const std::vector<int> counts{2, 3};
+ const auto src = permutation(5, 1, false), dst = permutation(5, 2, true);
+ const std::string tn = ggml_type_name(t);
+ const char* rep = std::getenv("MMQ_PARITY_REPEAT"); // repeat each product (flakiness screen)
+ const int reps = rep ? std::max(1, std::atoi(rep)) : 1;
+ const auto wgu = synthetic_q2(2, 1280, 2560, 5, t), wd = synthetic_q2(2, 2560, 640, 6, t);
+ for (int r = 0; r < reps; ++r) {
+ run_product(ctx, stream, "synthetic-" + tn + "-GU", t, 1280, 2560, wgu, counts,
+ make_activations(5, 2560, 5), src, dst);
+ run_product(ctx, stream, "synthetic-" + tn + "-down", t, 2560, 640, wd, counts,
+ make_activations(5, 640, 6), src, dst);
+ }
+ }
if (argc == 2) run_real_iq3_first_expert(ctx, stream, argv[1]);
}
hip_check(hipStreamSynchronize(stream), "final synchronize");