Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: opt-in fused int8 prompt experts for the native Q2_0 pack on sm_80+, STRATA_PF_FUSED_Q2N=1 (#3809)

chaog992c47c699486e3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e1aa9ffe8..89b2f6205 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2055,6 +2055,10 @@ if(STRATA_NATIVE_EXPERTS)
target_link_libraries(prefill_fused_iq_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
add_test(NAME prefill_fused_iq_test COMMAND prefill_fused_iq_test)
set_tests_properties(prefill_fused_iq_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
+ # the opt-in STRATA_PF_FUSED_Q2N=1 on its own (the test leaves STRATA_PF_FUSED unset): the native Q2_0 pack's Q2_0 / Q2_0
+ # layers on sm_80+ (77 below sm_80 and without a card)
+ add_test(NAME prefill_fused_iq_test_q2n COMMAND prefill_fused_iq_test "--only=Q2_0 / Q2_0" --unset-pf-fused)
+ set_tests_properties(prefill_fused_iq_test_q2n PROPERTIES ENVIRONMENT "STRATA_PF_FUSED_Q2N=1" SKIP_RETURN_CODE 77 TIMEOUT 600)
# the MoE input quantized per token and scattered to its rows (mmq::quantize_scatter) against the per-row
# gather: the same q8_1 bytes for every MMQ layout (--bench: both at 2K, 8K and 32K tokens)
add_executable(prefill_mmq_scatter_test tests/cuda/prefill_mmq_scatter_test.cpp)
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index a6e35c0b3..22bd4b65f 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -947,7 +947,28 @@ that wins in both directions: `STRATA_SM75_IL`, `STRATA_SM75_EXP`, `STRATA_GR_UP
`STRATA_TOPK_SPLIT`, `STRATA_TOPK_SAMPLE`, `STRATA_ATTN_V3`, `STRATA_Q2_WARP`, `STRATA_FUSE_WAIT=2`, `STRATA_QIL`,
`STRATA_TC_SM75`, `STRATA_PA_V1_PREFETCH`, `STRATA_SCORES_THREAD` (a thread-per-head block scorer; `=1` works on any CUDA card).
Options that change answers stay separate: `STRATA_SELECT_SIMT=1|2`, `STRATA_PA_V2T=1`, `STRATA_PF_FUSED=1 STRATA_PF_FUSED_SM75=1`
-(Q2_0 pack, 7.5 card only) and `STRATA_BF16_TC=1` with `STRATA_XN_F16`.
+(Q2_0 pack, 7.5 card only), `STRATA_PF_FUSED_Q2N=1` (the native Q2_0 pack's prompt experts on sm_80+, below) and
+`STRATA_BF16_TC=1` with `STRATA_XN_F16`.
+
+### The native Q2_0 pack's prompt experts on the fused int8 kernels (`STRATA_PF_FUSED_Q2N=1`, opt-in, sm_80+)
+
+A PC whose CPU lacks the AVX-512 set that setup looks for gets the native Q2_0 pack. On a CUDA card of sm_80 or newer its
+Q2_0 / Q2_0 layers read prompts through MMQ's Q2_0 kernel; the fused int8 kernel for that format (#2145) is set up only on a
+7.5 card. `STRATA_PF_FUSED_Q2N=1` sets it up on any sm_80+ card and takes it for those layers. It has its own variable, so
+`STRATA_PF_FUSED=1` keeps its bits, and `STRATA_PF_FUSED=0` or `STRATA_PF_FUSED_NATIVE=0` still turn it off. Unset, the engine
+is unchanged (first-token logits byte-identical to v0.2.2 on 40 prompts of 1-5K tokens, the author's check). With the switch
+the prompt arithmetic changes and the generated text can change.
+
+The author measured it on an RTX 5080 16 GB (sm_120, Linux, CUDA 13.0), i9-14900K, 2,100 experts in VRAM, CPU prompt share off,
+against the same build unset. At the same 4,864-token chunk (4 interleaved rounds) the 4K, 16K and 23K prompts read 2-3.5%
+faster (4/4 rounds each). With `--prefill auto` (5 rounds) the fused path's smaller buffers make the chunk 5,632 tokens instead of
+4,864 and the 16K prompt reads 32-33% faster (2,592 -> 3,438 tok/s), the 23K one 14%, a 4K one 0-2%; so most of the gain is the
+larger chunk, not the kernel. One layer (`prefill_fused_iq_test`, Q2_0 / Q2_0) runs at 1.92x / 1.37x / 1.30x the MMQ path at
+2,048 / 4,864 / 8,192 tokens. First-token KL against the FP16 prompt path over 30 prompts of 1.5-4.6K tokens: 0.0110 (MMQ
+0.0112), top-1 agreement 28/30 (MMQ 29/30). Not measured: a card that holds every expert, other sm_80+ cards, Windows, the
+generated text. On an RTX A4000 the same kernel in #2145 stopped matching the old output and read 4K prompts 4.5% slower, so
+Ampere is not verified. The ctest entry `prefill_fused_iq_test_q2n` runs the Q2_0 / Q2_0 layer with the switch alone (it skips
+below sm_80 and without a card).
---
diff --git a/src/prefill/moe_fused_iq.cu b/src/prefill/moe_fused_iq.cu
index de2726815..d239972f8 100644
--- a/src/prefill/moe_fused_iq.cu
+++ b/src/prefill/moe_fused_iq.cu
@@ -1188,6 +1188,7 @@ struct DevInfo {
bool done = false, ok = false;
bool w12 = false; // gfx12 (RDNA4): the native kernels only - the Q2_0 pack's (moe_fused.cu) are not ported there
bool t75 = false; // Turing (STRATA_PF_FUSED_SM75): the Q2_0 / Q2_0 layers only (what was set up and checked there)
+ bool q2n = false; // sm_80+ with STRATA_PF_FUSED_Q2N=1: the native Q2_0 pack's Q2_0 gate/up is set up as well (opt-in)
bool s120 = false; // compute capability 12.0 (sm120_series.hpp): the default for the formats measured there
int sms = 0, occ = 1;
};
@@ -1308,7 +1309,12 @@ const DevInfo& dev_info() {
}
d.ok = setup_one<T_IQ2_XXS, true>(occ) && setup_one<T_IQ2_XS, true>(occ) && setup_one<T_IQ2_S, true>(occ) &&
setup_one<T_IQ3_XXS, true>(occ) && setup_one<T_IQ3_S, true>(occ) && setup_one<T_IQ4_XS, true>(occ) &&
- setup_one<T_Q2_0, false>(occ) && setup_one<T_IQ4_NL, false>(occ); // (Q2_0 gate/up is the Turing opt-in's only)
+ setup_one<T_Q2_0, false>(occ) && setup_one<T_IQ4_NL, false>(occ); // (Q2_0 gate/up: the Turing opt-in's, or below)
+ // STRATA_PF_FUSED_Q2N=1: the native Q2_0 pack's Q2_0 gate/up on these kernels here too (what #2145 took with
+ // STRATA_PF_FUSED=1 on every CUDA card until b6443f3a kept it to Turing). Its own variable, so STRATA_PF_FUSED=1 keeps
+ // its bits: unset, the kernel is not set up and the Q2_0 layers keep MMQ. A failed setup leaves occ as it was.
+ static const bool q2n = [] { const char* v = std::getenv("STRATA_PF_FUSED_Q2N"); return v != nullptr && v[0] == '1'; }();
+ if (int o2 = occ; q2n && d.ok && setup_one<T_Q2_0, true>(o2)) { d.q2n = true; occ = o2; }
d.occ = d.ok ? occ : 1;
cudaGetLastError();
#endif
@@ -1332,7 +1338,7 @@ bool f16_d(int t) { return t == T_Q2_0 || t == T_IQ4_NL || t == T_Q8_0; }
bool gu_covered(int t) {
return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS
#if !defined(__HIPCC__)
- || (t == T_Q2_0 && dev_info().t75) // the canonical Q2_0 pack's gate/up: Turing opt-in only (sm_80+ keeps MMQ for it)
+ || (t == T_Q2_0 && (dev_info().t75 || dev_info().q2n)) // Q2_0 gate/up: the Turing opt-in, or STRATA_PF_FUSED_Q2N=1 on sm_80+
#endif
#if defined(__HIPCC__)
|| (t == T_Q2_0 && dev_info().w12) // gfx12: the native Q2_0 pack's gate/up on native_w11_kernel (64-value blocks)
@@ -1411,12 +1417,15 @@ bool native_supported(int gu_type, int d_type) {
// agent, needles, CJK and tool calls the same). IQ2_XXS / IQ2_XS gate/up (the IQ3_XXS and IQ2_XS packs) keep MMQ until
// measured. Every other CUDA card: opt-in as before.
if (env < 0) {
+ // the native Q2_0 pack's Q2_0 / Q2_0 layers: only with STRATA_PF_FUSED_Q2N=1 (d.q2n), on any sm_80+ card
+ if (gu_type == T_Q2_0) { const DevInfo& d = dev_info(); return d.ok && d.q2n && d_type == T_Q2_0 && d_covered_on(d_type, d); }
if (gu_type != T_IQ2_S && gu_type != T_IQ3_XXS && gu_type != T_IQ3_S && gu_type != T_IQ4_XS) return false;
const DevInfo& d = dev_info();
// 12.0 cards with 80+ SMs only (RTX PRO 5000 / 6000, RTX 5080 / 5090): on an RTX 5070 (48 SMs, IQ2_XS pack, experts
// partly in RAM) the fused kernels were 4.7% slower on 20K prompts (0/6 pairs) - it keeps MMQ unless STRATA_PF_FUSED=1
return d.ok && d.s120 && d.sms >= 80 && !d.t75 && d_covered_on(d_type, d);
}
+ if (gu_type == T_Q2_0 && d_type != T_Q2_0) return false; // the opt-in (STRATA_PF_FUSED=1 as well) takes the Q2_0 / Q2_0 pair only
#endif
// default: only these three; the Q2_0 one (148-amd) is on Linux only: on Windows it is opt-in (STRATA_PF_FUSED=1) until tested there
if (env < 0 && gu_type != T_IQ3_S && gu_type != T_IQ3_XXS && !(gu_type == T_Q2_0 && strata::core::kGfx12PromptDefaultsOn)) return false;
diff --git a/tests/cuda/prefill_fused_iq_test.cpp b/tests/cuda/prefill_fused_iq_test.cpp
index e07051e8a..aba76925c 100644
--- a/tests/cuda/prefill_fused_iq_test.cpp
+++ b/tests/cuda/prefill_fused_iq_test.cpp
@@ -13,7 +13,9 @@
// different expert ranges. Part 2 (timing): one layer at the engine's chunks (2048, 3584, 8192 tokens; 512 experts,
// top 10) on both paths and on MMQ's products alone (no gathers), for the IQ2_XS, IQ3_XXS and IQ3_S packs' most
// common layers, with the grouping checked and a sample of rows against the reference. --no-ref, --no-timing,
-// --only=NAME, --chunks=A,B. Exit 77 without a CUDA device of sm_80 or newer.
+// --only=NAME, --chunks=A,B, --unset-pf-fused (STRATA_PF_FUSED left unset instead of set to 1: the engine's default, for the
+// STRATA_PF_FUSED_Q2N=1 ctest entry). Exit 77 without a CUDA device of sm_80 or newer.
+#include "strata/core/emulate.hpp"
#include "strata/prefill/moe_fused_iq.hpp"
#include "strata/prefill/moe_mmq.hpp"
@@ -643,10 +645,15 @@ void timing_part(const Pair& p, int T, cudaStream_t s) {
int main(int argc, char** argv) {
try {
+ // --unset-pf-fused: STRATA_PF_FUSED unset, as the engine's default (the ctest entry for the opt-in STRATA_PF_FUSED_Q2N=1,
+ // whose Q2_0 / Q2_0 pair the engine takes with the old variable unset or at 1)
+ bool unset_pf = false;
+ for (int i = 1; i < argc; ++i)
+ if (std::strcmp(argv[i], "--unset-pf-fused") == 0) unset_pf = true;
#ifdef _WIN32
- _putenv_s("STRATA_PF_FUSED", "1");
+ if (unset_pf) _putenv_s("STRATA_PF_FUSED", ""); else _putenv_s("STRATA_PF_FUSED", "1"); // (an empty value removes it)
#else
- setenv("STRATA_PF_FUSED", "1", 1);
+ if (unset_pf) unsetenv("STRATA_PF_FUSED"); else setenv("STRATA_PF_FUSED", "1", 1);
setenv("STRATA_PF_FUSED_KQ", "1", 1); // (the HIP build's Q4_K / Q5_K / Q5_1 / Q8_0 pairs)
#endif
int n = 0;
@@ -656,6 +663,12 @@ int main(int argc, char** argv) {
std::printf("the fused kernels need sm_80 or newer (or gfx11 / gfx12 matrix cores): skipped\n");
return 77;
}
+ // with STRATA_PF_FUSED unset nothing is set up below sm_80 (Turing's pair is STRATA_PF_FUSED=1 STRATA_PF_FUSED_SM75=1's)
+ if (unset_pf) {
+ int major = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, 0);
+ if (strata::cc_major_of(major) < 8) { std::printf("STRATA_PF_FUSED unset: nothing is set up below sm_80 (skipped)\n"); return 77; }
+ }
if (!mmq::built()) { std::printf("no MMQ in this build\n"); return 1; }
const std::vector<Pair> pairs = {
{GGML_TYPE_IQ2_S, GGML_TYPE_Q2_0, "IQ2_S / Q2_0"}, {GGML_TYPE_IQ2_XXS, GGML_TYPE_Q2_0, "IQ2_XXS / Q2_0"},
@@ -691,8 +704,8 @@ int main(int argc, char** argv) {
for (const Pair& p : pairs) {
if (only ? std::string(p.name).find(only) == std::string::npos : p.gu == GGML_TYPE_Q4_K || p.gu == GGML_TYPE_Q5_K) continue;
if (!fused::native_supported((int) p.gu, (int) p.d)) {
- // the canonical Q2_0 pack's Q2_0 gate/up is the Turing opt-in's (STRATA_PF_FUSED_SM75=1), and on Turing that is
- // the only pair set up: skip what this device does not take (--only=NAME still insists)
+ // the Q2_0 / Q2_0 pair is the Turing opt-in's (STRATA_PF_FUSED_SM75=1; the only pair set up there) or, on sm_80+,
+ // STRATA_PF_FUSED_Q2N=1's: skip what this device does not take (--only=NAME still insists)
const bool turing_pair = p.gu == GGML_TYPE_Q2_0 && p.d == GGML_TYPE_Q2_0;
int major = 0;
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, 0);