Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: mmq::swiglu_quant, swiglu and the down product's q8_1 rows in one kernel (#3752)

sergiywithdd97421d3fbb
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3628d1443..0ac52a9a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2088,6 +2088,13 @@ if(STRATA_NATIVE_EXPERTS)
target_link_libraries(prefill_mmq_scatter_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
add_test(NAME prefill_mmq_scatter_test COMMAND prefill_mmq_scatter_test)
set_tests_properties(prefill_mmq_scatter_test PROPERTIES TIMEOUT 120)
+ if(STRATA_EXPERIMENTAL_SM60)
+ # mmq::swiglu_quant: swiglu + the down product's q8_1 rows in one kernel against the two, byte for byte
+ add_executable(prefill_swiglu_quant_test tests/cuda/prefill_swiglu_quant_test.cpp)
+ target_link_libraries(prefill_swiglu_quant_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+ add_test(NAME prefill_swiglu_quant_test COMMAND prefill_swiglu_quant_test)
+ set_tests_properties(prefill_swiglu_quant_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)
+ endif()
endif()
if(STRATA_BUILD_TESTS AND _strata_mmq_tab_on)
# STRATA_MMQ_DIRECT: the pointer-table MMQ bit for bit against the gathered MMQ (exit 77 below sm_80)
diff --git a/include/strata/prefill/moe_mmq.hpp b/include/strata/prefill/moe_mmq.hpp
index fee662fcc..6fba093e5 100644
--- a/include/strata/prefill/moe_mmq.hpp
+++ b/include/strata/prefill/moe_mmq.hpp
@@ -122,6 +122,11 @@ void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stre
/// h[r, k] = silu(gate) * up of GU rows [2 n_ff wide]: interleaved (gate 2k, up 2k+1: the Strata pack) or split
/// (gate k, up n_ff + k: GGUF). FP32 out (the down product's quantizer reads floats).
void swiglu(const float* gu, float* h, int64_t rows, int64_t n_ff, bool interleaved, void* stream);
+/// swiglu (n_ff 640), then quantize() of its rows for a down product of `ggml_type`, in one pass: H never stored,
+/// the same bytes in hq. Only where swiglu_quant_ok (a q8_1 D4 down type; gu 16-byte aligned): CUDA, the
+/// STRATA_EXPERIMENTAL_SM60 build.
+bool swiglu_quant_ok(int ggml_type);
+void swiglu_quant(const float* gu, void* hq, int64_t rows, bool interleaved, void* stream);
/// swiglu of an NVFP4 group's gate/up rows (split halves) with scale_gu_rows' scales applied as they are read.
void swiglu_scaled(const float* gu, float* h, int64_t rows, int64_t n_ff, const int32_t* bounds, int n,
const float* tails, int64_t row0, void* stream);
diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cu
index e7ff5d45b..e832cffe3 100644
--- a/src/prefill/moe_mmq.cu
+++ b/src/prefill/moe_mmq.cu
@@ -144,6 +144,54 @@ __global__ void swiglu_kernel(const float* __restrict__ gu, float* __restrict__
h[i] = g / (1.0f + __expf(-g)) * u;
}
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+// swiglu_kernel (n_ff 640), then quantize_mmq_q8_1<D4>'s rows of its output, in one pass: a block per row (128
+// threads, 4 values each, blockIdx.y the row's 512-value half), H never stored. Each value is swiglu_kernel's
+// expression and each 32-value block quantize_mmq_q8_1's (the same unit and flags: the same bits).
+template <bool IL>
+__global__ void __launch_bounds__(128) swiglu_quant_kernel(const float* __restrict__ gu, block_q8_1_mmq* __restrict__ y,
+ int64_t rows) {
+ constexpr int64_t n_ff = 640, ne0 = 1024; // the padded row: pad512(640)
+ const int64_t i0 = ((int64_t) blockDim.x * blockIdx.y + threadIdx.x) * 4;
+ if (i0 >= ne0) return;
+ const int64_t r = blockIdx.x;
+ float4 xi = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ if (i0 < n_ff) {
+ const float* row = gu + r * 2 * n_ff;
+ float4 g4, u4;
+ if constexpr (IL) { // gate 2k, up 2k + 1
+ const float4 a = *reinterpret_cast<const float4*>(row + 2 * i0), b = *reinterpret_cast<const float4*>(row + 2 * i0 + 4);
+ g4 = make_float4(a.x, a.z, b.x, b.z);
+ u4 = make_float4(a.y, a.w, b.y, b.w);
+ } else {
+ g4 = *reinterpret_cast<const float4*>(row + i0);
+ u4 = *reinterpret_cast<const float4*>(row + n_ff + i0);
+ }
+ xi.x = g4.x / (1.0f + __expf(-g4.x)) * u4.x;
+ xi.y = g4.y / (1.0f + __expf(-g4.y)) * u4.y;
+ xi.z = g4.z / (1.0f + __expf(-g4.z)) * u4.z;
+ xi.w = g4.w / (1.0f + __expf(-g4.w)) * u4.w;
+ }
+ float amax = fabsf(xi.x);
+ amax = fmaxf(amax, fabsf(xi.y));
+ amax = fmaxf(amax, fabsf(xi.z));
+ amax = fmaxf(amax, fabsf(xi.w));
+#pragma unroll
+ for (int offset = 32 / 8; offset > 0; offset >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
+ const float d_inv = 127.0f / amax;
+ char4 c;
+ c.x = roundf(xi.x * d_inv);
+ c.y = roundf(xi.y * d_inv);
+ c.z = roundf(xi.z * d_inv);
+ c.w = roundf(xi.w * d_inv);
+ const float d = 1.0f / d_inv;
+ const int64_t k_block = i0 / QK8_1_MMQ, iqs = i0 % QK8_1_MMQ;
+ block_q8_1_mmq* yb = y + k_block * rows + r;
+ reinterpret_cast<char4*>(yb->qs)[iqs / 4] = c;
+ if (iqs % 32 == 0) yb->d4[iqs / 32] = d;
+}
+#endif
+
// swiglu_kernel on the MMQ gate/up output with each expert's gate and up scales applied as it is read: the products
// scale_gu_rows stored, without the pass that stored them
__global__ void swiglu_scaled_kernel(const float* __restrict__ gu, float* __restrict__ h, int64_t rows, int64_t n_ff,
@@ -665,6 +713,28 @@ void swiglu(const float* gu, float* h, int64_t rows, int64_t n_ff, bool interlea
ck(cudaGetLastError(), "swiglu");
}
+bool swiglu_quant_ok(int t) {
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+ // the q8_1 D4 rows quantize() writes for a down product's H (K 640)
+ return supported(t) && !fp4_activations(t) && mmq_get_q8_1_ds_layout((ggml_type) t) == MMQ_Q8_1_DS_LAYOUT_D4;
+#else
+ (void) t;
+ return false; // HIP: not checked against the HIP build's quantizer; outside the SM60 build: not built
+#endif
+}
+
+void swiglu_quant(const float* gu, void* hq, int64_t rows, bool interleaved, void* stream) {
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+ if (rows <= 0) return;
+ const dim3 grid((unsigned) rows, 2);
+ if (interleaved) swiglu_quant_kernel<true><<<grid, 128, 0, (cudaStream_t) stream>>>(gu, (block_q8_1_mmq*) hq, rows);
+ else swiglu_quant_kernel<false><<<grid, 128, 0, (cudaStream_t) stream>>>(gu, (block_q8_1_mmq*) hq, rows);
+ ck(cudaGetLastError(), "swiglu_quant");
+#else
+ (void) gu; (void) hq; (void) rows; (void) interleaved; (void) stream;
+#endif
+}
+
void iota(int32_t* dst, int64_t n, void* stream) {
if (n <= 0) return;
iota_kernel<<<blocks(n), 256, 0, (cudaStream_t) stream>>>(dst, n);
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index a9d121f94..0f3ee3a09 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -87,6 +87,8 @@ bool gather_native_group(const GatherGroup&, size_t, size_t, size_t, size_t, voi
void*, size_t) { return false; }
void gather_strata_q2(const uint8_t*, void*, void*, void*) {}
void swiglu(const float*, float*, int64_t, int64_t, bool, void*) {}
+bool swiglu_quant_ok(int) { return false; }
+void swiglu_quant(const float*, void*, int64_t, bool, void*) {}
void swiglu_scaled(const float*, float*, int64_t, int64_t, const int32_t*, int, const float*, int64_t, void*) {}
void down_row_scales(float*, const int32_t*, int, const float*, int64_t, void*) {}
void iota(int32_t*, int64_t, void*) {}
diff --git a/tests/cuda/prefill_swiglu_quant_test.cpp b/tests/cuda/prefill_swiglu_quant_test.cpp
new file mode 100644
index 000000000..0d7b9d686
--- /dev/null
+++ b/tests/cuda/prefill_swiglu_quant_test.cpp
@@ -0,0 +1,94 @@
+// prefill_swiglu_quant_test - the prompt path's fused MMQ expert step against the kernels it replaces, byte for byte:
+// mmq::swiglu_quant against swiglu + quantize (the down product's q8_1 rows of H), for down types the packs have
+// (Q2_0, IQ4_NL, Q8_0), with the gate/up rows split (GGUF) and interleaved (the Strata pack), a zero row and a row
+// with one huge value. Exit 77 without a CUDA device.
+#include "strata/prefill/moe_mmq.hpp"
+
+#include "ggml.h"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+namespace mmq = strata::prefill::mmq;
+
+void ck(cudaError_t e, const char* what) {
+ if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));
+}
+
+struct Dev {
+ void* p = nullptr;
+ explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }
+ ~Dev() { cudaFree(p); }
+ Dev(const Dev&) = delete;
+ Dev& operator=(const Dev&) = delete;
+ template <class T> T* as() const { return (T*) p; }
+};
+
+int swiglu_quant_case(ggml_type t, int64_t rows, bool interleaved, uint32_t seed) {
+ constexpr int64_t NFF = 640;
+ std::mt19937 rng(seed);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::uniform_real_distribution<float> mag(0.05f, 40.f);
+ std::vector<float> gu((size_t) rows * 2 * NFF);
+ for (int64_t r = 0; r < rows; ++r) {
+ const float m = mag(rng);
+ for (int64_t k = 0; k < 2 * NFF; ++k) gu[(size_t) (r * 2 * NFF + k)] = nd(rng) * m;
+ }
+ if (rows > 3) { // a zero row, a row with one huge value
+ std::memset(gu.data() + 2 * NFF, 0, 2 * NFF * sizeof(float));
+ gu[(size_t) (3 * 2 * NFF + 5)] = 3e4f;
+ }
+ const size_t qb = mmq::q8_bytes(rows, NFF);
+ Dev dgu(gu.size() * 4), dh((size_t) rows * NFF * 4 + 16), dq0(qb), dq1(qb);
+ ck(cudaMemcpy(dgu.p, gu.data(), gu.size() * 4, cudaMemcpyHostToDevice), "upload");
+ ck(cudaMemset(dq0.p, 0x5a, qb), "memset");
+ ck(cudaMemset(dq1.p, 0x5a, qb), "memset");
+ // the two kernels it replaces, as prefill.cpp launches them
+ mmq::swiglu(dgu.as<float>(), dh.as<float>(), rows, NFF, interleaved, nullptr);
+ mmq::quantize(dh.as<float>(), nullptr, dq0.p, (int) t, NFF, NFF, rows, nullptr);
+ mmq::swiglu_quant(dgu.as<float>(), dq1.p, rows, interleaved, nullptr);
+ ck(cudaDeviceSynchronize(), "run");
+ const size_t used = (size_t) rows * (1024 / 128) * 144; // the rows' 8 blocks of 128 values (144 bytes each)
+ std::vector<uint8_t> a(used), b(used);
+ ck(cudaMemcpy(a.data(), dq0.p, used, cudaMemcpyDeviceToHost), "download");
+ ck(cudaMemcpy(b.data(), dq1.p, used, cudaMemcpyDeviceToHost), "download");
+ size_t bad = 0;
+ for (size_t i = 0; i < used; ++i) bad += a[i] != b[i];
+ std::printf(" swiglu_quant %-6s %5lld rows%s: %zu of %zu bytes differ\n", ggml_type_name(t), (long long) rows,
+ interleaved ? " interleaved" : "", bad, used);
+ return bad == 0 ? 0 : 1;
+}
+
+} // namespace
+
+int main() {
+ int ndev = 0;
+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
+ std::printf("prefill_swiglu_quant_test: no CUDA device (skipped)\n");
+ return 77;
+ }
+ int fails = 0;
+ try {
+ for (ggml_type t : {GGML_TYPE_Q2_0, GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0}) {
+ if (!mmq::swiglu_quant_ok((int) t)) { std::printf(" %s: not on the fused path here\n", ggml_type_name(t)); continue; }
+ for (bool il : {false, true}) {
+ fails += swiglu_quant_case(t, 1, il, 1);
+ fails += swiglu_quant_case(t, 37, il, 2);
+ fails += swiglu_quant_case(t, 4113, il, 3);
+ }
+ }
+ } catch (const std::exception& e) {
+ std::fprintf(stderr, "prefill_swiglu_quant_test: %s\n", e.what());
+ return 1;
+ }
+ std::printf("prefill_swiglu_quant_test: %s\n", fails ? "FAILED" : "ok");
+ return fails ? 1 : 0;
+}