Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: mmq::swiglu_quant, swiglu and the down product's q8_1 rows in one kernel (#3752)
sergiywithdd97421d3fbb
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 3628d1443..0ac52a9a5 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -2088,6 +2088,13 @@ if(STRATA_NATIVE_EXPERTS)target_link_libraries(prefill_mmq_scatter_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})add_test(NAME prefill_mmq_scatter_test COMMAND prefill_mmq_scatter_test)set_tests_properties(prefill_mmq_scatter_test PROPERTIES TIMEOUT 120)+ if(STRATA_EXPERIMENTAL_SM60)+ # mmq::swiglu_quant: swiglu + the down product's q8_1 rows in one kernel against the two, byte for byte+ add_executable(prefill_swiglu_quant_test tests/cuda/prefill_swiglu_quant_test.cpp)+ target_link_libraries(prefill_swiglu_quant_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})+ add_test(NAME prefill_swiglu_quant_test COMMAND prefill_swiglu_quant_test)+ set_tests_properties(prefill_swiglu_quant_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120)+ endif()endif()if(STRATA_BUILD_TESTS AND _strata_mmq_tab_on)# STRATA_MMQ_DIRECT: the pointer-table MMQ bit for bit against the gathered MMQ (exit 77 below sm_80)diff --git a/include/strata/prefill/moe_mmq.hpp b/include/strata/prefill/moe_mmq.hppindex fee662fcc..6fba093e5 100644--- a/include/strata/prefill/moe_mmq.hpp+++ b/include/strata/prefill/moe_mmq.hpp@@ -122,6 +122,11 @@ void gather_strata_q2(const uint8_t* blob, void* gu_dst, void* d_dst, void* stre/// h[r, k] = silu(gate) * up of GU rows [2 n_ff wide]: interleaved (gate 2k, up 2k+1: the Strata pack) or split/// (gate k, up n_ff + k: GGUF). FP32 out (the down product's quantizer reads floats).void swiglu(const float* gu, float* h, int64_t rows, int64_t n_ff, bool interleaved, void* stream);+/// swiglu (n_ff 640), then quantize() of its rows for a down product of `ggml_type`, in one pass: H never stored,+/// the same bytes in hq. Only where swiglu_quant_ok (a q8_1 D4 down type; gu 16-byte aligned): CUDA, the+/// STRATA_EXPERIMENTAL_SM60 build.+bool swiglu_quant_ok(int ggml_type);+void swiglu_quant(const float* gu, void* hq, int64_t rows, bool interleaved, void* stream);/// swiglu of an NVFP4 group's gate/up rows (split halves) with scale_gu_rows' scales applied as they are read.void swiglu_scaled(const float* gu, float* h, int64_t rows, int64_t n_ff, const int32_t* bounds, int n,const float* tails, int64_t row0, void* stream);diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cuindex e7ff5d45b..e832cffe3 100644--- a/src/prefill/moe_mmq.cu+++ b/src/prefill/moe_mmq.cu@@ -144,6 +144,54 @@ __global__ void swiglu_kernel(const float* __restrict__ gu, float* __restrict__h[i] = g / (1.0f + __expf(-g)) * u;}+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+// swiglu_kernel (n_ff 640), then quantize_mmq_q8_1<D4>'s rows of its output, in one pass: a block per row (128+// threads, 4 values each, blockIdx.y the row's 512-value half), H never stored. Each value is swiglu_kernel's+// expression and each 32-value block quantize_mmq_q8_1's (the same unit and flags: the same bits).+template <bool IL>+__global__ void __launch_bounds__(128) swiglu_quant_kernel(const float* __restrict__ gu, block_q8_1_mmq* __restrict__ y,+ int64_t rows) {+ constexpr int64_t n_ff = 640, ne0 = 1024; // the padded row: pad512(640)+ const int64_t i0 = ((int64_t) blockDim.x * blockIdx.y + threadIdx.x) * 4;+ if (i0 >= ne0) return;+ const int64_t r = blockIdx.x;+ float4 xi = make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ if (i0 < n_ff) {+ const float* row = gu + r * 2 * n_ff;+ float4 g4, u4;+ if constexpr (IL) { // gate 2k, up 2k + 1+ const float4 a = *reinterpret_cast<const float4*>(row + 2 * i0), b = *reinterpret_cast<const float4*>(row + 2 * i0 + 4);+ g4 = make_float4(a.x, a.z, b.x, b.z);+ u4 = make_float4(a.y, a.w, b.y, b.w);+ } else {+ g4 = *reinterpret_cast<const float4*>(row + i0);+ u4 = *reinterpret_cast<const float4*>(row + n_ff + i0);+ }+ xi.x = g4.x / (1.0f + __expf(-g4.x)) * u4.x;+ xi.y = g4.y / (1.0f + __expf(-g4.y)) * u4.y;+ xi.z = g4.z / (1.0f + __expf(-g4.z)) * u4.z;+ xi.w = g4.w / (1.0f + __expf(-g4.w)) * u4.w;+ }+ float amax = fabsf(xi.x);+ amax = fmaxf(amax, fabsf(xi.y));+ amax = fmaxf(amax, fabsf(xi.z));+ amax = fmaxf(amax, fabsf(xi.w));+#pragma unroll+ for (int offset = 32 / 8; offset > 0; offset >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));+ const float d_inv = 127.0f / amax;+ char4 c;+ c.x = roundf(xi.x * d_inv);+ c.y = roundf(xi.y * d_inv);+ c.z = roundf(xi.z * d_inv);+ c.w = roundf(xi.w * d_inv);+ const float d = 1.0f / d_inv;+ const int64_t k_block = i0 / QK8_1_MMQ, iqs = i0 % QK8_1_MMQ;+ block_q8_1_mmq* yb = y + k_block * rows + r;+ reinterpret_cast<char4*>(yb->qs)[iqs / 4] = c;+ if (iqs % 32 == 0) yb->d4[iqs / 32] = d;+}+#endif+// swiglu_kernel on the MMQ gate/up output with each expert's gate and up scales applied as it is read: the products// scale_gu_rows stored, without the pass that stored them__global__ void swiglu_scaled_kernel(const float* __restrict__ gu, float* __restrict__ h, int64_t rows, int64_t n_ff,@@ -665,6 +713,28 @@ void swiglu(const float* gu, float* h, int64_t rows, int64_t n_ff, bool interleack(cudaGetLastError(), "swiglu");}+bool swiglu_quant_ok(int t) {+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+ // the q8_1 D4 rows quantize() writes for a down product's H (K 640)+ return supported(t) && !fp4_activations(t) && mmq_get_q8_1_ds_layout((ggml_type) t) == MMQ_Q8_1_DS_LAYOUT_D4;+#else+ (void) t;+ return false; // HIP: not checked against the HIP build's quantizer; outside the SM60 build: not built+#endif+}++void swiglu_quant(const float* gu, void* hq, int64_t rows, bool interleaved, void* stream) {+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+ if (rows <= 0) return;+ const dim3 grid((unsigned) rows, 2);+ if (interleaved) swiglu_quant_kernel<true><<<grid, 128, 0, (cudaStream_t) stream>>>(gu, (block_q8_1_mmq*) hq, rows);+ else swiglu_quant_kernel<false><<<grid, 128, 0, (cudaStream_t) stream>>>(gu, (block_q8_1_mmq*) hq, rows);+ ck(cudaGetLastError(), "swiglu_quant");+#else+ (void) gu; (void) hq; (void) rows; (void) interleaved; (void) stream;+#endif+}+void iota(int32_t* dst, int64_t n, void* stream) {if (n <= 0) return;iota_kernel<<<blocks(n), 256, 0, (cudaStream_t) stream>>>(dst, n);diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex a9d121f94..0f3ee3a09 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -87,6 +87,8 @@ bool gather_native_group(const GatherGroup&, size_t, size_t, size_t, size_t, voivoid*, size_t) { return false; }void gather_strata_q2(const uint8_t*, void*, void*, void*) {}void swiglu(const float*, float*, int64_t, int64_t, bool, void*) {}+bool swiglu_quant_ok(int) { return false; }+void swiglu_quant(const float*, void*, int64_t, bool, void*) {}void swiglu_scaled(const float*, float*, int64_t, int64_t, const int32_t*, int, const float*, int64_t, void*) {}void down_row_scales(float*, const int32_t*, int, const float*, int64_t, void*) {}void iota(int32_t*, int64_t, void*) {}diff --git a/tests/cuda/prefill_swiglu_quant_test.cpp b/tests/cuda/prefill_swiglu_quant_test.cppnew file mode 100644index 000000000..0d7b9d686--- /dev/null+++ b/tests/cuda/prefill_swiglu_quant_test.cpp@@ -0,0 +1,94 @@+// prefill_swiglu_quant_test - the prompt path's fused MMQ expert step against the kernels it replaces, byte for byte:+// mmq::swiglu_quant against swiglu + quantize (the down product's q8_1 rows of H), for down types the packs have+// (Q2_0, IQ4_NL, Q8_0), with the gate/up rows split (GGUF) and interleaved (the Strata pack), a zero row and a row+// with one huge value. Exit 77 without a CUDA device.+#include "strata/prefill/moe_mmq.hpp"++#include "ggml.h"++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstring>+#include <random>+#include <stdexcept>+#include <string>+#include <vector>++namespace {+namespace mmq = strata::prefill::mmq;++void ck(cudaError_t e, const char* what) {+ if (e != cudaSuccess) throw std::runtime_error(std::string(what) + ": " + cudaGetErrorString(e));+}++struct Dev {+ void* p = nullptr;+ explicit Dev(size_t n) { ck(cudaMalloc(&p, n), "cudaMalloc"); }+ ~Dev() { cudaFree(p); }+ Dev(const Dev&) = delete;+ Dev& operator=(const Dev&) = delete;+ template <class T> T* as() const { return (T*) p; }+};++int swiglu_quant_case(ggml_type t, int64_t rows, bool interleaved, uint32_t seed) {+ constexpr int64_t NFF = 640;+ std::mt19937 rng(seed);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::uniform_real_distribution<float> mag(0.05f, 40.f);+ std::vector<float> gu((size_t) rows * 2 * NFF);+ for (int64_t r = 0; r < rows; ++r) {+ const float m = mag(rng);+ for (int64_t k = 0; k < 2 * NFF; ++k) gu[(size_t) (r * 2 * NFF + k)] = nd(rng) * m;+ }+ if (rows > 3) { // a zero row, a row with one huge value+ std::memset(gu.data() + 2 * NFF, 0, 2 * NFF * sizeof(float));+ gu[(size_t) (3 * 2 * NFF + 5)] = 3e4f;+ }+ const size_t qb = mmq::q8_bytes(rows, NFF);+ Dev dgu(gu.size() * 4), dh((size_t) rows * NFF * 4 + 16), dq0(qb), dq1(qb);+ ck(cudaMemcpy(dgu.p, gu.data(), gu.size() * 4, cudaMemcpyHostToDevice), "upload");+ ck(cudaMemset(dq0.p, 0x5a, qb), "memset");+ ck(cudaMemset(dq1.p, 0x5a, qb), "memset");+ // the two kernels it replaces, as prefill.cpp launches them+ mmq::swiglu(dgu.as<float>(), dh.as<float>(), rows, NFF, interleaved, nullptr);+ mmq::quantize(dh.as<float>(), nullptr, dq0.p, (int) t, NFF, NFF, rows, nullptr);+ mmq::swiglu_quant(dgu.as<float>(), dq1.p, rows, interleaved, nullptr);+ ck(cudaDeviceSynchronize(), "run");+ const size_t used = (size_t) rows * (1024 / 128) * 144; // the rows' 8 blocks of 128 values (144 bytes each)+ std::vector<uint8_t> a(used), b(used);+ ck(cudaMemcpy(a.data(), dq0.p, used, cudaMemcpyDeviceToHost), "download");+ ck(cudaMemcpy(b.data(), dq1.p, used, cudaMemcpyDeviceToHost), "download");+ size_t bad = 0;+ for (size_t i = 0; i < used; ++i) bad += a[i] != b[i];+ std::printf(" swiglu_quant %-6s %5lld rows%s: %zu of %zu bytes differ\n", ggml_type_name(t), (long long) rows,+ interleaved ? " interleaved" : "", bad, used);+ return bad == 0 ? 0 : 1;+}++} // namespace++int main() {+ int ndev = 0;+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {+ std::printf("prefill_swiglu_quant_test: no CUDA device (skipped)\n");+ return 77;+ }+ int fails = 0;+ try {+ for (ggml_type t : {GGML_TYPE_Q2_0, GGML_TYPE_IQ4_NL, GGML_TYPE_Q8_0}) {+ if (!mmq::swiglu_quant_ok((int) t)) { std::printf(" %s: not on the fused path here\n", ggml_type_name(t)); continue; }+ for (bool il : {false, true}) {+ fails += swiglu_quant_case(t, 1, il, 1);+ fails += swiglu_quant_case(t, 37, il, 2);+ fails += swiglu_quant_case(t, 4113, il, 3);+ }+ }+ } catch (const std::exception& e) {+ std::fprintf(stderr, "prefill_swiglu_quant_test: %s\n", e.what());+ return 1;+ }+ std::printf("prefill_swiglu_quant_test: %s\n", fails ? "FAILED" : "ok");+ return fails ? 1 : 0;+}