Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dense GEMM: the chunked prompt's gemm() prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu, now linked) over hipBLAS - 4.97 vs 20-40 TFLOPS at our shapes

Niko1221cca3dd2e33dd
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c4e6e8a18..6e7a9b67b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -822,8 +822,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
endif()
set_source_files_properties(src/dsv41/ds4_main.cpp PROPERTIES COMPILE_DEFINITIONS "DS4_BUILD_ID=\"${STRATA_BUILD_ID}\"")
target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
- ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
- target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/prefill)
+ # strata_prefill: the RDNA3 WMMA dense GEMM (src/prefill/wmma_gemm.cu), which gemm() prefers over hipBLAS on gfx11 (measured
+ # 4.97 -> 20-40 TFLOPS on this part: the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate)
+ target_link_libraries(strata-dsv41 PRIVATE strata_prefill strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
find_package(Threads REQUIRED)
target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)
if(WIN32 AND STRATA_ENABLE_HIP) # cudaMemGetInfo is the DXGI-budgeted hip_compat::mem_get_info (src/core/device.cu): on a Strix Halo the
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 9e983c6fb..807ac0031 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -30,6 +30,7 @@
#include "strata/kernels/f16_bits.hpp"
#include "strata/dsv41/moe_win.hpp"
#include "strata/dsv41/sampler.hpp"
+#include "wmma_gemm.h" // strata_pf_gemm_f16_ld: the RDNA3 WMMA dense GEMM (gemm() prefers it over hipBLAS on gfx11; -Isrc/prefill)
#include <cublas_v2.h>
#include <cuda_runtime.h>
@@ -2972,7 +2973,13 @@ struct Model::Impl {
split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
const size_t C = (size_t) chunk;
c.C = chunk;
- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
+ // the attention slab's rows: 8 MB of q by default. Every dense GEMM of a layer is issued once a slab, so the slab
+ // width divides the prompt's dense weight traffic (the matrices are re-read once a slab): STRATA_DS4_A_MB widens it.
+ static const size_t a_mb = [] {
+ const char* v = std::getenv("STRATA_DS4_A_MB");
+ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 8);
+ }();
+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);
// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
c.w16_elems = (size_t) 8 << 20;
@@ -3041,15 +3048,27 @@ struct Model::Impl {
return true;
}
- // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS
+ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, then the product.
+ // The product prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu) over hipBLAS: on gfx1151 hipBLAS
+ // measures 4.97 TFLOPS at this call convention while that kernel measures 20-40 (BM 128 x BN 256, grouped tile order), and
+ // the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate. Same operands and the same values (the
+ // k order differs, so it is rounding-level: the quality gate and the acceptance gate cover it); STRATA_DENSE_PFGEMM=0 keeps
+ // hipBLAS for an A/B.
bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
std::string& err) {
if (rows <= 0) return true;
+ static const bool pf = [] {
+ const char* v = std::getenv("STRATA_DENSE_PFGEMM");
+ return v == nullptr || v[0] != '0';
+ }();
const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
for (int c0 = 0; c0 < m.n; c0 += cols) {
const int nc = std::min(cols, m.n - c0);
if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
+ if (pf && rows >= 64 && ((uintptr_t) x16 & 15u) == 0 &&
+ strata_pf_gemm_f16_ld(x16, ldx, c.w16, m.k, y + c0, rows, nc, m.k, ldy, beta, d.st))
+ continue;
const float one = 1.0f;
const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,