Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dense GEMM: the chunked prompt's gemm() prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu, now linked) over hipBLAS - 4.97 vs 20-40 TFLOPS at our shapes
Niko1221cca3dd2e33dd
diff --git a/CMakeLists.txt b/CMakeLists.txtindex c4e6e8a18..6e7a9b67b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -822,8 +822,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)endif()set_source_files_properties(src/dsv41/ds4_main.cpp PROPERTIES COMPILE_DEFINITIONS "DS4_BUILD_ID=\"${STRATA_BUILD_ID}\"")target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}- ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)- target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})+ ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml ${CMAKE_CURRENT_SOURCE_DIR}/src/prefill)+ # strata_prefill: the RDNA3 WMMA dense GEMM (src/prefill/wmma_gemm.cu), which gemm() prefers over hipBLAS on gfx11 (measured+ # 4.97 -> 20-40 TFLOPS on this part: the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate)+ target_link_libraries(strata-dsv41 PRIVATE strata_prefill strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})find_package(Threads REQUIRED)target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)if(WIN32 AND STRATA_ENABLE_HIP) # cudaMemGetInfo is the DXGI-budgeted hip_compat::mem_get_info (src/core/device.cu): on a Strix Halo thediff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 9e983c6fb..807ac0031 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -30,6 +30,7 @@#include "strata/kernels/f16_bits.hpp"#include "strata/dsv41/moe_win.hpp"#include "strata/dsv41/sampler.hpp"+#include "wmma_gemm.h" // strata_pf_gemm_f16_ld: the RDNA3 WMMA dense GEMM (gemm() prefers it over hipBLAS on gfx11; -Isrc/prefill)#include <cublas_v2.h>#include <cuda_runtime.h>@@ -2972,7 +2973,13 @@ struct Model::Impl {split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)const size_t C = (size_t) chunk;c.C = chunk;- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);+ // the attention slab's rows: 8 MB of q by default. Every dense GEMM of a layer is issued once a slab, so the slab+ // width divides the prompt's dense weight traffic (the matrices are re-read once a slab): STRATA_DS4_A_MB widens it.+ static const size_t a_mb = [] {+ const char* v = std::getenv("STRATA_DS4_A_MB");+ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 8);+ }();+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));c.w16_elems = (size_t) 8 << 20;@@ -3041,15 +3048,27 @@ struct Model::Impl {return true;}- // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS+ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, then the product.+ // The product prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu) over hipBLAS: on gfx1151 hipBLAS+ // measures 4.97 TFLOPS at this call convention while that kernel measures 20-40 (BM 128 x BN 256, grouped tile order), and+ // the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate. Same operands and the same values (the+ // k order differs, so it is rounding-level: the quality gate and the acceptance gate cover it); STRATA_DENSE_PFGEMM=0 keeps+ // hipBLAS for an A/B.bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,std::string& err) {if (rows <= 0) return true;+ static const bool pf = [] {+ const char* v = std::getenv("STRATA_DENSE_PFGEMM");+ return v == nullptr || v[0] != '0';+ }();const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);for (int c0 = 0; c0 < m.n; c0 += cols) {const int nc = std::min(cols, m.n - c0);if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);+ if (pf && rows >= 64 && ((uintptr_t) x16 & 15u) == 0 &&+ strata_pf_gemm_f16_ld(x16, ldx, c.w16, m.k, y + c0, rows, nc, m.k, ldy, beta, d.st))+ continue;const float one = 1.0f;const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,