Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
gfx12: the matrix-core prompt attention (STRATA_HIP_WMMA) is the default; STRATA_HIP_WMMA=0 turns it off
Niko122199f6d4723343
diff --git a/include/strata/kernels/qsa_prompt_attn.hpp b/include/strata/kernels/qsa_prompt_attn.hppindex 046cd6fca..22710d91b 100644--- a/include/strata/kernels/qsa_prompt_attn.hpp+++ b/include/strata/kernels/qsa_prompt_attn.hpp@@ -29,6 +29,11 @@ namespace strata::kernels {bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);+/// HIP only (false in every other build): the prompt attention runs on the RDNA4 matrix cores on the current device -+/// the default on gfx12 (STRATA_HIP_WMMA=0 turns it off), STRATA_HIP_WMMA=1 on gfx11 as well. The prompt path sizes its+/// expert ring for it (prefill.cpp).+bool qsa_hip_wmma_active();+/// STRATA_QSA_SPLIT_ATTN=1 (the idea of fork Eddoursul fd23d91): `qsa_decode_attn_batch`'s decode/// attention through the int8 tensor-core kernel above with each query's cells split among up to 20 blocks, merged with/// the log-sum-exp rescale, its partials in `scratch` (`scratch_floats` available). False (nothingdiff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cuindex 1f18b9c6b..f5397e650 100644--- a/src/kernels/cuda/qsa_prompt_attn.cu+++ b/src/kernels/cuda/qsa_prompt_attn.cu@@ -2175,30 +2175,33 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_wmma_kernel(const float*#endif}-// gfx12 (RDNA4) and gfx11 (RDNA3 / RDNA3.5), only on request: the output differs from the default kernel's in its-// last bits+// gfx12 (RDNA4): on by default (148-amd; STRATA_HIP_WMMA=0 turns it off). gfx11 (RDNA3 / RDNA3.5): only on request+// (STRATA_HIP_WMMA=1). The output differs from the old kernel's in its last bits (FP32-level accuracy, another summation+// order): teacher-forced KL against the old kernel 0.0003 (28 prompts) / 0.0014 (8 long documents) on an R9700.bool hip_wmma_usable() {- static const bool want = [] {+ static const int env = [] { // -1 unset, 0 off, 1 onconst char* e = std::getenv("STRATA_HIP_WMMA");- return e != nullptr && e[0] == '1';+ return e == nullptr || e[0] == '\0' ? -1 : (e[0] == '1' ? 1 : 0);}();- if (!want) return false;- static int arch[64] = {}; // 0 unknown, 1 gfx11 / gfx12, 2 other+ if (env == 0) return false;+ static int arch[64] = {}; // 0 unknown, 1 gfx12, 2 other, 3 gfx11 matrix coresint dev = 0;if (hipGetDevice(&dev) != hipSuccess || dev < 0 || dev >= 64) { (void) hipGetLastError(); return false; }if (arch[dev] == 0) {hipDeviceProp_t prop{};if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return false; }- arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 || strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName)- ? 1 : 2;+ arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 ? 1+ : strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName) ? 3 : 2;static bool told = false;- if (!told) {+ if (!told && (env == 1 || arch[dev] == 1)) {told = true;- std::fprintf(stderr, "strata: STRATA_HIP_WMMA: the prompt attention on matrix cores %s (%s)\n",- arch[dev] == 1 ? "on" : "unavailable", prop.gcnArchName);+ const bool on = arch[dev] == 1 || (arch[dev] == 3 && env == 1);+ std::fprintf(stderr, "strata: %sthe prompt attention on matrix cores %s (%s)%s\n", env == 1 ? "STRATA_HIP_WMMA: " : "",+ on ? "on" : "unavailable", prop.gcnArchName,+ env == 1 ? "" : " by default; STRATA_HIP_WMMA=0 turns it off");}}- return arch[dev] == 1;+ return arch[dev] == 1 || (arch[dev] == 3 && env == 1);}bool launch_wmma(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,@@ -2284,6 +2287,14 @@ bool qsa_decode_attn_split(const float* q, const QsaAttnPools& pools, const int3}void qsa_prompt_attn_set_imma(int on) { g_pa_imma = on < 0 ? -1 : (on ? 1 : 0); }+bool qsa_hip_wmma_active() {+#if defined(__HIPCC__)+ return hip_wmma_usable();+#else+ return false;+#endif+}+bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream) {if (n_q <= 0) return true;diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex bceacbda3..0fe99c128 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -307,13 +307,9 @@ inline int ring_budget_slots() {inline int ring_slots(size_t T) {const char* v = std::getenv("STRATA_PREFILL_RING");#if defined(STRATA_USE_HIP)- // S6: with the opt-in RDNA4 matrix-core attention (STRATA_HIP_WMMA=1) a 96-slot ring: measured with it, 9070 XT- // 4K prompts 718 -> 1,211 tok/s (16K 1,949 -> 2,032), R9700 4K 2,426 -> 2,483 (16K the same)- static const bool wmma = [] {- const char* e = std::getenv("STRATA_HIP_WMMA");- return e != nullptr && e[0] == '1';- }();- if (!v && g_ring_override <= 0 && wmma) return (int64_t) T >= stream_all_min() ? 96 : STAGE;+ // S6: with the RDNA4 matrix-core attention (the gfx12 default since 148-amd; STRATA_HIP_WMMA=1 elsewhere) a 96-slot+ // ring: measured with it, 9070 XT 4K prompts 718 -> 1,211 tok/s (16K 1,949 -> 2,032), R9700 4K 2,426 -> 2,483 (16K the same)+ if (!v && g_ring_override <= 0 && strata::kernels::qsa_hip_wmma_active()) return (int64_t) T >= stream_all_min() ? 96 : STAGE;#endif// the byte budget as this pack's slots: 1024 fused / 384 not on Q2_0, fewer on a pack with bigger blobs. The// unpinned arm stays a slot count (96): it was measured where the host copies are the limit, and there the ringdiff --git a/src/program/generate.cpp b/src/program/generate.cppindex aae516316..cedaf94ad 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -5329,13 +5329,8 @@ int main(int argc, char** argv) {const char* name = named && p.name[0] ? p.name : "(an unnamed GPU)";#if defined(STRATA_USE_HIP)std::fprintf(stderr, "strata generate: GPU %d: %s (%s)\n", dev, name, named ? p.gcnArchName : "?");- // A hint only, never an action: on RDNA4 with an int8 KV the opt-in matrix-core prompt attention reads prompts- // about 30% faster (R9700, PR #1318), but its output differs from the default kernel's in the last bits, so it- // stays the user's choice. Silent when the variable is set to anything (STRATA_HIP_WMMA=0 included).- if (named && o.kv == "int8" && std::strncmp(p.gcnArchName, "gfx12", 5) == 0 && std::getenv("STRATA_HIP_WMMA") == nullptr)- std::fprintf(stderr, "strata generate: hint: STRATA_HIP_WMMA=1 reads prompts about 30%% faster on this card with "- "--kv int8 (prompt attention on the matrix cores); it is off by default because the output "- "bits change (last-place rounding). Nothing was enabled.\n");+ // (RDNA4's matrix-core prompt attention is the default since 148-amd: the engine says so once, at its first prompt;+ // STRATA_HIP_WMMA=0 turns it off.)#if defined(_WIN32)// #468 #461: which HIP runtime was loaded - the bundled one beside the exe, or an AMD driver's System32 copyif (HMODULE h = GetModuleHandleA("amdhip64_7.dll")) {