Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

gfx12: the matrix-core prompt attention (STRATA_HIP_WMMA) is the default; STRATA_HIP_WMMA=0 turns it off

Niko122199f6d4723343
diff --git a/include/strata/kernels/qsa_prompt_attn.hpp b/include/strata/kernels/qsa_prompt_attn.hpp
index 046cd6fca..22710d91b 100644
--- a/include/strata/kernels/qsa_prompt_attn.hpp
+++ b/include/strata/kernels/qsa_prompt_attn.hpp
@@ -29,6 +29,11 @@ namespace strata::kernels {
bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
+/// HIP only (false in every other build): the prompt attention runs on the RDNA4 matrix cores on the current device -
+/// the default on gfx12 (STRATA_HIP_WMMA=0 turns it off), STRATA_HIP_WMMA=1 on gfx11 as well. The prompt path sizes its
+/// expert ring for it (prefill.cpp).
+bool qsa_hip_wmma_active();
+
/// STRATA_QSA_SPLIT_ATTN=1 (the idea of fork Eddoursul fd23d91): `qsa_decode_attn_batch`'s decode
/// attention through the int8 tensor-core kernel above with each query's cells split among up to 20 blocks, merged with
/// the log-sum-exp rescale, its partials in `scratch` (`scratch_floats` available). False (nothing
diff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cu
index 1f18b9c6b..f5397e650 100644
--- a/src/kernels/cuda/qsa_prompt_attn.cu
+++ b/src/kernels/cuda/qsa_prompt_attn.cu
@@ -2175,30 +2175,33 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_wmma_kernel(const float*
#endif
}
-// gfx12 (RDNA4) and gfx11 (RDNA3 / RDNA3.5), only on request: the output differs from the default kernel's in its
-// last bits
+// gfx12 (RDNA4): on by default (148-amd; STRATA_HIP_WMMA=0 turns it off). gfx11 (RDNA3 / RDNA3.5): only on request
+// (STRATA_HIP_WMMA=1). The output differs from the old kernel's in its last bits (FP32-level accuracy, another summation
+// order): teacher-forced KL against the old kernel 0.0003 (28 prompts) / 0.0014 (8 long documents) on an R9700.
bool hip_wmma_usable() {
- static const bool want = [] {
+ static const int env = [] { // -1 unset, 0 off, 1 on
const char* e = std::getenv("STRATA_HIP_WMMA");
- return e != nullptr && e[0] == '1';
+ return e == nullptr || e[0] == '\0' ? -1 : (e[0] == '1' ? 1 : 0);
}();
- if (!want) return false;
- static int arch[64] = {}; // 0 unknown, 1 gfx11 / gfx12, 2 other
+ if (env == 0) return false;
+ static int arch[64] = {}; // 0 unknown, 1 gfx12, 2 other, 3 gfx11 matrix cores
int dev = 0;
if (hipGetDevice(&dev) != hipSuccess || dev < 0 || dev >= 64) { (void) hipGetLastError(); return false; }
if (arch[dev] == 0) {
hipDeviceProp_t prop{};
if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return false; }
- arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 || strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName)
- ? 1 : 2;
+ arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 ? 1
+ : strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName) ? 3 : 2;
static bool told = false;
- if (!told) {
+ if (!told && (env == 1 || arch[dev] == 1)) {
told = true;
- std::fprintf(stderr, "strata: STRATA_HIP_WMMA: the prompt attention on matrix cores %s (%s)\n",
- arch[dev] == 1 ? "on" : "unavailable", prop.gcnArchName);
+ const bool on = arch[dev] == 1 || (arch[dev] == 3 && env == 1);
+ std::fprintf(stderr, "strata: %sthe prompt attention on matrix cores %s (%s)%s\n", env == 1 ? "STRATA_HIP_WMMA: " : "",
+ on ? "on" : "unavailable", prop.gcnArchName,
+ env == 1 ? "" : " by default; STRATA_HIP_WMMA=0 turns it off");
}
}
- return arch[dev] == 1;
+ return arch[dev] == 1 || (arch[dev] == 3 && env == 1);
}
bool launch_wmma(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
@@ -2284,6 +2287,14 @@ bool qsa_decode_attn_split(const float* q, const QsaAttnPools& pools, const int3
}
void qsa_prompt_attn_set_imma(int on) { g_pa_imma = on < 0 ? -1 : (on ? 1 : 0); }
+bool qsa_hip_wmma_active() {
+#if defined(__HIPCC__)
+ return hip_wmma_usable();
+#else
+ return false;
+#endif
+}
+
bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream) {
if (n_q <= 0) return true;
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index bceacbda3..0fe99c128 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -307,13 +307,9 @@ inline int ring_budget_slots() {
inline int ring_slots(size_t T) {
const char* v = std::getenv("STRATA_PREFILL_RING");
#if defined(STRATA_USE_HIP)
- // S6: with the opt-in RDNA4 matrix-core attention (STRATA_HIP_WMMA=1) a 96-slot ring: measured with it, 9070 XT
- // 4K prompts 718 -> 1,211 tok/s (16K 1,949 -> 2,032), R9700 4K 2,426 -> 2,483 (16K the same)
- static const bool wmma = [] {
- const char* e = std::getenv("STRATA_HIP_WMMA");
- return e != nullptr && e[0] == '1';
- }();
- if (!v && g_ring_override <= 0 && wmma) return (int64_t) T >= stream_all_min() ? 96 : STAGE;
+ // S6: with the RDNA4 matrix-core attention (the gfx12 default since 148-amd; STRATA_HIP_WMMA=1 elsewhere) a 96-slot
+ // ring: measured with it, 9070 XT 4K prompts 718 -> 1,211 tok/s (16K 1,949 -> 2,032), R9700 4K 2,426 -> 2,483 (16K the same)
+ if (!v && g_ring_override <= 0 && strata::kernels::qsa_hip_wmma_active()) return (int64_t) T >= stream_all_min() ? 96 : STAGE;
#endif
// the byte budget as this pack's slots: 1024 fused / 384 not on Q2_0, fewer on a pack with bigger blobs. The
// unpinned arm stays a slot count (96): it was measured where the host copies are the limit, and there the ring
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index aae516316..cedaf94ad 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -5329,13 +5329,8 @@ int main(int argc, char** argv) {
const char* name = named && p.name[0] ? p.name : "(an unnamed GPU)";
#if defined(STRATA_USE_HIP)
std::fprintf(stderr, "strata generate: GPU %d: %s (%s)\n", dev, name, named ? p.gcnArchName : "?");
- // A hint only, never an action: on RDNA4 with an int8 KV the opt-in matrix-core prompt attention reads prompts
- // about 30% faster (R9700, PR #1318), but its output differs from the default kernel's in the last bits, so it
- // stays the user's choice. Silent when the variable is set to anything (STRATA_HIP_WMMA=0 included).
- if (named && o.kv == "int8" && std::strncmp(p.gcnArchName, "gfx12", 5) == 0 && std::getenv("STRATA_HIP_WMMA") == nullptr)
- std::fprintf(stderr, "strata generate: hint: STRATA_HIP_WMMA=1 reads prompts about 30%% faster on this card with "
- "--kv int8 (prompt attention on the matrix cores); it is off by default because the output "
- "bits change (last-place rounding). Nothing was enabled.\n");
+ // (RDNA4's matrix-core prompt attention is the default since 148-amd: the engine says so once, at its first prompt;
+ // STRATA_HIP_WMMA=0 turns it off.)
#if defined(_WIN32)
// #468 #461: which HIP runtime was loaded - the bundled one beside the exe, or an AMD driver's System32 copy
if (HMODULE h = GetModuleHandleA("amdhip64_7.dll")) {