Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
gfx12: STRATA_HIP_WMMA=auto (prompt attention on matrix cores chosen per call, FP32 fallback on a failed launch); Split traits for Q4_K/Q5_K/Q8_0 (STRATA_KQ_SPLIT=1, bitwise, opt-in) with iq_multi_parity cases
Niko122179ac3164a060
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex b2f74b56d..c8e2b90ff 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -808,6 +808,11 @@ template<int TY> struct Split;// per-entry ones, which call Fmt<TY>::dot per column exactly as before #242 (the launchers test kSplit at compile// time, so the multi kernels are never instantiated for a type without a Split).template<int TY> inline constexpr bool kSplit = false;+// The K-quant / Q8_0 / Q5_1 formats (UD-Q4_K_XL) have Split traits (SplitK below), but their decode-once multi kernels+// run only with STRATA_KQ_SPLIT=1 until the A/B on the measured cards decides (bitwise the old kernels either way).+template<int TY> inline constexpr bool kq_type = (TY == 12 || TY == 13 || TY == 8); // not Q5_1: its Split apply is not bitwise the old dot (FMA contraction), see notes-151-amd+inline bool g_kq_split = [] { const char* v = std::getenv("STRATA_KQ_SPLIT"); return v != nullptr && v[0] == '1'; }();+template<int TY> inline bool split_off() { return kq_type<TY> && !g_kq_split; }template<int TY> inline constexpr bool kStageIqGrid = (TY == 16 || TY == 17 || TY == 18 || TY == 21 || TY == 22 || TY == 29);template<> inline constexpr bool kSplit<16> = true;@@ -1185,6 +1190,141 @@ template<> struct Split<42> { // Q2_0return r.d2 * d8 * sumi;}};+template<int TY> struct S27;+template<> struct S27<12> { // Q4_K+ struct W { int v[2]; uint16_t aux[2]; half2 dm; };+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {+ const block_q4_K* b = (const block_q4_K*) vbq + kbx;+ W r;+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));+ const int* q4 = (const int*) (b->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));+ r.v[0] = q4[0];+ r.v[1] = q4[4];+ k_scale_min(b->scales, bq8_offset, r.aux);+ r.dm = b->dm;+ return r;+ }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));+ int u[2 * QR4_K];+ float d8[QR4_K];+#pragma unroll+ for (int i = 0; i < QR4_K; ++i) {+ const block_q8_1* bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);+ const int* q8 = (const int*) bq8i->qs + ((iqs / 2) % 4);+ u[2 * i + 0] = q8[0];+ u[2 * i + 1] = q8[4];+ }+ const uint8_t* sc = (const uint8_t*) r.aux;+ const uint8_t* m = sc + 2;+ return vec_dot_q4_K_q8_1_impl_vmmq(r.v, u, sc, m, r.dm, d8);+ }+};+template<> struct S27<13> { // Q5_K+ struct W { int vl[2]; int vh[2]; uint16_t aux[2]; half2 dm; };+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {+ const block_q5_K* b = (const block_q5_K*) vbq + kbx;+ W r;+ const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));+ const int* ql = (const int*) (b->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));+ const int* qh = (const int*) (b->qh + 4 * ((iqs / 2) % 4));+ r.vl[0] = ql[0];+ r.vl[1] = ql[4];+ r.vh[0] = qh[0] >> bq8_offset;+ r.vh[1] = qh[4] >> bq8_offset;+ k_scale_min(b->scales, bq8_offset, r.aux);+ r.dm = b->dm;+ return r;+ }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));+ int u[2 * QR5_K];+ float d8[QR5_K];+#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const block_q8_1* bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);+ const int* q8 = (const int*) bq8i->qs + ((iqs / 2) % 4);+ u[2 * i + 0] = q8[0];+ u[2 * i + 1] = q8[4];+ }+ const uint8_t* sc = (const uint8_t*) r.aux;+ const uint8_t* m = sc + 2;+ return vec_dot_q5_K_q8_1_impl_vmmq(r.vl, r.vh, u, sc, m, r.dm, d8);+ }+};+template<> struct S27<7> { // Q5_1+ struct W { int vl[VDR_Q5_1]; int qh; half2 dm; };+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {+ const block_q5_1* b = (const block_q5_1*) vbq + kbx;+ W r;+#pragma unroll+ for (int i = 0; i < VDR_Q5_1; ++i) r.vl[i] = get_int_b4(b->qs, iqs + i);+ r.qh = get_int_b4(b->qh, 0);+ r.dm = b->dm;+ return r;+ }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ int sumi = 0, sumu = 0;+#pragma unroll+ for (int i = 0; i < VDR_Q5_1; ++i) {+ const int vl = r.vl[i];+ const int vh = r.qh >> (4 * (iqs + i));+ const int u0 = get_int_b4(bq8_1->qs, iqs + i), u1 = get_int_b4(bq8_1->qs, iqs + i + QI5_1);+ int vi0 = (vl >> 0) & 0x0F0F0F0F;+ vi0 |= (vh << 4) & 0x00000010;+ vi0 |= (vh << 11) & 0x00001000;+ vi0 |= (vh << 18) & 0x00100000;+ vi0 |= (vh << 25) & 0x10000000;+ sumi = ggml_cuda_dp4a(vi0, u0, sumi);+ int vi1 = (vl >> 4) & 0x0F0F0F0F;+ vi1 |= (vh >> 12) & 0x00000010;+ vi1 |= (vh >> 5) & 0x00001000;+ vi1 |= (vh << 2) & 0x00100000;+ vi1 |= (vh << 9) & 0x10000000;+ sumi = ggml_cuda_dp4a(vi1, u1, sumi);+ sumu = ggml_cuda_dp4a(0x01010101, u1, ggml_cuda_dp4a(0x01010101, u0, sumu));+ }+ const float2 dm5 = __half22float2(r.dm);+ const float d8 = __low2float(bq8_1->ds);+ return sumi * (dm5.x * d8) + sumu * (dm5.y * d8);+ }+};+template<> struct S27<8> { // Q8_0+ struct W { int q[VDR_Q8_0]; float d; };+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {+ const block_q8_0* b = (const block_q8_0*) vbq + kbx;+ W r;+#pragma unroll+ for (int i = 0; i < VDR_Q8_0; ++i) r.q[i] = get_int_b2(b->qs, iqs + i);+ r.d = __half2float(b->d);+ return r;+ }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {+ int sumi = 0;+#pragma unroll+ for (int i = 0; i < VDR_Q8_0; ++i) sumi = ggml_cuda_dp4a(r.q[i], get_int_b4(bq8_1->qs, iqs + i), sumi);+ const float d8_1 = __low2float(bq8_1->ds);+ return r.d * d8_1 * ((float) sumi);+ }+};+// K-quants and Q5_1 / Q8_0 (UD-Q4_K_XL): the S27 load / apply (below the same file used them first for the V2K+// grouped kernels) as Split traits, so the generic multi-column kernels decode a weight once per call for these+// formats too. Bitwise the old per-column Fmt<TY>::dot (the S27 apply is that dot's own expression; iq_multi_parity).+// STRATA_OLD_IQ_MMVQ=1 keeps the old kernels; STRATA_KQ_SPLIT=0 keeps them for these four formats only.+template<int TY> struct SplitK : S27<TY> {+ template<bool STAGE_GRID = false>+ __device__ static typename S27<TY>::W load(const void* __restrict__ vbq, int kbx, int iqs, const uint32_t* __restrict__ = nullptr) {+ return S27<TY>::load(vbq, kbx, iqs);+ }+};+template<> inline constexpr bool kSplit<12> = true;+template<> inline constexpr bool kSplit<13> = true;+template<> inline constexpr bool kSplit<8> = true;+template<> struct Split<12> : SplitK<12> {};+template<> struct Split<13> : SplitK<13> {};+template<> struct Split<8> : SplitK<8> {};template<> inline constexpr bool kSplit<40> = true;template<> struct Split<40> { // NVFP4: vec_dot_nvfp4_q8_1's two 16-value sub-blocks, weights decoded once per callstruct W { int2 v0[2], v1[2]; float sc[2]; };@@ -2909,7 +3049,7 @@ int d_qk(int t) {}bool gu_split(int t) {switch (t) {-#define STRATA_SP(T) case T: return kSplit<T>;+#define STRATA_SP(T) case T: return kSplit<T> && !split_off<T>();STRATA_GU_FMTS(STRATA_SP)#undef STRATA_SPdefault: return false;@@ -2971,7 +3111,7 @@ void launch_mmvq(const uint8_t* W, size_t rb, const block_q8_1* X, float* y, intcudaStream_t s) {const dim3 grid((unsigned) ((n_out + 3) / 4)), block(32, 4);if constexpr (!kSplit<TY>) mmvq_kernel<TY><<<grid, block, 0, s>>>(W, rb, X, y, n_in, n_out, ncols);- else if (g_old_kernels) mmvq_kernel<TY><<<grid, block, 0, s>>>(W, rb, X, y, n_in, n_out, ncols);+ else if (g_old_kernels || split_off<TY>()) mmvq_kernel<TY><<<grid, block, 0, s>>>(W, rb, X, y, n_in, n_out, ncols);else if constexpr (kStageIqGrid<TY>) {if (!g_stage_grid_mmvq) {launch_mmvq_multi_nc<TY, false>(grid, block, s, W, rb, X, y, n_in, n_out, ncols);@@ -3333,125 +3473,6 @@ void s26_launch_l(const NativeExpertLayout& L, int64_t cap_groups, cudaStream_t// words re-read per column). Each Fmt<TY>::dot is split into S27<TY>::load (everything that depends on the weight) and// apply (the activation words and the same integer / float expression), the same impl functions, so a row's lane-strided// k order, its terms and warp_sum are the old kernels': every output is bitwise the old one's (iq harness, memcmp).-template<int TY> struct S27;-template<> struct S27<12> { // Q4_K- struct W { int v[2]; uint16_t aux[2]; half2 dm; };- __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {- const block_q4_K* b = (const block_q4_K*) vbq + kbx;- W r;- const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));- const int* q4 = (const int*) (b->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));- r.v[0] = q4[0];- r.v[1] = q4[4];- k_scale_min(b->scales, bq8_offset, r.aux);- r.dm = b->dm;- return r;- }- __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {- const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));- int u[2 * QR4_K];- float d8[QR4_K];-#pragma unroll- for (int i = 0; i < QR4_K; ++i) {- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);- const int* q8 = (const int*) bq8i->qs + ((iqs / 2) % 4);- u[2 * i + 0] = q8[0];- u[2 * i + 1] = q8[4];- }- const uint8_t* sc = (const uint8_t*) r.aux;- const uint8_t* m = sc + 2;- return vec_dot_q4_K_q8_1_impl_vmmq(r.v, u, sc, m, r.dm, d8);- }-};-template<> struct S27<13> { // Q5_K- struct W { int vl[2]; int vh[2]; uint16_t aux[2]; half2 dm; };- __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {- const block_q5_K* b = (const block_q5_K*) vbq + kbx;- W r;- const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));- const int* ql = (const int*) (b->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));- const int* qh = (const int*) (b->qh + 4 * ((iqs / 2) % 4));- r.vl[0] = ql[0];- r.vl[1] = ql[4];- r.vh[0] = qh[0] >> bq8_offset;- r.vh[1] = qh[4] >> bq8_offset;- k_scale_min(b->scales, bq8_offset, r.aux);- r.dm = b->dm;- return r;- }- __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {- const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));- int u[2 * QR5_K];- float d8[QR5_K];-#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);- const int* q8 = (const int*) bq8i->qs + ((iqs / 2) % 4);- u[2 * i + 0] = q8[0];- u[2 * i + 1] = q8[4];- }- const uint8_t* sc = (const uint8_t*) r.aux;- const uint8_t* m = sc + 2;- return vec_dot_q5_K_q8_1_impl_vmmq(r.vl, r.vh, u, sc, m, r.dm, d8);- }-};-template<> struct S27<7> { // Q5_1- struct W { int vl[VDR_Q5_1]; int qh; half2 dm; };- __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {- const block_q5_1* b = (const block_q5_1*) vbq + kbx;- W r;-#pragma unroll- for (int i = 0; i < VDR_Q5_1; ++i) r.vl[i] = get_int_b4(b->qs, iqs + i);- r.qh = get_int_b4(b->qh, 0);- r.dm = b->dm;- return r;- }- __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {- int sumi = 0, sumu = 0;-#pragma unroll- for (int i = 0; i < VDR_Q5_1; ++i) {- const int vl = r.vl[i];- const int vh = r.qh >> (4 * (iqs + i));- const int u0 = get_int_b4(bq8_1->qs, iqs + i), u1 = get_int_b4(bq8_1->qs, iqs + i + QI5_1);- int vi0 = (vl >> 0) & 0x0F0F0F0F;- vi0 |= (vh << 4) & 0x00000010;- vi0 |= (vh << 11) & 0x00001000;- vi0 |= (vh << 18) & 0x00100000;- vi0 |= (vh << 25) & 0x10000000;- sumi = ggml_cuda_dp4a(vi0, u0, sumi);- int vi1 = (vl >> 4) & 0x0F0F0F0F;- vi1 |= (vh >> 12) & 0x00000010;- vi1 |= (vh >> 5) & 0x00001000;- vi1 |= (vh << 2) & 0x00100000;- vi1 |= (vh << 9) & 0x10000000;- sumi = ggml_cuda_dp4a(vi1, u1, sumi);- sumu = ggml_cuda_dp4a(0x01010101, u1, ggml_cuda_dp4a(0x01010101, u0, sumu));- }- const float2 dm5 = __half22float2(r.dm);- const float d8 = __low2float(bq8_1->ds);- return sumi * (dm5.x * d8) + sumu * (dm5.y * d8);- }-};-template<> struct S27<8> { // Q8_0- struct W { int q[VDR_Q8_0]; float d; };- __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs) {- const block_q8_0* b = (const block_q8_0*) vbq + kbx;- W r;-#pragma unroll- for (int i = 0; i < VDR_Q8_0; ++i) r.q[i] = get_int_b2(b->qs, iqs + i);- r.d = __half2float(b->d);- return r;- }- __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) {- int sumi = 0;-#pragma unroll- for (int i = 0; i < VDR_Q8_0; ++i) sumi = ggml_cuda_dp4a(r.q[i], get_int_b4(bq8_1->qs, iqs + i), sumi);- const float d8_1 = __low2float(bq8_1->ds);- return r.d * d8_1 * ((float) sumi);- }-};template<int TG, bool LX, int RPW, bool TS>__global__ void __launch_bounds__(256) s27_gu_kernel(const unsigned long long* __restrict__ grp_ptr,@@ -3790,7 +3811,7 @@ void launch_gu(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, con// (qk 128, ipb 8: 160 dots at n_embd 2560) take the full-warp multi kernel, as before the sub-16 kernels.const bool g_no_sub16_gu = ::strata::kernels::g_no_sub16_gu || (L.n_embd / Fmt<TG>::qk) * Fmt<TG>::ipb != 80;if constexpr (!kSplit<TG>) native_gu_kernel<TG><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);- else if (g_old_kernels) native_gu_kernel<TG><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);+ else if (g_old_kernels || split_off<TG>()) native_gu_kernel<TG><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);else if constexpr (kStageIqGrid<TG>) {if (!g_stage_grid || ((TG == 18 || TG == 21) && !stage_tables_on())) {if (g_no_sub16_gu) native_gu_multi_kernel<TG, false, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_tok, X, L, gate, up);@@ -3835,7 +3856,9 @@ bool launch_gu_swiglu(const NativeExpertLayout& L, int64_t gy, cudaStream_t s, cif constexpr (!kSplit<TG>) {return false;} else {- if (g_old_kernels || g_no_sub16_gu || L.n_embd != 2560 || L.n_ff % 32 != 0 || L.n_ff <= 0) return false;+ if (g_old_kernels || split_off<TG>() || g_no_sub16_gu || L.n_embd != 2560 || L.n_ff % 32 != 0 || L.n_ff <= 0) return false;+ // the 16-lane rows assume 80 dots a row (as launch_gu): the 256-value K-quants have 160+ if (kq_type<TG> && (L.n_embd / Fmt<TG>::qk) * Fmt<TG>::ipb != 80) return false;const dim3 grid((unsigned) (L.n_ff / 32), (unsigned) gy);bool stage = false;if constexpr (kStageIqGrid<TG>) stage = g_stage_grid && !((TG == 18 || TG == 21) && !stage_tables_on());@@ -3854,7 +3877,7 @@ void launch_down(dim3 grid, cudaStream_t s, const unsigned long long* grp_ptr, cconst int32_t* n_groups, const int32_t* ent_dst, const block_q8_1* hq, const NativeExpertLayout& L,float* out) {if constexpr (!kSplit<TD>) native_down_kernel<TD><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);- else if (g_old_kernels) native_down_kernel<TD><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);+ else if (g_old_kernels || split_off<TD>()) native_down_kernel<TD><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);else if (g_no_sub16_gu) native_down_multi_kernel<TD, false><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);else native_down_multi_kernel<TD, true><<<grid, 256, 0, s>>>(grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);}@@ -4810,7 +4833,7 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long longconst bool v1 = g_grouped_v1;const int64_t gy = (v1 || grid_groups <= 0 || grid_groups > cap_groups) ? cap_groups : grid_groups;const int gu_rows = (!g_old_kernels && !g_no_sub16_gu && L.n_embd == 2560 && gu_split(L.gu_type) &&- gu_qk(L.gu_type) != 128) ? 16 : GU_ROWS; // qk 128 (APR trellis): no sub-16 rows, see launch_gu+ gu_qk(L.gu_type) != 128 && L.gu_type != 12 && L.gu_type != 13) ? 16 : GU_ROWS; // qk 128 (APR trellis): no sub-16 rows, see launch_guconst dim3 ggu((unsigned) ((2 * L.n_ff + gu_rows - 1) / gu_rows), (unsigned) gy);#if STRATA_EXP_LAYOUTS#if defined(STRATA_HIP_GFX906)diff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cuindex 1f18b9c6b..3583f834a 100644--- a/src/kernels/cuda/qsa_prompt_attn.cu+++ b/src/kernels/cuda/qsa_prompt_attn.cu@@ -14,6 +14,7 @@#include <algorithm>#include <cmath>+#include <atomic>#include <cstdio>#include <cstdlib>#include <cstring>@@ -2176,29 +2177,57 @@ __global__ void __launch_bounds__(THREADS) prompt_attn_wmma_kernel(const float*}// gfx12 (RDNA4) and gfx11 (RDNA3 / RDNA3.5), only on request: the output differs from the default kernel's in its-// last bits-bool hip_wmma_usable() {- static const bool want = [] {+// last bits. STRATA_HIP_WMMA=1 forces it on every call; =auto takes it where it was measured to beat the FP32 kernel+// (hip_wmma_auto_pick: a gfx12 card and a batch of STRATA_HIP_WMMA_MIN_Q queries or more; other calls keep the FP32+// kernel); unset or =0 keeps the FP32 kernel. A launch that fails switches the path off for the rest of the process+// (the caller then runs the FP32 kernel) instead of ending it.+int hip_wmma_mode_env() {+ static const int mode = [] {const char* e = std::getenv("STRATA_HIP_WMMA");- return e != nullptr && e[0] == '1';+ if (e == nullptr) return 0;+ if (e[0] == '1') return 1;+ if (!std::strcmp(e, "auto")) return 2;+ return 0;}();- if (!want) return false;- static int arch[64] = {}; // 0 unknown, 1 gfx11 / gfx12, 2 other- int dev = 0;- if (hipGetDevice(&dev) != hipSuccess || dev < 0 || dev >= 64) { (void) hipGetLastError(); return false; }+ return mode;+}+std::atomic<bool> g_wmma_failed{false};+// the device's class: 1 gfx11 matrix-core part, 2 gfx12 (RDNA4), 3 anything else+int hip_wmma_arch(int dev) {+ static int arch[64] = {};+ if (dev < 0 || dev >= 64) return 3;if (arch[dev] == 0) {hipDeviceProp_t prop{};- if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return false; }- arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 || strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName)- ? 1 : 2;+ if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return 3; }+ arch[dev] = std::strncmp(prop.gcnArchName, "gfx12", 5) == 0 ? 2+ : strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName) ? 1 : 3;static bool told = false;if (!told) {told = true;- std::fprintf(stderr, "strata: STRATA_HIP_WMMA: the prompt attention on matrix cores %s (%s)\n",- arch[dev] == 1 ? "on" : "unavailable", prop.gcnArchName);+ const int mode = hip_wmma_mode_env();+ std::fprintf(stderr, "strata: STRATA_HIP_WMMA=%s: the prompt attention on matrix cores %s (%s)\n",+ mode == 2 ? "auto" : "1", arch[dev] == 3 ? "unavailable" : (mode == 2 ? "chosen per call" : "on"),+ prop.gcnArchName);}}- return arch[dev] == 1;+ return arch[dev];+}+// auto: where the matrix-core kernel beats the FP32 one. Measured on the R9700 (gfx1201) only; the gfx11 parts take+// it only when forced with =1.+bool hip_wmma_auto_pick(int arch, int64_t n_q, int64_t cap) {+ if (arch != 2) return false;+ static const int64_t min_q = [] { const char* e = std::getenv("STRATA_HIP_WMMA_MIN_Q"); return e ? (int64_t) std::atoll(e) : (int64_t) 64; }();+ static const int64_t min_cap = [] { const char* e = std::getenv("STRATA_HIP_WMMA_MIN_CAP"); return e ? (int64_t) std::atoll(e) : (int64_t) 0; }();+ return n_q >= min_q && cap >= min_cap;+}+bool hip_wmma_usable(int64_t n_q, int64_t cap) {+ const int mode = hip_wmma_mode_env();+ if (mode == 0 || g_wmma_failed.load(std::memory_order_relaxed)) return false;+ int dev = 0;+ if (hipGetDevice(&dev) != hipSuccess) { (void) hipGetLastError(); return false; }+ const int arch = hip_wmma_arch(dev);+ if (arch == 3) return false;+ return mode == 1 || hip_wmma_auto_pick(arch, n_q, cap);}bool launch_wmma(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,@@ -2219,8 +2248,9 @@ bool launch_wmma(const float* q, const QsaAttnPools& pools, const int32_t* ids,}const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) {- std::fprintf(stderr, "qsa_prompt_attn_batch (wmma): %s\n", cudaGetErrorString(e));- std::exit(1);+ std::fprintf(stderr, "qsa_prompt_attn_batch (wmma): %s; the FP32 prompt attention runs from here on\n", cudaGetErrorString(e));+ g_wmma_failed.store(true);+ return false;}return true;}@@ -2324,7 +2354,7 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3(void) turing; (void) volta;if (pools.k_q != nullptr && pools.v_q != nullptr && pools.k_scale != nullptr && pools.v_scale != nullptr &&pools.k_q4 == nullptr && pools.v_q4 == nullptr && s.head_dim == HD && s.n_head == (int64_t) G * s.n_head_kv &&- cap > 0 && ids && steps && pools.page_table && hip_wmma_usable())+ cap > 0 && ids && steps && pools.page_table && hip_wmma_usable(n_q, cap))return launch_wmma(q, pools, ids, steps, cap, s, attn, n_q, (cudaStream_t) stream);return false;#endifdiff --git a/src/kernels/iq_multi_parity.cpp b/src/kernels/iq_multi_parity.cppindex 27b06d1e4..cc0ae16df 100644--- a/src/kernels/iq_multi_parity.cpp+++ b/src/kernels/iq_multi_parity.cpp@@ -62,10 +62,14 @@ const char* name_of(int t) {case 23: return "IQ4_XS";case 29: return "IQ1_M";case 42: return "Q2_0";+ case 12: return "Q4_K";+ case 13: return "Q5_K";+ case 7: return "Q5_1";+ case 8: return "Q8_0";default: return "?";}}-int block_values(int t) { return t == 20 ? 32 : t == 42 ? 64 : 256; }+int block_values(int t) { return (t == 20 || t == 7 || t == 8) ? 32 : t == 42 ? 64 : 256; }// `rows` rows of `n` values of format t: random bytes, then a finite fp16 scale in every blockstd::vector<uint8_t> random_rows(int t, int64_t rows, int64_t n, std::mt19937& rng) {@@ -82,6 +86,10 @@ std::vector<uint8_t> random_rows(int t, int64_t rows, int64_t n, std::mt19937& r} else {const uint16_t h = (uint16_t) ((sgn(rng) == 0 ? 0x8000 : 0) | (ex(rng) << 10) | man(rng)); // 2^-13 .. 2^-6std::memcpy(&w[o], &h, 2);+ if (t == 12 || t == 13 || t == 7) { // Q4_K / Q5_K (d, dmin) and Q5_1 (d, m): the second half is finite too+ const uint16_t h2 = (uint16_t) ((sgn(rng) == 0 ? 0x8000 : 0) | (ex(rng) << 10) | man(rng));+ std::memcpy(&w[o + 2], &h2, 2);+ }}}return w;@@ -409,7 +417,7 @@ int main(int argc, char** argv) {cudaStream_t s;ck(cudaStreamCreate(&s), "stream");std::mt19937 rng(18);- for (int t : {16, 17, 18, 20, 21, 22, 23, 29, 42}) {+ for (int t : {16, 17, 18, 20, 21, 22, 23, 29, 42, 12, 13, 7, 8}) {check_mmvq(t, 2560, 67, s, rng); // the model's n_embd; 67 rows: a partial block of 4 rowscheck_mmvq(t, 1024, 5, s, rng);}@@ -417,6 +425,11 @@ int main(int argc, char** argv) {for (int dt : {20, 42}) check_grouped(gu, dt, 2560, 640, s, rng); // the model's shapecheck_grouped(gu, 23, 1024, 512, s, rng); // IQ4_XS down needs n_ff % 256 == 0}+ // UD-Q4_K_XL (run with STRATA_KQ_SPLIT=1 to compare the Split path with the old kernels; STRATA_EXPERT_V2K=0 / 1 picks+ // the generic multi kernel or the V2K one on gfx12: both are bitwise the old kernels)+ for (int gu : {12, 13})+ for (int dt : {7, 8}) check_grouped(gu, dt, 2560, 640, s, rng);+ for (int gu : {12, 13}) check_grouped(gu, 13, 2048, 512, s, rng); // Qwen3.6-35B-A3B's geometrycheck_q8_1_finite(s, rng);if (do_bench) bench(s, rng);std::printf("iq_multi_parity: %d failures\n", g_fail);diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 5fee9fe98..7138b85b3 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -312,7 +312,7 @@ inline int ring_slots(size_t T) {// 4K prompts 718 -> 1,211 tok/s (16K 1,949 -> 2,032), R9700 4K 2,426 -> 2,483 (16K the same)static const bool wmma = [] {const char* e = std::getenv("STRATA_HIP_WMMA");- return e != nullptr && e[0] == '1';+ return e != nullptr && (e[0] == '1' || std::strcmp(e, "auto") == 0);}();if (!v && g_ring_override <= 0 && wmma) return (int64_t) T >= stream_all_min() ? 96 : STAGE;#endif