Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: gfx11 WMMA expert kernels for the prompt sets, on by default on gfx1151 (Strix Halo: prompts +22..29% at the same KL), opt-in on other gfx11 parts; STRATA_DS4_HIPK=0 turns them off
Niko1221e6c59970919f
diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.incindex 29abeda85..4fd9c3f40 100644--- a/src/dsv41/ds4_iq_hip.inc+++ b/src/dsv41/ds4_iq_hip.inc@@ -26,6 +26,20 @@ __device__ __forceinline__ fv8 wmma16(hv8 a, hv8 b, fv8 c) {return c;#endif}+using hv16 = _Float16 __attribute__((ext_vector_type(16)));+// RDNA3 / RDNA3.5 (gfx11): the same v_wmma_f32_16x16x16_f16 on wave32, but a lane holds ALL 16 k of its row / column (the two+// half-waves the same values) and the D rows a lane holds are 2 q + hi (gfx12: 8 hi + q). DS4_HIPK_G11 is set in the device pass only.+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || defined(__gfx1150__) || \+ defined(__gfx1151__) || defined(__gfx1152__)+#define DS4_HIPK_G11 1+#endif+__device__ __forceinline__ fv8 wmma16g11(hv16 a, hv16 b, fv8 c) {+#if defined(DS4_HIPK_G11)+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);+#else+ return c;+#endif+}__device__ __forceinline__ uint32_t as_u32(hv2 v) { return __builtin_bit_cast(uint32_t, v); }__device__ __forceinline__ hv2 as_hv2(uint32_t v) { return __builtin_bit_cast(hv2, v); }// 4 int8 codes (a word) -> 4 halves times the scale: 1024 + (code + 128) is exact in a half's mantissa, less 1152@@ -225,11 +239,32 @@ inline bool ok_device() { // gfx12 unless STRATA_DS4_HIPK=0int v = state[dev].load(std::memory_order_relaxed);if (v == 0) {cudaDeviceProp prop{};- const bool gfx12 = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;+ const bool ok = cudaGetDeviceProperties(&prop, dev) == cudaSuccess;+ const bool gfx12 = ok && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;+ const bool gfx11 = ok && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0;const char* e = std::getenv("STRATA_DS4_HIPK");v = gfx12 && !(e != nullptr && e[0] == '0') ? 1 : 2; // (on by default on gfx12; STRATA_DS4_HIPK=0 off)+ // gfx11: the register-weights tiles only. On by default on gfx1151 (Strix Halo: prompts +22..29% at the same KL, measured), opt-in+ // (STRATA_DS4_HIPK=1) on the other gfx11 parts, which were not measured; STRATA_DS4_HIPK=0 turns it off everywhere+ if (gfx11) {+ const bool halo = std::strncmp(prop.gcnArchName, "gfx1151", 7) == 0;+ v = (e != nullptr ? e[0] == '1' : halo) ? 3 : 2;+ }state[dev].store(v, std::memory_order_relaxed);}+ return v == 1 || v == 3;+}+inline bool is_g11() {+ int dev = 0;+ cudaGetDevice(&dev);+ cudaDeviceProp prop{};+ static std::atomic<int> g[16];+ if (dev < 0 || dev >= 16) return false;+ int v = g[dev].load(std::memory_order_relaxed);+ if (v == 0) {+ v = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0 ? 1 : 2;+ g[dev].store(v, std::memory_order_relaxed);+ }return v == 1;}@@ -252,6 +287,7 @@ inline bool launch(const SetArgs& a, cudaStream_t s) {if (K % 256 != 0 || N % 64 != 0 || a.ldx % 8 != 0 || a.ldo % 2 != 0) return false;if (((uintptr_t) (GU ? (const void*) a.xf : (const void*) a.xh) & 15) != 0) return false;static const int var = std::getenv("STRATA_DS4_HIPK") ? std::atoi(std::getenv("STRATA_DS4_HIPK")) : 0;+ if (is_g11() && var == 2) return false; // (the shared-memory weights kernel is gfx12 only)if (var != 2) { // 2: the shared-memory weights kernelstatic const int cfg = std::getenv("HIPK_CFG") ? std::atoi(std::getenv("HIPK_CFG")) : 0;if (cfg == 1 && a.max_rows <= 32) return runr<T, GU, 32, 4, 1>(a, s);diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.incindex 736606605..38747c0f5 100644--- a/src/dsv41/ds4_iq_hip_r.inc+++ b/src/dsv41/ds4_iq_hip_r.inc@@ -109,7 +109,11 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgfetchx(s + 1, R);load_stage(nxt, s + 1);}+#if defined(DS4_HIPK_G11)+ hv16 bf[NM][NTW]; // (gfx11: all 16 k of the step in a lane: its own 8 and the other half-wave's)+#elsehv8 bf[NM][NTW];+#endifauto step = [&](auto tt) {constexpr int t = decltype(tt)::value;constexpr int sub = t >> 1, ks = t & 1;@@ -122,17 +126,41 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgstep_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);const hv2 h = {(_Float16) sc, (_Float16) sc};const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);+#if defined(DS4_HIPK_G11)+ const uint32_t own[4] = {u0.x, u0.y, u1.x, u1.y};+ uint32_t both[8];+#pragma unroll+ for (int w = 0; w < 4; ++w) {+ const uint32_t other = __shfl_xor_sync(0xffffffffu, own[w], 16);+ both[w] = hi ? other : own[w];+ both[4 + w] = hi ? own[w] : other;+ }+ bf[m][j] = __builtin_bit_cast(hv16, both);+#elsebf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));+#endif}+#if defined(DS4_HIPK_G11)+ const int kcol = sub * 32 + ks * 16;+#elseconst int kcol = sub * 32 + ks * 16 + hi * 8;+#endif#pragma unrollfor (int i = 0; i < MT; ++i) {if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)+#if defined(DS4_HIPK_G11)+ const hv16 af = *(const hv16*) &xs[b][i * 16 + l16][kcol];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16g11(af, bf[m][j], acc[m][i][j]);+#elseconst hv8 af = *(const hv8*) &xs[b][i * 16 + l16][kcol];#pragma unrollfor (int m = 0; m < NM; ++m)#pragma unrollfor (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16(af, bf[m][j], acc[m][i][j]);+#endif}}};@@ -153,7 +181,11 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbgfor (int i = 0; i < MT; ++i)#pragma unrollfor (int q = 0; q < 8; ++q) {+#if defined(DS4_HIPK_G11)+ const int row = i * 16 + 2 * q + hi;+#elseconst int row = i * 16 + 8 * hi + q;+#endifif (row >= nr) continue;const size_t lr = (size_t) (r0 + row);#pragma unroll