Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: gfx11 WMMA expert kernels for the prompt sets, on by default on gfx1151 (Strix Halo: prompts +22..29% at the same KL), opt-in on other gfx11 parts; STRATA_DS4_HIPK=0 turns them off

Niko1221e6c59970919f
diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.inc
index 29abeda85..4fd9c3f40 100644
--- a/src/dsv41/ds4_iq_hip.inc
+++ b/src/dsv41/ds4_iq_hip.inc
@@ -26,6 +26,20 @@ __device__ __forceinline__ fv8 wmma16(hv8 a, hv8 b, fv8 c) {
return c;
#endif
}
+using hv16 = _Float16 __attribute__((ext_vector_type(16)));
+// RDNA3 / RDNA3.5 (gfx11): the same v_wmma_f32_16x16x16_f16 on wave32, but a lane holds ALL 16 k of its row / column (the two
+// half-waves the same values) and the D rows a lane holds are 2 q + hi (gfx12: 8 hi + q). DS4_HIPK_G11 is set in the device pass only.
+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || defined(__gfx1150__) || \
+ defined(__gfx1151__) || defined(__gfx1152__)
+#define DS4_HIPK_G11 1
+#endif
+__device__ __forceinline__ fv8 wmma16g11(hv16 a, hv16 b, fv8 c) {
+#if defined(DS4_HIPK_G11)
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#else
+ return c;
+#endif
+}
__device__ __forceinline__ uint32_t as_u32(hv2 v) { return __builtin_bit_cast(uint32_t, v); }
__device__ __forceinline__ hv2 as_hv2(uint32_t v) { return __builtin_bit_cast(hv2, v); }
// 4 int8 codes (a word) -> 4 halves times the scale: 1024 + (code + 128) is exact in a half's mantissa, less 1152
@@ -225,11 +239,32 @@ inline bool ok_device() { // gfx12 unless STRATA_DS4_HIPK=0
int v = state[dev].load(std::memory_order_relaxed);
if (v == 0) {
cudaDeviceProp prop{};
- const bool gfx12 = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;
+ const bool ok = cudaGetDeviceProperties(&prop, dev) == cudaSuccess;
+ const bool gfx12 = ok && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;
+ const bool gfx11 = ok && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0;
const char* e = std::getenv("STRATA_DS4_HIPK");
v = gfx12 && !(e != nullptr && e[0] == '0') ? 1 : 2; // (on by default on gfx12; STRATA_DS4_HIPK=0 off)
+ // gfx11: the register-weights tiles only. On by default on gfx1151 (Strix Halo: prompts +22..29% at the same KL, measured), opt-in
+ // (STRATA_DS4_HIPK=1) on the other gfx11 parts, which were not measured; STRATA_DS4_HIPK=0 turns it off everywhere
+ if (gfx11) {
+ const bool halo = std::strncmp(prop.gcnArchName, "gfx1151", 7) == 0;
+ v = (e != nullptr ? e[0] == '1' : halo) ? 3 : 2;
+ }
state[dev].store(v, std::memory_order_relaxed);
}
+ return v == 1 || v == 3;
+}
+inline bool is_g11() {
+ int dev = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceProp prop{};
+ static std::atomic<int> g[16];
+ if (dev < 0 || dev >= 16) return false;
+ int v = g[dev].load(std::memory_order_relaxed);
+ if (v == 0) {
+ v = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx11", 5) == 0 ? 1 : 2;
+ g[dev].store(v, std::memory_order_relaxed);
+ }
return v == 1;
}
@@ -252,6 +287,7 @@ inline bool launch(const SetArgs& a, cudaStream_t s) {
if (K % 256 != 0 || N % 64 != 0 || a.ldx % 8 != 0 || a.ldo % 2 != 0) return false;
if (((uintptr_t) (GU ? (const void*) a.xf : (const void*) a.xh) & 15) != 0) return false;
static const int var = std::getenv("STRATA_DS4_HIPK") ? std::atoi(std::getenv("STRATA_DS4_HIPK")) : 0;
+ if (is_g11() && var == 2) return false; // (the shared-memory weights kernel is gfx12 only)
if (var != 2) { // 2: the shared-memory weights kernel
static const int cfg = std::getenv("HIPK_CFG") ? std::atoi(std::getenv("HIPK_CFG")) : 0;
if (cfg == 1 && a.max_rows <= 32) return runr<T, GU, 32, 4, 1>(a, s);
diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.inc
index 736606605..38747c0f5 100644
--- a/src/dsv41/ds4_iq_hip_r.inc
+++ b/src/dsv41/ds4_iq_hip_r.inc
@@ -109,7 +109,11 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
fetchx(s + 1, R);
load_stage(nxt, s + 1);
}
+#if defined(DS4_HIPK_G11)
+ hv16 bf[NM][NTW]; // (gfx11: all 16 k of the step in a lane: its own 8 and the other half-wave's)
+#else
hv8 bf[NM][NTW];
+#endif
auto step = [&](auto tt) {
constexpr int t = decltype(tt)::value;
constexpr int sub = t >> 1, ks = t & 1;
@@ -122,17 +126,41 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);
const hv2 h = {(_Float16) sc, (_Float16) sc};
const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);
+#if defined(DS4_HIPK_G11)
+ const uint32_t own[4] = {u0.x, u0.y, u1.x, u1.y};
+ uint32_t both[8];
+#pragma unroll
+ for (int w = 0; w < 4; ++w) {
+ const uint32_t other = __shfl_xor_sync(0xffffffffu, own[w], 16);
+ both[w] = hi ? other : own[w];
+ both[4 + w] = hi ? own[w] : other;
+ }
+ bf[m][j] = __builtin_bit_cast(hv16, both);
+#else
bf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));
+#endif
}
+#if defined(DS4_HIPK_G11)
+ const int kcol = sub * 32 + ks * 16;
+#else
const int kcol = sub * 32 + ks * 16 + hi * 8;
+#endif
#pragma unroll
for (int i = 0; i < MT; ++i) {
if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)
+#if defined(DS4_HIPK_G11)
+ const hv16 af = *(const hv16*) &xs[b][i * 16 + l16][kcol];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16g11(af, bf[m][j], acc[m][i][j]);
+#else
const hv8 af = *(const hv8*) &xs[b][i * 16 + l16][kcol];
#pragma unroll
for (int m = 0; m < NM; ++m)
#pragma unroll
for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16(af, bf[m][j], acc[m][i][j]);
+#endif
}
}
};
@@ -153,7 +181,11 @@ __global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg
for (int i = 0; i < MT; ++i)
#pragma unroll
for (int q = 0; q < 8; ++q) {
+#if defined(DS4_HIPK_G11)
+ const int row = i * 16 + 2 * q + hi;
+#else
const int row = i * 16 + 8 * hi + q;
+#endif
if (row >= nr) continue;
const size_t lr = (size_t) (r0 + row);
#pragma unroll