Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: a padded activation is gfx11 only, and Gemm::native copies it unpadded instead of ending the engine
johnpiab61f892d8d3
diff --git a/docs/AMD_HIP.md b/docs/AMD_HIP.mdindex 9330f0cd7..f765368c7 100644--- a/docs/AMD_HIP.md+++ b/docs/AMD_HIP.md@@ -575,6 +575,17 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39Q2_0 fused experts x1.28 / x1.19, both x1.73 / x1.79. IQ3_XXS 32 GB, 5 pairs (only the attention is new): 4K 1,369 -> 1,436 (the base starts in a 1,170 or a1,370 mode), 20K 1,576 -> 1,997 (x1.27). The identity gate with both switched off (STRATA_HIP_WMMA=0, STRATA_PF_FUSED=0): 10 of 10 prompts and the4K / 20K residual dumps identical on both packs.+ - **`STRATA_PF_GEMM` / `STRATA_PF_PAD` are gfx11 only** (0.1.46): their product is `strata_pf_gemm_f16_ld`,+ built into the gfx11 matrix-core block of `wmma_gemm.cu`, so an RDNA4 card runs neither. Until 0.1.46 the padded+ half was worse than inert: `STRATA_PF_PAD=1 STRATA_PF_GEMM=1` let the two K 6144 producers write row stride 6,208+ into a GEMM that only the gfx11 kernel takes, and `Gemm::native` ended the engine (`prefill gemm: a padded X (ldx+ 6208, K 6144) needs STRATA_PF_PAD's path`, exit code 1 - under `serve/` an engine restart loop per request).+ Measured on an RX 9070 (gfx1201, 16 GB, system ROCm 7.2.4, kernel 7.2.11, 31 GiB RAM, Coder IQ1_M): 3 of 3 starts+ exited that way at `--vram-reserve-mib` 700 and 1,536, so it was not the VRAM guard. Now the producer asks+ `strata_pf_padded_gemm_available()` and does not pad where no kernel reads the stride (one line at the first prompt),+ and `Gemm::native` copies a padded X's rows unpadded and runs the ordinary paths instead of exiting - checked with+ the gate forced off on that card: 4K prompt answers byte-identical to the unpadded run, no exit. `STRATA_PF_GEMM=1`+ alone is inert there too (4K 1,247 against 1,243 tok/s, 8K 1,330 against 1,321, medians of 3 starts, i.e. noise).- **Known:** rarely (about 1 start in 10) a HIP run's greedy output differs from another start's at some token, onone card or two and on engine 0.1.29 as well; not yet explained.- **Pictures:** the card's reader (`strata-vision-hip`) is measured on an R9700 (gfx1201): 57 ms for a 260-token picture and 0.40 s for adiff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hppindex 4800bb400..596b3e4bf 100644--- a/include/strata/prefill/gemm.hpp+++ b/include/strata/prefill/gemm.hpp@@ -98,6 +98,12 @@ private:int64_t tc_x_elems_ = 0;bool native_mmq(const uint16_t* X, int ggml_type, const void* W_blocks, float* Y, int64_t T, int64_t N, int64_t K,int64_t ldy);+ /// A padded X (ldx > K) that no kernel above took: its T rows copied to a contiguous FP16 buffer of stride K, grown+ /// as needed; nullptr when the allocation or the copy failed (then the caller reports it). The padded product is+ /// the gfx11 matrix cores' (STRATA_PF_PAD), so this is the path of another card whose activations arrived padded.+ const uint16_t* unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t ldx);+ uint16_t* unpadded_x_ = nullptr;+ int64_t unpadded_x_elems_ = 0;void* mmq_ctx_ = nullptr;void* mmq_buf_ = nullptr;bool mmq_failed_ = false;diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cuindex 538e17eb3..b8889a843 100644--- a/src/prefill/gemm.cu+++ b/src/prefill/gemm.cu@@ -560,6 +560,28 @@ bool prompt_f16() {#endif}+// A padded X (ldx > K) no kernel took: its rows copied to a contiguous FP16 buffer of stride K. Grown, never shrunk+// (one buffer per Gemm; a padded X only arrives from STRATA_PF_PAD's producers). One 2-D copy: the same values, and+// the ordinary paths below then read them exactly as if the producer had written them unpadded.+const uint16_t* Gemm::unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t ldx) {+ if (X == nullptr || T <= 0 || K <= 0 || ldx <= K) return nullptr;+ const int64_t need = T * K;+ if (need > unpadded_x_elems_) {+ if (unpadded_x_ != nullptr) { cudaFree(unpadded_x_); unpadded_x_ = nullptr; unpadded_x_elems_ = 0; }+ void* p = nullptr;+ if (cudaMalloc(&p, (size_t) need * sizeof(uint16_t)) != cudaSuccess) { cudaGetLastError(); return nullptr; }+ unpadded_x_ = (uint16_t*) p;+ unpadded_x_elems_ = need;+ }+ if (cudaMemcpy2DAsync(unpadded_x_, (size_t) K * sizeof(uint16_t), X, (size_t) ldx * sizeof(uint16_t),+ (size_t) K * sizeof(uint16_t), (size_t) T, cudaMemcpyDeviceToDevice,+ (cudaStream_t) stream_) != cudaSuccess) {+ cudaGetLastError();+ return nullptr;+ }+ return unpadded_x_;+}+Gemm::~Gemm() {#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)delete static_cast<strata::prefill::mmq::Context*>(mmq_ctx_);@@ -574,6 +596,7 @@ Gemm::~Gemm() {if (handle_) cublasDestroy((cublasHandle_t) handle_);if (tc_w_) cudaFree(tc_w_);if (tc_x_) cudaFree(tc_x_);+ if (unpadded_x_) cudaFree(unpadded_x_);if (!external_) {if (scratch_) cudaFree(scratch_);if (workspace_) cudaFree(workspace_);@@ -1259,8 +1282,17 @@ void Gemm::native(const uint16_t* X, int ggml_type, const void* W_blocks, float*}if (ok) return;#endif- std::fprintf(stderr, "prefill gemm: a padded X (ldx %lld, K %lld) needs STRATA_PF_PAD's path\n",- (long long) ldx, (long long) K);+ // Nothing took a padded X: the padded product is the gfx11 matrix cores' (STRATA_PF_PAD asks+ // strata_pf_padded_gemm_available() before padding now, so this is a stale producer or another card's+ // activation). Copy the rows unpadded and run the ordinary paths - the same products, the same bits - rather+ // than ending the engine.+ const uint16_t* Xu = unpad_x(X, T, K, ldx);+ if (Xu != nullptr) {+ native(Xu, ggml_type, W_blocks, Y, T, N, K, ldy, beta, 0);+ return;+ }+ std::fprintf(stderr, "prefill gemm: a padded X (ldx %lld, K %lld) has no path on this card and could not be "+ "copied unpadded (no device memory)\n", (long long) ldx, (long long) K);std::exit(1);}#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 4505a9ab8..c9b453540 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -468,7 +468,16 @@ inline bool cvec_fuse() {inline bool pf_pad() {static const bool v = [] {const char* p = std::getenv("STRATA_PF_PAD"); const char* g = std::getenv("STRATA_PF_GEMM");- return p && p[0] == '1' && g && g[0] == '1';+ if (!(p && p[0] == '1' && g && g[0] == '1')) return false;+ // The padded stride pays only for strata_pf_gemm_f16_ld, which is built for the gfx11 matrix cores. On another+ // card nothing takes a padded X (it used to end the engine: "a padded X ... needs STRATA_PF_PAD's path", #1237):+ // do not pad, and say so once.+ if (!strata_pf_padded_gemm_available()) {+ std::fprintf(stderr, "prefill gemm: STRATA_PF_PAD needs the padded gfx11 product, which this card does not run; "+ "not padding\n");+ return false;+ }+ return true;}();return v;}diff --git a/src/prefill/wmma_gemm.cu b/src/prefill/wmma_gemm.cuindex 11af8c89f..cf5f17899 100644--- a/src/prefill/wmma_gemm.cu+++ b/src/prefill/wmma_gemm.cu@@ -56,6 +56,9 @@ static bool strata_wmma_gfx11_device() {return c == 1;}+// The same answer for host code that decides whether to pad an activation's row stride (prefill.cpp, STRATA_PF_PAD).+bool strata_pf_padded_gemm_available() { return strata_wmma_gfx11_device(); }+template <typename ElemT>struct WmmaTraits;@@ -716,6 +719,8 @@ bool strata_pf_gemm_f16_ld(const uint16_t*, int64_t, const uint16_t*, int64_t, fint64_t, float, void*) {return false;}+// No padded product in this build (no gfx11 target): prefill.cpp does not pad an activation's row stride.+bool strata_pf_padded_gemm_available() { return false; }bool strata_pf_hcdown_bf16(const uint16_t*, int64_t, const uint16_t*, const uint16_t*, int64_t, int64_t, float*, float*,int64_t, int64_t, void*) {return false;diff --git a/src/prefill/wmma_gemm.h b/src/prefill/wmma_gemm.hindex f9551c918..c54b22c4a 100644--- a/src/prefill/wmma_gemm.h+++ b/src/prefill/wmma_gemm.h@@ -41,6 +41,14 @@ bool strata_pf_hcdown_exact_bf16(const uint16_t* X, int64_t ldx, const uint16_t*int64_t K, void* stream = nullptr);bool strata_pf_gemm_f16_ld(const uint16_t* X, int64_t ldx, const uint16_t* W, int64_t ldw, float* Y, int64_t T,int64_t N, int64_t K, int64_t ldy, float beta, void* stream = nullptr);+/// True when the products above can run on the current device (the gfx11 matrix cores). Host code that pads an+/// activation's row stride for them (STRATA_PF_PAD, prefill.cpp) must ask this first: on another card no kernel takes+/// a padded X, and Gemm::native would have to copy the rows unpadded.+#if defined(STRATA_USE_HIP)+bool strata_pf_padded_gemm_available();+#else+inline bool strata_pf_padded_gemm_available() { return false; }+#endifbool strata_wmma_gemm_bf16(const uint16_t* X, const uint16_t* W, float* Y,int64_t T, int64_t N, int64_t K, int64_t ldy, float beta,