Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: a padded activation is gfx11 only, and Gemm::native copies it unpadded instead of ending the engine

johnpiab61f892d8d3
diff --git a/docs/AMD_HIP.md b/docs/AMD_HIP.md
index 9330f0cd7..f765368c7 100644
--- a/docs/AMD_HIP.md
+++ b/docs/AMD_HIP.md
@@ -575,6 +575,17 @@ an RX 9070 XT 16 GB and a Radeon AI PRO R9700 32 GB (both gfx1201), a Ryzen 9 39
Q2_0 fused experts x1.28 / x1.19, both x1.73 / x1.79. IQ3_XXS 32 GB, 5 pairs (only the attention is new): 4K 1,369 -> 1,436 (the base starts in a 1,170 or a
1,370 mode), 20K 1,576 -> 1,997 (x1.27). The identity gate with both switched off (STRATA_HIP_WMMA=0, STRATA_PF_FUSED=0): 10 of 10 prompts and the
4K / 20K residual dumps identical on both packs.
+ - **`STRATA_PF_GEMM` / `STRATA_PF_PAD` are gfx11 only** (0.1.46): their product is `strata_pf_gemm_f16_ld`,
+ built into the gfx11 matrix-core block of `wmma_gemm.cu`, so an RDNA4 card runs neither. Until 0.1.46 the padded
+ half was worse than inert: `STRATA_PF_PAD=1 STRATA_PF_GEMM=1` let the two K 6144 producers write row stride 6,208
+ into a GEMM that only the gfx11 kernel takes, and `Gemm::native` ended the engine (`prefill gemm: a padded X (ldx
+ 6208, K 6144) needs STRATA_PF_PAD's path`, exit code 1 - under `serve/` an engine restart loop per request).
+ Measured on an RX 9070 (gfx1201, 16 GB, system ROCm 7.2.4, kernel 7.2.11, 31 GiB RAM, Coder IQ1_M): 3 of 3 starts
+ exited that way at `--vram-reserve-mib` 700 and 1,536, so it was not the VRAM guard. Now the producer asks
+ `strata_pf_padded_gemm_available()` and does not pad where no kernel reads the stride (one line at the first prompt),
+ and `Gemm::native` copies a padded X's rows unpadded and runs the ordinary paths instead of exiting - checked with
+ the gate forced off on that card: 4K prompt answers byte-identical to the unpadded run, no exit. `STRATA_PF_GEMM=1`
+ alone is inert there too (4K 1,247 against 1,243 tok/s, 8K 1,330 against 1,321, medians of 3 starts, i.e. noise).
- **Known:** rarely (about 1 start in 10) a HIP run's greedy output differs from another start's at some token, on
one card or two and on engine 0.1.29 as well; not yet explained.
- **Pictures:** the card's reader (`strata-vision-hip`) is measured on an R9700 (gfx1201): 57 ms for a 260-token picture and 0.40 s for a
diff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hpp
index 4800bb400..596b3e4bf 100644
--- a/include/strata/prefill/gemm.hpp
+++ b/include/strata/prefill/gemm.hpp
@@ -98,6 +98,12 @@ private:
int64_t tc_x_elems_ = 0;
bool native_mmq(const uint16_t* X, int ggml_type, const void* W_blocks, float* Y, int64_t T, int64_t N, int64_t K,
int64_t ldy);
+ /// A padded X (ldx > K) that no kernel above took: its T rows copied to a contiguous FP16 buffer of stride K, grown
+ /// as needed; nullptr when the allocation or the copy failed (then the caller reports it). The padded product is
+ /// the gfx11 matrix cores' (STRATA_PF_PAD), so this is the path of another card whose activations arrived padded.
+ const uint16_t* unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t ldx);
+ uint16_t* unpadded_x_ = nullptr;
+ int64_t unpadded_x_elems_ = 0;
void* mmq_ctx_ = nullptr;
void* mmq_buf_ = nullptr;
bool mmq_failed_ = false;
diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cu
index 538e17eb3..b8889a843 100644
--- a/src/prefill/gemm.cu
+++ b/src/prefill/gemm.cu
@@ -560,6 +560,28 @@ bool prompt_f16() {
#endif
}
+// A padded X (ldx > K) no kernel took: its rows copied to a contiguous FP16 buffer of stride K. Grown, never shrunk
+// (one buffer per Gemm; a padded X only arrives from STRATA_PF_PAD's producers). One 2-D copy: the same values, and
+// the ordinary paths below then read them exactly as if the producer had written them unpadded.
+const uint16_t* Gemm::unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t ldx) {
+ if (X == nullptr || T <= 0 || K <= 0 || ldx <= K) return nullptr;
+ const int64_t need = T * K;
+ if (need > unpadded_x_elems_) {
+ if (unpadded_x_ != nullptr) { cudaFree(unpadded_x_); unpadded_x_ = nullptr; unpadded_x_elems_ = 0; }
+ void* p = nullptr;
+ if (cudaMalloc(&p, (size_t) need * sizeof(uint16_t)) != cudaSuccess) { cudaGetLastError(); return nullptr; }
+ unpadded_x_ = (uint16_t*) p;
+ unpadded_x_elems_ = need;
+ }
+ if (cudaMemcpy2DAsync(unpadded_x_, (size_t) K * sizeof(uint16_t), X, (size_t) ldx * sizeof(uint16_t),
+ (size_t) K * sizeof(uint16_t), (size_t) T, cudaMemcpyDeviceToDevice,
+ (cudaStream_t) stream_) != cudaSuccess) {
+ cudaGetLastError();
+ return nullptr;
+ }
+ return unpadded_x_;
+}
+
Gemm::~Gemm() {
#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)
delete static_cast<strata::prefill::mmq::Context*>(mmq_ctx_);
@@ -574,6 +596,7 @@ Gemm::~Gemm() {
if (handle_) cublasDestroy((cublasHandle_t) handle_);
if (tc_w_) cudaFree(tc_w_);
if (tc_x_) cudaFree(tc_x_);
+ if (unpadded_x_) cudaFree(unpadded_x_);
if (!external_) {
if (scratch_) cudaFree(scratch_);
if (workspace_) cudaFree(workspace_);
@@ -1259,8 +1282,17 @@ void Gemm::native(const uint16_t* X, int ggml_type, const void* W_blocks, float*
}
if (ok) return;
#endif
- std::fprintf(stderr, "prefill gemm: a padded X (ldx %lld, K %lld) needs STRATA_PF_PAD's path\n",
- (long long) ldx, (long long) K);
+ // Nothing took a padded X: the padded product is the gfx11 matrix cores' (STRATA_PF_PAD asks
+ // strata_pf_padded_gemm_available() before padding now, so this is a stale producer or another card's
+ // activation). Copy the rows unpadded and run the ordinary paths - the same products, the same bits - rather
+ // than ending the engine.
+ const uint16_t* Xu = unpad_x(X, T, K, ldx);
+ if (Xu != nullptr) {
+ native(Xu, ggml_type, W_blocks, Y, T, N, K, ldy, beta, 0);
+ return;
+ }
+ std::fprintf(stderr, "prefill gemm: a padded X (ldx %lld, K %lld) has no path on this card and could not be "
+ "copied unpadded (no device memory)\n", (long long) ldx, (long long) K);
std::exit(1);
}
#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 4505a9ab8..c9b453540 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -468,7 +468,16 @@ inline bool cvec_fuse() {
inline bool pf_pad() {
static const bool v = [] {
const char* p = std::getenv("STRATA_PF_PAD"); const char* g = std::getenv("STRATA_PF_GEMM");
- return p && p[0] == '1' && g && g[0] == '1';
+ if (!(p && p[0] == '1' && g && g[0] == '1')) return false;
+ // The padded stride pays only for strata_pf_gemm_f16_ld, which is built for the gfx11 matrix cores. On another
+ // card nothing takes a padded X (it used to end the engine: "a padded X ... needs STRATA_PF_PAD's path", #1237):
+ // do not pad, and say so once.
+ if (!strata_pf_padded_gemm_available()) {
+ std::fprintf(stderr, "prefill gemm: STRATA_PF_PAD needs the padded gfx11 product, which this card does not run; "
+ "not padding\n");
+ return false;
+ }
+ return true;
}();
return v;
}
diff --git a/src/prefill/wmma_gemm.cu b/src/prefill/wmma_gemm.cu
index 11af8c89f..cf5f17899 100644
--- a/src/prefill/wmma_gemm.cu
+++ b/src/prefill/wmma_gemm.cu
@@ -56,6 +56,9 @@ static bool strata_wmma_gfx11_device() {
return c == 1;
}
+// The same answer for host code that decides whether to pad an activation's row stride (prefill.cpp, STRATA_PF_PAD).
+bool strata_pf_padded_gemm_available() { return strata_wmma_gfx11_device(); }
+
template <typename ElemT>
struct WmmaTraits;
@@ -716,6 +719,8 @@ bool strata_pf_gemm_f16_ld(const uint16_t*, int64_t, const uint16_t*, int64_t, f
int64_t, float, void*) {
return false;
}
+// No padded product in this build (no gfx11 target): prefill.cpp does not pad an activation's row stride.
+bool strata_pf_padded_gemm_available() { return false; }
bool strata_pf_hcdown_bf16(const uint16_t*, int64_t, const uint16_t*, const uint16_t*, int64_t, int64_t, float*, float*,
int64_t, int64_t, void*) {
return false;
diff --git a/src/prefill/wmma_gemm.h b/src/prefill/wmma_gemm.h
index f9551c918..c54b22c4a 100644
--- a/src/prefill/wmma_gemm.h
+++ b/src/prefill/wmma_gemm.h
@@ -41,6 +41,14 @@ bool strata_pf_hcdown_exact_bf16(const uint16_t* X, int64_t ldx, const uint16_t*
int64_t K, void* stream = nullptr);
bool strata_pf_gemm_f16_ld(const uint16_t* X, int64_t ldx, const uint16_t* W, int64_t ldw, float* Y, int64_t T,
int64_t N, int64_t K, int64_t ldy, float beta, void* stream = nullptr);
+/// True when the products above can run on the current device (the gfx11 matrix cores). Host code that pads an
+/// activation's row stride for them (STRATA_PF_PAD, prefill.cpp) must ask this first: on another card no kernel takes
+/// a padded X, and Gemm::native would have to copy the rows unpadded.
+#if defined(STRATA_USE_HIP)
+bool strata_pf_padded_gemm_available();
+#else
+inline bool strata_pf_padded_gemm_available() { return false; }
+#endif
bool strata_wmma_gemm_bf16(const uint16_t* X, const uint16_t* W, float* Y,
int64_t T, int64_t N, int64_t K, int64_t ldy, float beta,