Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
KVA: state-only GDN recurrence (no gate projection, no output norm) behind STRATA_KVA_GDN_NORM for the A/B
Niko1221c5cf218e542f
diff --git a/include/strata/prefill/kernels.hpp b/include/strata/prefill/kernels.hppindex b99b64250..9bbf6732b 100644--- a/include/strata/prefill/kernels.hpp+++ b/include/strata/prefill/kernels.hpp@@ -74,7 +74,12 @@ void gdn_recurrence(float* state, const float* h, const float* gate, const float/// bits as 0 on an Arc A770; gdn_recurrence takes it by default on Alchemist (STRATA_GDN_XE=0 off, =1 any Intel GPU).void gdn_recurrence_variant(int variant, float* state, const float* h, const float* gate, const float* beta,const float* z, const float* gamma, float eps, float* y, uint16_t* y16, int64_t T,- void* stream, int64_t ld16 = 0);+ void* stream, int64_t ld16 = 0, bool with_out = true);+/// KVA: the same recurrence kernels as gdn_recurrence, bit for bit, without the output normalization: state, h,+/// gate, beta and the scratch `y` only. The output projection and the gated residual write have no consumer when+/// only the state advances, and their input (z, the gate projection) is not needed either.+void gdn_recurrence_state_only(float* state, const float* h, const float* gate, const float* beta, float* y,+ int64_t T, void* stream);// ---- MoE/// softmax over 512, top-10 (ties to the lower id), weights renormalised over the ten (the native router).diff --git a/src/prefill/kernels.cu b/src/prefill/kernels.cuindex a4f56f10c..5d1701306 100644--- a/src/prefill/kernels.cu+++ b/src/prefill/kernels.cu@@ -2608,7 +2608,7 @@ void gdn_conv(float* history, const float* qkv, const float* conv_w, float* h, i}void gdn_recurrence_variant(int variant, float* state, const float* h, const float* gate, const float* beta,const float* z, const float* gamma, float eps, float* y, uint16_t* y16, int64_t T,- void* stream, int64_t ld16) {+ void* stream, int64_t ld16, bool with_out) {if (ld16 <= 0) ld16 = (int64_t) HV * S;static const bool serial = std::getenv("STRATA_GDN_REC_HEADS") != nullptr; // the one-block-per-head kernel (A/B)#if defined(__HIPCC__)@@ -2624,7 +2624,8 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const floif (variant == 3) { // diagnostics: the quad recurrence + the old norm kernelif (T > 0) {gdn_rec_quad_kernel<<<HV * (S / QCB), QTH, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);- gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ if (with_out)+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);}return;}@@ -2634,8 +2635,9 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const flogdn_rec_quad2c_kernel<<<HV * (S / C2CB), QTH, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);elsegdn_rec_quad_pp_kernel<<<HV * (S / QCB), QTH, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);- gdn_out_norm_loop_kernel<true><<<(unsigned) std::min<int64_t>(T * HV, 4096), S, 0, (cudaStream_t) stream>>>(- z, gamma, eps, y, y16, T * HV, ld16);+ if (with_out)+ gdn_out_norm_loop_kernel<true><<<(unsigned) std::min<int64_t>(T * HV, 4096), S, 0, (cudaStream_t) stream>>>(+ z, gamma, eps, y, y16, T * HV, ld16);}check("gdn_recurrence (quad pp)");return;@@ -2650,10 +2652,10 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const floelsegdn_rec_quad_kernel<<<HV * (S / QCB), QTH, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);static const bool noy = [] { const char* v = std::getenv("STRATA_GDN_NOY"); return v && std::atoi(v) != 0; }();- if (noy)+ if (with_out && noy)gdn_out_norm_loop_kernel<false><<<(unsigned) std::min<int64_t>(T * HV, 4096), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, T * HV, ld16);- else+ else if (with_out)gdn_out_norm_loop_kernel<true><<<(unsigned) std::min<int64_t>(T * HV, 4096), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, T * HV, ld16);}@@ -2689,7 +2691,8 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const floif (T > T64)gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h + T64 * C, gate + T64 * HV, beta + T64 * HV, y + T64 * (size_t) HV * S, T - T64);- gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ if (with_out)+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);} else if (warp_rec) {// =2: the 4-accumulator chain split on the pipelined structure (FP32-level, not bit-exact);// =1: the sync-free bit-exact warp reduction (measured slower - kept as the bitwise control)@@ -2698,10 +2701,11 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const floy, T);elsegdn_rec_cols_warp_kernel<<<HV * NCB, S, 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);- gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ if (with_out)+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);} else#endif- if (serial || T <= 0 || variant == 2) {+ if ((serial && with_out) || T <= 0 || variant == 2) { // the fused kernel needs z/gamma: only with the outputgdn_rec_kernel<<<HV, dim3(S, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, z, gamma, eps, y, y16, T, ld16);} else {static const bool pipe = [] { const char* v = std::getenv("STRATA_GDN_PIPELINE"); return v == nullptr || std::atoi(v) != 0; }();@@ -2718,7 +2722,8 @@ void gdn_recurrence_variant(int variant, float* state, const float* h, const flogdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);elsegdn_rec_cols_kernel<<<HV * NCB, dim3(CB, RG), 0, (cudaStream_t) stream>>>(state, h, gate, beta, y, T);- gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);+ if (with_out)+ gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S, 0, (cudaStream_t) stream>>>(z, gamma, eps, y, y16, ld16);}check("gdn_recurrence");}@@ -2731,6 +2736,16 @@ void gdn_recurrence(float* state, const float* h, const float* gate, const floatstatic const bool serial = std::getenv("STRATA_GDN_REC_HEADS") != nullptr;gdn_recurrence_variant(head && !serial ? 1 : 0, state, h, gate, beta, z, gamma, eps, y, y16, T, stream, ld16);}+/// KVA: the same recurrence, bit for bit, without the output normalisation (its readers - the output projection+/// and the gated residual write - have no consumer when only the state advances). `y` is still the scratch the+/// recurrence kernels write; `z`/`gamma` are not read.+void gdn_recurrence_state_only(float* state, const float* h, const float* gate, const float* beta, float* y,+ int64_t T, void* stream) {+ static const bool head = [] { const char* v = std::getenv("STRATA_GDN_HEAD"); return v != nullptr && std::atoi(v) != 0; }();+ static const bool serial = std::getenv("STRATA_GDN_REC_HEADS") != nullptr;+ gdn_recurrence_variant(head && !serial ? 1 : 0, state, h, gate, beta, nullptr, nullptr, 0.0f, y, nullptr, T,+ stream, 0, /*with_out=*/false);+}void route(const float* logits, int32_t* ids, float* weights, int64_t T, int64_t n_expert, void* stream) {if (n_expert == 512)route_kernel<16><<<(unsigned) ((T + 7) / 8), 256, 0, (cudaStream_t) stream>>>(logits, ids, weights, T);diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex b5d23b0ab..286b215c6 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -3427,7 +3427,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strifloat* state = ss.gdn_state + (size_t) (gdn_index - ss.gdn_ord0) * gdn_floats;float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;if (!native_proj(m.gemm, wqkv, m.mixed_h, m.qkv, T, v.name("attn_qkv.weight"), err)) return false;- if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;+ // KVA: the gate projection feeds only the output normalization, which a state-only layer skips+ // (STRATA_KVA_GDN_NORM=1 keeps the full recurrence as the A/B control for the state-only one)+ static const bool kva_gdn_norm = [] {+ const char* v = std::getenv("STRATA_KVA_GDN_NORM");+ return v != nullptr && std::atoi(v) != 0;+ }();+ if ((!kva_layer || kva_gdn_norm) &&+ !native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;pt.mark(kPfGdnConv, cs); // "gdn" is the projections in; the rest on their own lines@@ -3435,8 +3442,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strigdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, m.cs);pt.mark(kPfGdnRec, cs);const int64_t ld_y = pf_pad() && T >= std::max<int64_t>(pf_switch_min_t(), 64) ? ZV + ZV_PAD : 0;- gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs,- ld_y);+ if (kva_layer && !kva_gdn_norm) // KVA: the recurrence without the output normalization+ gdn_recurrence_state_only(state, m.hbuf, m.gate, m.beta, m.y, T, m.cs);+ else+ gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs,+ ld_y);pt.mark(kPfGdnOut, cs);if (!kva_layer) // KVA: the out projection feeds only the residual write and the MoE halfif (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;