Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
gr: fold the hyper-connection inject and gate/mean into their neighbours
Niko1221aec327ee518c
diff --git a/src/kernels/cuda/gr.cu b/src/kernels/cuda/gr.cuindex 9081e9f9e..949714b8a 100644--- a/src/kernels/cuda/gr.cu+++ b/src/kernels/cuda/gr.cu@@ -163,8 +163,23 @@ __global__ void gr_norm_kernel(const float* __restrict__ R, const float* __restr/// `lo = silu((bf16(xn) @ w_down.T) / hc)`, ONE BLOCK PER OUTPUT ROW with its warps splitting the reduction.template <typename Activation>__global__ void gr_down_kernel(const Activation* __restrict__ xq, const uint16_t* __restrict__ w_down, int hc_dim,- int hc_lr, int hc, Activation* __restrict__ lq) {+ int hc_lr, int hc, Activation* __restrict__ lq,+ const uint16_t* __restrict__ w_inject, float* __restrict__ inject, bool do_inject) {const int k = blockIdx.x;+ if (do_inject && k == hc_lr) {+ // Track F: the inject projection is the down grid's last block. One warp per stream, the same+ // lane stride and warp_sumf as `gr_inject_kernel`, so the four values are bitwise the old kernel's.+ const int c = threadIdx.x >> 5;+ if (c >= hc) return;+ const int lane = threadIdx.x & 31;+ const uint16_t* row = w_inject + (size_t) c * hc_dim;+ float acc = 0.0f;+ for (int i = lane; i < hc_dim; i += 32)+ acc += activation_f32(xq[i]) * bf16_bits_to_f32(row[i]);+ acc = warp_sumf(acc);+ if (lane == 0) inject[c] = acc;+ return;+ }if (k >= hc_lr) return;const int lane = threadIdx.x & 31;const int warp = threadIdx.x >> 5;@@ -256,6 +271,39 @@ __global__ void gr_mean_kernel(const float* __restrict__ gated, int n_embd, intmixed[d] = m / (float) hc;}+/// Track F: `gr_gate_kernel` and `gr_mean_kernel` in one launch. One block owns WARPS columns for every+/// stream; a warp computes one column's dot per stream (the same lane stride and warp_sumf as the old gate),+/// the per-stream values meet in shared, and the mean adds them in the same c order as the old kernel.+template <typename Activation>+__global__ void gr_gate_mean_kernel(const Activation* __restrict__ lq, const uint16_t* __restrict__ w_up,+ const float* __restrict__ xn, int hc_lr, int n_embd, int hc,+ float* __restrict__ mixed) {+ const int lane = threadIdx.x & 31;+ const int warp = threadIdx.x >> 5;+ const int d = blockIdx.x * WARPS + warp;+ __shared__ float g[4][WARPS];+ if (d < n_embd) {+ for (int c = 0; c < hc; ++c) {+ const int i = c * n_embd + d;+ const uint16_t* row = w_up + (size_t) i * hc_lr;+ float acc = 0.0f;+ for (int k = lane; k < hc_lr; k += 32)+ acc += activation_f32(lq[k]) * bf16_bits_to_f32(row[k]);+ acc = warp_sumf(acc);+ if (lane == 0) g[c][warp] = xn[i] * sigmoid_f(acc);+ }+ }+ __syncthreads();+ if (threadIdx.x < WARPS) {+ const int dd = blockIdx.x * WARPS + threadIdx.x;+ if (dd < n_embd) {+ float m = 0.0f;+ for (int c = 0; c < hc; ++c) m += g[c][threadIdx.x];+ mixed[dd] = m / (float) hc;+ }+ }+}+/// `inject[c] = bf16(xn) @ w_inject[c]`, one value per stream: ONE WARP PER STREAM, lanes striding hc_dim.////// This mirrors `gr_down_kernel` rather than using a block-wide reduction. `hc` is 4 at the real geometry,@@ -362,6 +410,7 @@ void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, const// One setting selects every projection in this call. Captured graphs retain these kernel variants.const bool use_native = native_mmvf;const bool use_fp32 = fp32_activations || use_native;+ const bool fused_gm = hc <= 4; // gr_gate_mean_kernel's shared array holds hc <= 4 streamsif (use_native) {if ((hc_dim & 1) != 0 || (hc_lr & 1) != 0)throw std::invalid_argument("gr_read native MMVF requires even hc*n_embd and hc_lr");@@ -374,28 +423,31 @@ void gr_read(const float* R, const float* w_norm, const uint16_t* w_down, constnative_gr_pre_gated(ws.xn, ws.gated, mixed, n_embd, hc, w_inject != nullptr, stream);} else if (use_fp32) {gr_norm_kernel<true><<<hc, THREADS, 0, st>>>(R, w_norm, eps, n_embd, ws.xn, ws.xq);- gr_down_kernel<float><<<hc_lr, THREADS, 0, st>>>(ws.xn, w_down, hc_dim, hc_lr, hc, ws.lo);- gr_gate_kernel<float><<<(hc_dim + WARPS - 1) / WARPS, THREADS, 0, st>>>(- ws.lo, w_up, ws.xn, hc_dim, hc_lr, ws.gated);+ gr_down_kernel<float><<<hc_lr + (w_inject != nullptr ? 1 : 0), THREADS, 0, st>>>(+ ws.xn, w_down, hc_dim, hc_lr, hc, ws.lo, w_inject, inject, w_inject != nullptr);+ if (fused_gm)+ gr_gate_mean_kernel<float><<<(n_embd + WARPS - 1) / WARPS, THREADS, 0, st>>>(+ ws.lo, w_up, ws.xn, hc_lr, n_embd, hc, mixed);+ else+ gr_gate_kernel<float><<<(hc_dim + WARPS - 1) / WARPS, THREADS, 0, st>>>(+ ws.lo, w_up, ws.xn, hc_dim, hc_lr, ws.gated);} else {gr_norm_kernel<false><<<hc, THREADS, 0, st>>>(R, w_norm, eps, n_embd, ws.xn, ws.xq);- gr_down_kernel<uint16_t><<<hc_lr, THREADS, 0, st>>>(ws.xq, w_down, hc_dim, hc_lr, hc, ws.lq);- gr_gate_kernel<uint16_t><<<(hc_dim + WARPS - 1) / WARPS, THREADS, 0, st>>>(- ws.lq, w_up, ws.xn, hc_dim, hc_lr, ws.gated);+ gr_down_kernel<uint16_t><<<hc_lr + (w_inject != nullptr ? 1 : 0), THREADS, 0, st>>>(+ ws.xq, w_down, hc_dim, hc_lr, hc, ws.lq, w_inject, inject, w_inject != nullptr);+ if (fused_gm)+ gr_gate_mean_kernel<uint16_t><<<(n_embd + WARPS - 1) / WARPS, THREADS, 0, st>>>(+ ws.lq, w_up, ws.xn, hc_lr, n_embd, hc, mixed);+ else+ gr_gate_kernel<uint16_t><<<(hc_dim + WARPS - 1) / WARPS, THREADS, 0, st>>>(+ ws.lq, w_up, ws.xn, hc_dim, hc_lr, ws.gated);}- if (!use_native)+ if (!use_native && !fused_gm)gr_mean_kernel<<<(n_embd + THREADS - 1) / THREADS, THREADS, 0, st>>>(ws.gated, n_embd, hc, mixed);// Absent for the final mixer, and then nothing is written - computing an injection nobody reads would be- // a claim, not a convenience.- if (w_inject != nullptr) {- const int nthreads = 32 * hc;- if (use_native)- bf16_gemv_fp32_mmvf(ws.xn, w_inject, inject, hc_dim, hc, stream);- else if (use_fp32)- gr_inject_kernel<float><<<1, nthreads, 0, st>>>(ws.xn, w_inject, hc_dim, hc, inject);- else- gr_inject_kernel<uint16_t><<<1, nthreads, 0, st>>>(ws.xq, w_inject, hc_dim, hc, inject);- }+ // a claim, not a convenience. The non-native calls compute it in the down grid's last block (Track F).+ if (w_inject != nullptr && use_native)+ bf16_gemv_fp32_mmvf(ws.xn, w_inject, inject, hc_dim, hc, stream);const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) {