Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: kernels for a layer-major expert pass (moe_shared_init, moe_scatter_add) (#3976, #1670)
vizdatoma6df0023a412
diff --git a/include/strata/prefill/kernels.hpp b/include/strata/prefill/kernels.hppindex 5f1bf3554..9be33b718 100644--- a/include/strata/prefill/kernels.hpp+++ b/include/strata/prefill/kernels.hpp@@ -104,6 +104,14 @@ void gather_rows16(const uint16_t* x16, const int32_t* src, uint16_t* dst16, int/// bo[t, :] = shared[t, :] * sigmoid(sg[t]) + sum_k w[t, k] * D[slot[t, k], :]void moe_combine(const float* D, const int32_t* slot, const float* w, const float* shared, const float* sg, float* bo,int64_t T, void* stream, const float* row_sd = nullptr);+/// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): acc[t, :] = shared[t, :] * sigmoid(sg[t]) - the shared expert's+/// part of moe_combine, written before the routed experts are known.+void moe_shared_init(const float* shared, const float* sg, float* acc, int64_t T, void* stream);+/// Layer-major prefill: one expert group's rows added into the super-chunk's accumulator, per token in a fixed order+/// (deterministic, no atomics): for each u < n_u, acc[u_tok[u], :] += sum_{j in [u_off[u], u_off[u+1])} u_w[j] *+/// Dm[u_rows[j], :]. The sum runs in the group's row order, not moe_combine's k order (FP32 rounding only).+void moe_scatter_add(const float* Dm, const int32_t* u_tok, const int32_t* u_off, const int32_t* u_rows,+ const float* u_w, int64_t n_u, float* acc, void* stream);/// --peer-device's prompt share as sums: bo[t, :] = (sum over the k with slot[t, k] < rows_local of w[t, k] */// D[slot[t, k], :], in k order) + peer[t, :] + shared[t, :] * sigmoid(sg[t]). `peer` (the peer's per-token sums)/// may be mapped host memory.diff --git a/src/prefill/kernels.cu b/src/prefill/kernels.cuindex e99f5ba6e..cdace4d52 100644--- a/src/prefill/kernels.cu+++ b/src/prefill/kernels.cu@@ -2120,6 +2120,24 @@ __global__ void moe_combine4_kernel(const float* __restrict__ Dm, const int32_t*bo[i] = make_float4(s.x + sh.x * g, s.y + sh.y * g, s.z + sh.z * g, s.w + sh.w * g);}+__global__ void moe_shared_init_kernel(const float* __restrict__ shared, const float* __restrict__ sg,+ float* __restrict__ acc, int64_t T) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= T * N) return;+ acc[i] = shared[i] * sigm(sg[i / N]);+}+// grid (n_u, ceil(N / 256)): one token of the group per block row, its rows summed in order+__global__ void moe_scatter_add_kernel(const float* __restrict__ Dm, const int32_t* __restrict__ u_tok,+ const int32_t* __restrict__ u_off, const int32_t* __restrict__ u_rows,+ const float* __restrict__ u_w, float* __restrict__ acc) {+ const int64_t u = blockIdx.x;+ const int64_t d = (int64_t) blockIdx.y * blockDim.x + threadIdx.x;+ if (d >= N) return;+ float s = 0.0f;+ for (int32_t j = u_off[u]; j < u_off[u + 1]; ++j) s = fmaf(u_w[j], Dm[(int64_t) u_rows[j] * N + d], s);+ acc[(int64_t) u_tok[u] * N + d] += s;+}+// ---------------------------------------------------------------- QSA helpers__global__ void rms_rows_kernel(float* __restrict__ x, const float* __restrict__ w, int64_t cols, int64_t ld, float eps) {__shared__ float sh[32];@@ -2931,6 +2949,18 @@ void moe_combine(const float* Dm, const int32_t* slot, const float* w, const flomoe_combine_kernel<<<blocks_for(T * N), 256, 0, (cudaStream_t) stream>>>(Dm, slot, w, shared, sg, bo, T, row_sd);check("moe_combine");}+void moe_shared_init(const float* shared, const float* sg, float* acc, int64_t T, void* stream) {+ if (T <= 0) return;+ moe_shared_init_kernel<<<blocks_for(T * N), 256, 0, (cudaStream_t) stream>>>(shared, sg, acc, T);+ check("moe_shared_init");+}+void moe_scatter_add(const float* Dm, const int32_t* u_tok, const int32_t* u_off, const int32_t* u_rows,+ const float* u_w, int64_t n_u, float* acc, void* stream) {+ if (n_u <= 0) return;+ const dim3 grid((unsigned) n_u, (unsigned) ((N + 255) / 256));+ moe_scatter_add_kernel<<<grid, 256, 0, (cudaStream_t) stream>>>(Dm, u_tok, u_off, u_rows, u_w, acc);+ check("moe_scatter_add");+}void rms_rows(float* x, const float* w, int64_t rows, int64_t cols, int64_t ld, float eps, void* stream) {if (rows <= 0) return;rms_rows_kernel<<<(unsigned) rows, 256, 0, (cudaStream_t) stream>>>(x, w, cols, ld, eps);