Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/prefill/gdn_chunk.cu - see include/strata/prefill/gdn_chunk.hpp for the math and the contract.
//
// Three kernels, the same three stages as llama.cpp-v100's gdn-chunk-sm70 (cumsum / kkt / fwd), hand-written
// plain CUDA - FP32 throughout, no FP16 staging and no tensor cores yet (the TileLang m8n8k4 output of the
// reference is a later, speed-only step). What the chunking buys is the shape of the serial chain: T tokens
// become T/64 chunk steps, each step a batch of dense matmuls over 64 tokens instead of one token's rank-1
// update behind two __syncthreads.
//
// Per CTA: one (v head, 32-column slice) of the state, all chunks in order (the state slice lives in
// registers - the chunk-serial scan of the reference's fwd kernel). The per-chunk 64x64 matrices A and P are
// computed once per (chunk, head) by the kkt kernel and read by the four column slices.
#include "strata/prefill/gdn_chunk.hpp"
#include <cuda_runtime.h>
#if !defined(__HIPCC__)
#include <cuda_fp16.h>
#include <mma.h>
#endif
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <mutex>
namespace strata::prefill {
namespace {
constexpr int S = 128, HK = 16, HV = 48, C = 10240; // the geometry of src/prefill/kernels.cu
constexpr int CH = 64; // tokens per chunk (the reference's chunk)
constexpr int NS = 32; // value columns per fwd CTA
constexpr int NSE = S / NS; // 4 slices per head
constexpr int SEG = 32; // chunks per pipeline segment (sizes the workspace)
// the wmma kernel's dynamic shared memory: Ks/Qs f16, W16 f16, Ug f32 (Gram scratch / U out), As/Ps f16,
// dlt f32 and its f16 image - 84 KB, past the 48 KB default (opt-in), under the 96 KB Volta ceiling
constexpr int WMMA_SMEM = (2 * CH * S + S * NS) * (int) sizeof(__half) + S * NS * (int) sizeof(float) +
2 * CH * CH * (int) sizeof(__half) + CH * NS * (int) sizeof(float) +
CH * NS * (int) sizeof(__half);
void check(const char* what) {
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
std::fprintf(stderr, "gdn_chunk %s: %s\n", what, cudaGetErrorString(e));
std::exit(1);
}
}
// g_cumsum[lt * HV + h] = inclusive cumsum of gate over the segment's tokens (lt = segment-local token).
// One block of 64 per (chunk, head): a Kogge-Stone scan of 64 values, two tokens' worth of nothing.
__global__ void __launch_bounds__(CH) gdn_chunk_cumsum_kernel(float* __restrict__ gcum,
const float* __restrict__ gate, int ntok) {
const int h = blockIdx.y;
const int lt = blockIdx.x * CH + threadIdx.x;
__shared__ float sh[CH];
sh[threadIdx.x] = lt < ntok ? gate[(size_t) lt * HV + h] : 0.0f;
__syncthreads();
#pragma unroll
for (int off = 1; off < CH; off <<= 1) {
const float add = threadIdx.x >= off ? sh[threadIdx.x - off] : 0.0f;
__syncthreads();
sh[threadIdx.x] += add;
__syncthreads();
}
if (lt < ntok) gcum[(size_t) lt * HV + h] = sh[threadIdx.x];
}
// Per (chunk, head): the decayed Gram matrices the fwd kernel's solve and output sum read,
// A[t][s] = beta_t exp(c_t - c_s) (k_t . k_s) for s < t (the (I + strict_lower) solve matrix)
// P[t][s] = exp(c_t - c_s) (q_t . k_s) for s <= t (the output sum)
// in f32 (CH x CH per chunk-head; the upper triangle of A and P is never written and never read). Every
// decay factor is a decay from a LATER token back to an EARLIER one - <= 1 for a real GDN gate - so nothing
// here can overflow, and delta stays O(beta v) in the fwd solve (gdn_chunk.hpp derives the form).
__global__ void __launch_bounds__(128) gdn_chunk_kkt_kernel(float* __restrict__ A, float* __restrict__ P,
const float* __restrict__ h,
const float* __restrict__ beta,
const float* __restrict__ gcum, int nchunks) {
// 2 * CH * SP f32: the k and q tiles (opt-in, > 48 KB). SP pads the row so the dot loop's row-strided
// reads do not all land in one shared bank (S = 128 floats is a whole number of bank rows).
constexpr int SP = S + 1;
extern __shared__ float sh[];
float* ks = sh;
float* qs = sh + CH * SP;
const int hh = blockIdx.y, ch = blockIdx.x, qh = hh % HK;
const int64_t base = (size_t) ch * CH;
const float* kh = h + base * C + (size_t) HK * S + (size_t) qh * S;
const float* qh_ = h + base * C + (size_t) qh * S;
for (int i = threadIdx.x; i < CH * S; i += blockDim.x) {
const int t = i / S, j = i % S;
ks[t * SP + j] = kh[(size_t) t * C + j];
qs[t * SP + j] = qh_[(size_t) t * C + j];
}
__syncthreads();
float* Ap = A + ((size_t) ch * HV + hh) * CH * CH;
float* Pp = P + ((size_t) ch * HV + hh) * CH * CH;
for (int p = threadIdx.x; p < CH * CH; p += blockDim.x) {
const int t = p / CH, s = p % CH;
if (s > t) continue;
const float dec = expf(gcum[(base + t) * HV + hh] - gcum[(base + s) * HV + hh]);
float pq = 0.0f, pq1 = 0.0f;
#pragma unroll 4
for (int i = 0; i < S; i += 2) {
pq = fmaf(qs[t * SP + i], ks[s * SP + i], pq);
pq1 = fmaf(qs[t * SP + i + 1], ks[s * SP + i + 1], pq1);
}
Pp[t * CH + s] = (pq + pq1) * dec;
if (s < t) {
float pk = 0.0f, pk1 = 0.0f;
#pragma unroll 4
for (int i = 0; i < S; i += 2) {
pk = fmaf(ks[t * SP + i], ks[s * SP + i], pk);
pk1 = fmaf(ks[t * SP + i + 1], ks[s * SP + i + 1], pk1);
}
Ap[t * CH + s] = (pk + pk1) * dec * beta[(base + t) * HV + hh];
}
}
}
// The chunk-serial scan: one CTA per (v head, 32-column slice), state in registers, all chunks in order.
// Per token: the two row-block reductions (k_t.W_in, q_t.W_in) give u_t and qW_t; the w==0 lane of each
// column solves its delta and writes o. The A/P tiles are staged to shared memory at the top of each chunk:
// the solve and output sums are serial FMA chains, and a global load inside the chain (L2, ~200 cycles)
// throttles the whole CTA to it. One __syncthreads per token (the shared dlt row plus the partials), one
// after the token loop (the last dlt row before the state update).
__global__ void __launch_bounds__(128) gdn_chunk_fwd_kernel(float* __restrict__ state, float* __restrict__ oc,
const float* __restrict__ h,
const float* __restrict__ beta,
const float* __restrict__ gcum,
const float* __restrict__ A,
const float* __restrict__ P, int nchunks) {
const int hh = blockIdx.y, sl = blockIdx.x, j0 = sl * NS, qh = hh % HK;
const int w = threadIdx.x >> 5, c = threadIdx.x & 31, col = j0 + c;
// red is double-buffered by token parity: the w==0 lane reads token t's partials while the other warps
// may already be staging token t+1's into the other slot (one barrier per token, no reader/writer race)
__shared__ float red[2][2][4][NS], dlt[CH][NS], As[CH][CH], Ps[CH][CH], ex[CH][2];
float W[NS];
#pragma unroll
for (int r = 0; r < NS; ++r)
W[r] = state[((size_t) (w * NS + r) * HV + hh) * S + col];
const float rsqrt_S = rsqrtf((float) S);
for (int ch = 0; ch < nchunks; ++ch) {
const int64_t base = (size_t) ch * CH;
const float sg = gcum[(base + CH - 1) * HV + hh]; // sigma: the chunk's total log-decay
const float* Ap = A + ((size_t) ch * HV + hh) * CH * CH;
const float* Pp = P + ((size_t) ch * HV + hh) * CH * CH;
// stage the two 64x64 tiles and the per-token decay factors (exp(c_t), exp(sigma - c_t))
for (int i = threadIdx.x; i < CH * CH; i += blockDim.x) {
As[i / CH][i % CH] = Ap[i];
Ps[i / CH][i % CH] = Pp[i];
}
if (threadIdx.x < CH) {
const float gc = gcum[(base + threadIdx.x) * HV + hh];
ex[threadIdx.x][0] = expf(gc);
ex[threadIdx.x][1] = expf(sg - gc);
}
__syncthreads();
const float sg_exp = ex[CH - 1][0]; // exp(sigma)
for (int t = 0; t < CH; ++t) {
const float* kt = h + (base + t) * C + (size_t) HK * S + (size_t) qh * S + w * NS;
const float* qt = h + (base + t) * C + (size_t) qh * S + w * NS;
float up = 0.0f, up1 = 0.0f, qp = 0.0f, qp1 = 0.0f;
#pragma unroll
for (int r = 0; r < NS; r += 2) {
const float w0 = W[r], w1 = W[r + 1];
up = fmaf(kt[r], w0, up);
up1 = fmaf(kt[r + 1], w1, up1);
qp = fmaf(qt[r], w0, qp);
qp1 = fmaf(qt[r + 1], w1, qp1);
}
red[t & 1][0][w][c] = up + up1;
red[t & 1][1][w][c] = qp + qp1;
__syncthreads();
if (w == 0) {
const float u_t = red[t & 1][0][0][c] + red[t & 1][0][1][c] + red[t & 1][0][2][c] + red[t & 1][0][3][c];
const float qW_t =
red[t & 1][1][0][c] + red[t & 1][1][1][c] + red[t & 1][1][2][c] + red[t & 1][1][3][c];
const float b = beta[(base + t) * HV + hh];
const float v_t = h[(base + t) * C + (size_t) 2 * HK * S + (size_t) hh * S + col];
// delta_t = beta_t (v_t - exp(c_t) u_t) - sum_{s<t} A[t][s] delta_s
float d = b * (v_t - ex[t][0] * u_t), d1 = 0.0f;
#pragma unroll 1
for (int s = 0; s < t; s += 2) {
d = fmaf(-As[t][s], dlt[s][c], d);
if (s + 1 < t) d1 = fmaf(-As[t][s + 1], dlt[s + 1][c], d1);
}
d += d1;
dlt[t][c] = d;
float o = ex[t][0] * qW_t + Ps[t][t] * d, o1 = 0.0f;
#pragma unroll 1
for (int s = 0; s < t; s += 2) {
o = fmaf(Ps[t][s], dlt[s][c], o);
if (s + 1 < t) o1 = fmaf(Ps[t][s + 1], dlt[s + 1][c], o1);
}
oc[((base + t) * HV + hh) * S + col] = (o + o1) * rsqrt_S;
}
}
__syncthreads(); // the last dlt row is visible for the update
// W_out = exp(sigma) W_in + sum_s k_s exp(sigma - c_s) delta_s^T (the decay folded into the delta)
#pragma unroll
for (int r = 0; r < NS; ++r) W[r] *= sg_exp;
for (int s = 0; s < CH; ++s) {
const float* kt = h + (base + s) * C + (size_t) HK * S + (size_t) qh * S + w * NS;
const float ds = dlt[s][c] * ex[s][1];
#pragma unroll
for (int r = 0; r < NS; ++r) W[r] = fmaf(kt[r], ds, W[r]);
}
}
#pragma unroll
for (int r = 0; r < NS; ++r) state[((size_t) (w * NS + r) * HV + hh) * S + col] = W[r];
}
// ---------------------------------------------------------------- wmma (m16n16k16) variant
//
// The FMA scan above is correct but latency-bound: per token it streams k_t and q_t from global into
// dependent FMA chains, and V100 FMA throughput cannot pay for that at this granularity (measured on the
// V100: ~2x slower than the recurrence it was meant to beat). This variant runs the same math the way
// llama.cpp-v100's gdn-chunk-sm70 does it - q/k/v and the state staged through FP16, the matmuls on the
// FP16 tensor cores with FP32 accumulate, the triangular solve still FMA - and folds the kkt stage's Gram
// matrices into the same CTA (recomputed per column slice, which the tensor cores make cheap).
//
// Per chunk: stage k/q, -> f16; Gram K K^T and Q K^T -> A and P (decay + beta baked); the GEMM
// U = [K; Q] W gives u_t and qW_t for every token at once; the solve and output sums are the FMA scan's;
// W += K^T (delta * decay) on the tensor cores with W itself as the f32 accumulator fragments - the state
// never leaves the registers across chunks. W's f16 image exists only as the U GEMM's operand.
//
// Precision: the FP16 staging is the reference's "f32 -> f16 暂存" (10-bit mantissa on the operands);
// accumulations, the solve and the state are f32. FP32-level, not bit-exact - see gdn_chunk_parity.
#if !defined(__HIPCC__)
__global__ void __launch_bounds__(128) gdn_chunk_wmma_kernel(float* __restrict__ state, float* __restrict__ oc,
const float* __restrict__ h,
const float* __restrict__ beta,
const float* __restrict__ gcum, int nchunks) {
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700 // named in every device pass of a multi-arch build; only sm_70+ has wmma
namespace wmma = ::nvcuda::wmma;
// smem (84.5 KB, opt-in): Ks/Qs the chunk's k and q rows f16; W16 the state slice f16 (U's operand);
// Ug[128][32] f32 aliasing the 64x64 Gram scratch and then U = [K;Q] W; As/Ps f16; dlt f32 + f16.
extern __shared__ __align__(1024) unsigned char smem[];
__half* Ks = (__half*) smem; // [CH][S]
__half* Qs = Ks + CH * S; // [CH][S]
__half* W16 = Qs + CH * S; // [S][NS]
float* Ug = (float*) (W16 + S * NS); // [S][NS] f32 (Gram scratch / U out)
__half* As = (__half*) (Ug + S * NS); // [CH][CH]
__half* Ps = As + CH * CH; // [CH][CH]
float* dlt = (float*) (Ps + CH * CH); // [CH][NS]
__half* dlt16 = (__half*) (dlt + CH * NS); // [CH][NS]
const int hh = blockIdx.y, sl = blockIdx.x, j0 = sl * NS, qh = hh % HK;
const int w = threadIdx.x >> 5, c = threadIdx.x & 31, col = j0 + c;
// the state slice in through smem (load_matrix_sync of an accumulator tile is a shared-memory op on
// sm_70): Ug first as the f32 staging, then W's f32 image in the fragments and its f16 image in W16
for (int i = threadIdx.x; i < S * NS; i += blockDim.x) {
const int r = i / NS, j = i % NS;
Ug[i] = state[(size_t) r * HV * S + (size_t) hh * S + j0 + j];
W16[i] = __float2half_rn(Ug[i]);
}
__syncthreads();
// W as f32 accumulator fragments: warp w owns rows w*32..w*32+31 of the 128x32 slice, 2x2 tiles
wmma::fragment<wmma::accumulator, 16, 16, 16, float> wF[2][2];
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj)
wmma::load_matrix_sync(wF[mi][nj], Ug + (size_t) (w * 32 + 16 * mi) * NS + 16 * nj, NS,
wmma::mem_row_major);
const float rsqrt_S = rsqrtf((float) S);
for (int ch = 0; ch < nchunks; ++ch) {
const int64_t base = (size_t) ch * CH;
const float sg = gcum[(base + CH - 1) * HV + hh]; // sigma: the chunk's total log-decay
// stage the chunk's k and q rows through f16 - float4 loads, four in flight per thread: a scalar
// load loop here is a serial global-latency chain per thread (measured: it WAS the kernel's floor)
for (int i4 = threadIdx.x; i4 < CH * S / 4; i4 += blockDim.x) {
const int t = (i4 * 4) / S, j4 = (i4 * 4) % S;
const float4 k4 = *reinterpret_cast<const float4*>(h + (base + t) * C + (size_t) HK * S + (size_t) qh * S + j4);
const float4 q4 = *reinterpret_cast<const float4*>(h + (base + t) * C + (size_t) qh * S + j4);
Ks[t * S + j4 + 0] = __float2half_rn(k4.x);
Ks[t * S + j4 + 1] = __float2half_rn(k4.y);
Ks[t * S + j4 + 2] = __float2half_rn(k4.z);
Ks[t * S + j4 + 3] = __float2half_rn(k4.w);
Qs[t * S + j4 + 0] = __float2half_rn(q4.x);
Qs[t * S + j4 + 1] = __float2half_rn(q4.y);
Qs[t * S + j4 + 2] = __float2half_rn(q4.z);
Qs[t * S + j4 + 3] = __float2half_rn(q4.w);
}
// the state slice as f16: the fragments to Ug, then to W16
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj)
wmma::store_matrix_sync(Ug + (size_t) (w * 32 + 16 * mi) * NS + 16 * nj, wF[mi][nj], NS,
wmma::mem_row_major);
__syncthreads();
for (int i = threadIdx.x; i < S * NS; i += blockDim.x) W16[i] = __float2half_rn(Ug[i]);
__syncthreads(); // W16 done before Ug becomes the Gram scratch
// Gram: Ug = K K^T, then As[t][s] = beta_t exp(c_t-c_s) Ug[t][s]; Ug = Q K^T, then Ps (no beta)
{
wmma::fragment<wmma::matrix_a, 16, 16, 16, __half, wmma::row_major> aF;
wmma::fragment<wmma::matrix_b, 16, 16, 16, __half, wmma::col_major> bF;
wmma::fragment<wmma::accumulator, 16, 16, 16, float> gF[4];
for (int pass = 0; pass < 2; ++pass) {
#pragma unroll
for (int nj = 0; nj < 4; ++nj) wmma::fill_fragment(gF[nj], 0.0f);
for (int kk = 0; kk < S / 16; ++kk) {
wmma::load_matrix_sync(aF, (pass ? Qs : Ks) + (size_t) (w * 16) * S + kk * 16, S);
for (int nj = 0; nj < 4; ++nj) {
wmma::load_matrix_sync(bF, Ks + kk * 16 + (size_t) (16 * nj) * S, S);
wmma::mma_sync(gF[nj], aF, bF, gF[nj]);
}
}
for (int nj = 0; nj < 4; ++nj)
wmma::store_matrix_sync(Ug + (size_t) (w * 16) * CH + 16 * nj, gF[nj], CH, wmma::mem_row_major);
__syncthreads();
// fold decay (and beta) into the f16 tiles
for (int p = threadIdx.x; p < CH * CH; p += blockDim.x) {
const int t = p / CH, s = p % CH;
const float dec = expf(gcum[(base + t) * HV + hh] - gcum[(base + s) * HV + hh]);
if (pass) // Ps: s <= t live, the upper triangle zeroed - the o GEMM reads the whole tile
Ps[t * CH + s] = s <= t ? __float2half_rn(Ug[t * CH + s] * dec) : __float2half_rn(0.0f);
else if (s < t) // As: strict lower only, the solve reads exactly that
As[t * CH + s] = __float2half_rn(Ug[t * CH + s] * dec * beta[(base + t) * HV + hh]);
}
__syncthreads();
}
}
// U = [K; Q] W16 : rows 0..63 are u_t = k_t.W_in, rows 64..127 are qW_t = q_t.W_in. The 128 output
// rows split as 32 per warp: warps 0-1 the K half, warps 2-3 the Q half.
{
wmma::fragment<wmma::matrix_a, 16, 16, 16, __half, wmma::row_major> aF;
wmma::fragment<wmma::matrix_b, 16, 16, 16, __half, wmma::row_major> bF[2];
wmma::fragment<wmma::accumulator, 16, 16, 16, float> uF[2][2];
const __half* src = w < 2 ? Ks : Qs;
const int row0 = (w & 1) * 32;
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj) wmma::fill_fragment(uF[mi][nj], 0.0f);
for (int kk = 0; kk < S / 16; ++kk) {
for (int nj = 0; nj < 2; ++nj)
wmma::load_matrix_sync(bF[nj], W16 + kk * 16 * NS + 16 * nj, NS);
#pragma unroll
for (int mi = 0; mi < 2; ++mi) {
wmma::load_matrix_sync(aF, src + (size_t) (row0 + 16 * mi) * S + kk * 16, S);
for (int nj = 0; nj < 2; ++nj) wmma::mma_sync(uF[mi][nj], aF, bF[nj], uF[mi][nj]);
}
}
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj)
wmma::store_matrix_sync(Ug + (size_t) (w * 32 + 16 * mi) * NS + 16 * nj, uF[mi][nj], NS,
wmma::mem_row_major);
__syncthreads();
}
// the solve - 8 independent accumulator chains so the smem loads of one group hide behind the FMAs
// of the last (the 2-chain was smem-latency-bound and WAS half the kernel's runtime)
for (int t = 0; t < CH; ++t) {
if (w == 0) {
const float gc = gcum[(base + t) * HV + hh];
const float ex0 = expf(gc);
const float b = beta[(base + t) * HV + hh];
const float v_t = h[(base + t) * C + (size_t) 2 * HK * S + (size_t) hh * S + col];
float d[8];
d[0] = b * (v_t - ex0 * Ug[t * NS + c]);
#pragma unroll
for (int q = 1; q < 8; ++q) d[q] = 0.0f;
int s0 = 0;
#pragma unroll 1
for (; s0 + 8 <= t; s0 += 8) {
#pragma unroll
for (int q = 0; q < 8; ++q)
d[q] = fmaf(-__half2float(As[t * CH + s0 + q]), dlt[(s0 + q) * NS + c], d[q]);
}
#pragma unroll 1
for (; s0 < t; ++s0) d[0] = fmaf(-__half2float(As[t * CH + s0]), dlt[s0 * NS + c], d[0]);
dlt[t * NS + c] = (d[0] + d[1] + d[2] + d[3]) + (d[4] + d[5] + d[6] + d[7]);
}
}
__syncthreads(); // the deltas are complete
for (int i = threadIdx.x; i < CH * NS; i += blockDim.x) dlt16[i] = __float2half_rn(dlt[i]);
__syncthreads();
// o_intra = Ps delta on the tensor cores (the o FMA sum was the other latency chain), staged into the
// As tile (dead since the solve); 64 output rows split 16 per warp
{
wmma::fragment<wmma::matrix_a, 16, 16, 16, __half, wmma::row_major> aF;
wmma::fragment<wmma::matrix_b, 16, 16, 16, __half, wmma::row_major> bF[2];
wmma::fragment<wmma::accumulator, 16, 16, 16, float> oF[2];
for (int nj = 0; nj < 2; ++nj) wmma::fill_fragment(oF[nj], 0.0f);
for (int kk = 0; kk < CH / 16; ++kk) {
for (int nj = 0; nj < 2; ++nj)
wmma::load_matrix_sync(bF[nj], dlt16 + (size_t) (kk * 16) * NS + 16 * nj, NS);
wmma::load_matrix_sync(aF, Ps + (size_t) (w * 16) * CH + kk * 16, CH);
for (int nj = 0; nj < 2; ++nj) wmma::mma_sync(oF[nj], aF, bF[nj], oF[nj]);
}
for (int nj = 0; nj < 2; ++nj)
wmma::store_matrix_sync(Ug + S * NS + (size_t) (w * 16) * NS + 16 * nj, oF[nj], NS,
wmma::mem_row_major); // (the As tile's byte range)
}
__syncthreads();
for (int i = threadIdx.x; i < CH * NS; i += blockDim.x) {
const int t = i / NS;
oc[((base + t) * HV + hh) * S + j0 + (i % NS)] =
(Ug[S * NS + i] + expf(gcum[(base + t) * HV + hh]) * Ug[(64 + t) * NS + (i % NS)]) * rsqrt_S;
}
__syncthreads();
// dlt16 = f16(delta_s * exp(sigma - c_s)) - the decay folded in for the state update
for (int i = threadIdx.x; i < CH * NS; i += blockDim.x) {
const int s = i / NS;
dlt16[i] = __float2half_rn(__half2float(dlt16[i]) * expf(sg - gcum[(base + s) * HV + hh]));
}
const float sg_exp = expf(sg);
__syncthreads();
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj)
#pragma unroll
for (int q = 0; q < 8; ++q) wF[mi][nj].x[q] *= sg_exp;
{
{
wmma::fragment<wmma::matrix_a, 16, 16, 16, __half, wmma::col_major> aF;
wmma::fragment<wmma::matrix_b, 16, 16, 16, __half, wmma::row_major> bF[2];
for (int kk = 0; kk < CH / 16; ++kk) {
for (int nj = 0; nj < 2; ++nj)
wmma::load_matrix_sync(bF[nj], dlt16 + (size_t) (kk * 16) * NS + 16 * nj, NS);
#pragma unroll
for (int mi = 0; mi < 2; ++mi) {
wmma::load_matrix_sync(aF, Ks + (w * 32 + 16 * mi) + (size_t) (kk * 16) * S, S);
for (int nj = 0; nj < 2; ++nj) wmma::mma_sync(wF[mi][nj], aF, bF[nj], wF[mi][nj]);
}
} }
}
}
// the state slice out through smem as well
#pragma unroll
for (int mi = 0; mi < 2; ++mi)
#pragma unroll
for (int nj = 0; nj < 2; ++nj)
wmma::store_matrix_sync(Ug + (size_t) (w * 32 + 16 * mi) * NS + 16 * nj, wF[mi][nj], NS,
wmma::mem_row_major);
__syncthreads();
for (int i = threadIdx.x; i < S * NS; i += blockDim.x) {
const int r = i / NS, j = i % NS;
state[(size_t) r * HV * S + (size_t) hh * S + j0 + j] = Ug[i];
}
#endif
}
#endif // !HIPCC
// The grow-only workspace, one per (thread, device): a layer split runs the chunked recurrence on more than
// one card, and each card's pointers are only valid on that card - a single per-thread buffer would hand a
// second card the first card's pointers.
struct Ws {
float* A = nullptr; // [SEG][HV][CH][CH]
float* P = nullptr; // [SEG][HV][CH][CH]
float* gcum = nullptr; // [SEG*CH][HV]
size_t cap = 0; // in chunks
void ensure(size_t chunks) {
if (chunks <= cap) return;
if (A) cudaFree(A);
if (P) cudaFree(P);
if (gcum) cudaFree(gcum);
cudaGetLastError(); // a free of another device's pointer must not surface in the mallocs below
cap = chunks;
const size_t mat = chunks * HV * CH * CH;
if (cudaMalloc((void**) &A, mat * sizeof(float)) != cudaSuccess ||
cudaMalloc((void**) &P, mat * sizeof(float)) != cudaSuccess ||
cudaMalloc((void**) &gcum, (size_t) chunks * CH * HV * sizeof(float)) != cudaSuccess) {
int dev = 0;
cudaGetDevice(&dev);
size_t fb = 0, tb = 0;
cudaMemGetInfo(&fb, &tb);
std::fprintf(stderr, "gdn_chunk: workspace alloc failed (%zu chunks, device %d, %.1f of %.1f MiB free): %s\n",
chunks, dev, (double) fb / 1048576.0, (double) tb / 1048576.0,
cudaGetErrorString(cudaGetLastError()));
std::exit(1);
}
}
~Ws() {
// A prompt's stage threads are created per request (std::async in the layer split): without this, each
// request would leak its device's ~50 MB workspace until the card runs out.
if (A) cudaFree(A);
if (P) cudaFree(P);
if (gcum) cudaFree(gcum);
}
};
Ws& ws() {
static thread_local Ws w[8]; // [device]: the cards a layer split can run on in one process
int dev = 0;
if (cudaGetDevice(&dev) != cudaSuccess) {
cudaGetLastError();
dev = 0;
}
if (dev < 0 || dev >= 8) dev = 0;
return w[dev];
}
int gdn_chunk_cc_of(int dev) {
// Per DEVICE: a layer split runs on more than one card, and the capability is a card property - a
// process-global cache would dispatch a second card by the first card's capability.
static int cc[64] = {};
if (dev < 0 || dev >= 64) return 0;
if (cc[dev] == 0) {
cudaDeviceProp p{};
if (cudaGetDeviceProperties(&p, dev) != cudaSuccess) {
cudaGetLastError();
return 0;
}
cc[dev] = p.major * 10 + p.minor;
}
return cc[dev];
}
int gdn_chunk_cc() {
int dev = 0;
if (cudaGetDevice(&dev) != cudaSuccess) {
cudaGetLastError();
return 0;
}
return gdn_chunk_cc_of(dev);
}
} // namespace
bool gdn_chunk_available() {
#if defined(__HIPCC__) || !defined(STRATA_EXPERIMENTAL_SM60)
return false; // the chunked path is compiled and dispatched only in the Volta experimental build
#else
return gdn_chunk_cc() == 70;
#endif
}
void gdn_chunk_recurrence(float* state, const float* h, const float* gate, const float* beta, float* oc,
int64_t T, void* stream) {
#if defined(__HIPCC__)
(void) state; (void) h; (void) gate; (void) beta; (void) oc; (void) T; (void) stream;
std::fprintf(stderr, "gdn_chunk_recurrence: not built for HIP\n");
std::exit(1);
#else
if (T <= 0) return;
const int64_t T64 = T - T % CH;
if (T64 <= 0) return;
cudaStream_t cs = (cudaStream_t) stream;
// the wmma path is the default; STRATA_GDN_CHUNK=2 selects the FP32-only FMA scan (the parity test's
// oracle - same math, no FP16 staging) and anything below sm_70 has no wmma to use. Read per call (the
// tests flip it between launches).
const char* forced_v = std::getenv("STRATA_GDN_CHUNK");
const int forced = forced_v && *forced_v ? std::atoi(forced_v) : -1;
int dev = 0;
if (cudaGetDevice(&dev) != cudaSuccess) {
cudaGetLastError();
dev = 0;
}
#if !defined(__HIPCC__)
const bool wmma_ok = gdn_chunk_cc_of(dev) == 70;
#else
const bool wmma_ok = false;
#endif
// The shared-memory opt-in is a property of the DEVICE's context, not the process: a layer split runs
// this call on more than one card, and each card needs its own cudaFuncSetAttribute before that card's
// past-48KB launches (a process-global std::call_once left the second card at the 48 KB default, and its
// first chunked launch failed with "invalid argument"). Per device, once; every attempt clears the
// runtime error so a refused opt-in cannot surface in a later check() as a spurious failure.
static std::mutex attr_mu;
static bool attr_done[64] = {};
static bool wmma_attr[64] = {};
{
std::lock_guard<std::mutex> lk(attr_mu);
if (dev >= 0 && dev < 64 && !attr_done[dev]) {
const cudaError_t ek = cudaFuncSetAttribute(gdn_chunk_kkt_kernel,
cudaFuncAttributeMaxDynamicSharedMemorySize,
2 * CH * (S + 1) * (int) sizeof(float));
const cudaError_t ew = cudaFuncSetAttribute(gdn_chunk_wmma_kernel,
cudaFuncAttributeMaxDynamicSharedMemorySize, WMMA_SMEM);
cudaGetLastError();
(void) ek; // a refused opt-in leaves the FMA path's own launch to report it
wmma_attr[dev] = ew == cudaSuccess;
attr_done[dev] = true;
}
}
const bool use_wmma = wmma_ok && forced != 2 && dev >= 0 && dev < 64 && wmma_attr[dev];
Ws& w = ws();
const size_t chunks_total = (size_t) (T64 / CH);
w.ensure(chunks_total < SEG ? (chunks_total ? chunks_total : 1) : SEG);
// STRATA_GDN_CHUNK_PROF=1: per-stage milliseconds over the whole call (a dev tool, not a hot path)
static const bool prof = std::getenv("STRATA_GDN_CHUNK_PROF") != nullptr;
cudaEvent_t pe0{}, pe1{};
float ms[3] = {0, 0, 0};
if (prof) cudaEventCreate(&pe0), cudaEventCreate(&pe1);
for (int64_t t0 = 0; t0 < T64; t0 += (int64_t) SEG * CH) {
const int64_t left = (T64 - t0) / CH;
const int nch = (int) (left < SEG ? left : SEG);
const int ntok = nch * CH;
if (prof) cudaEventRecord(pe0, cs);
gdn_chunk_cumsum_kernel<<<dim3((unsigned) nch, HV), CH, 0, cs>>>(w.gcum, gate + t0 * HV, ntok);
check("cumsum");
if (prof) {
cudaEventRecord(pe1, cs);
cudaEventSynchronize(pe1);
cudaEventElapsedTime(&ms[0], pe0, pe1);
cudaEventRecord(pe0, cs);
}
if (use_wmma) {
gdn_chunk_wmma_kernel<<<dim3(NSE, HV), 128, WMMA_SMEM, cs>>>(
state, oc + t0 * HV * S, h + t0 * C, beta + t0 * HV, w.gcum, nch);
check("wmma");
} else {
gdn_chunk_kkt_kernel<<<dim3((unsigned) nch, HV), 128, 2 * CH * (S + 1) * sizeof(float), cs>>>(
w.A, w.P, h + t0 * C, beta + t0 * HV, w.gcum, nch);
check("kkt");
}
if (prof) {
cudaEventRecord(pe1, cs);
cudaEventSynchronize(pe1);
cudaEventElapsedTime(&ms[1], pe0, pe1);
cudaEventRecord(pe0, cs);
}
if (!use_wmma) {
gdn_chunk_fwd_kernel<<<dim3(NSE, HV), 128, 0, cs>>>(state, oc + t0 * HV * S, h + t0 * C, beta + t0 * HV,
w.gcum, w.A, w.P, nch);
check("fwd");
}
if (prof) {
cudaEventRecord(pe1, cs);
cudaEventSynchronize(pe1);
cudaEventElapsedTime(&ms[2], pe0, pe1);
}
}
if (prof) {
std::printf("gdn_chunk prof T=%lld: cumsum %.3fms stage2 %.3fms stage3 %.3fms\n", (long long) T, ms[0],
ms[1], ms[2]);
cudaEventDestroy(pe0);
cudaEventDestroy(pe1);
}
#endif
}
} // namespace strata::prefill