Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/prefill/gdn_rec_parity.cu - the prompt path's DeltaNet recurrence, bit for bit and timed.
//
// src/prefill/kernels.cu runs gdn_rec_kh_kernel (the three value heads that share a key head in one thread, the
// inputs staged in blocks of 8 tokens) on CUDA sm_80+ cards that hold all 64 of its blocks at once, and
// gdn_rec_cols_pipe_kernel elsewhere and with STRATA_GDN_KEYHEAD=0. Both are copied below verbatim, with the output
// norm (gdn_out_norm_kernel) and the norm as it was before, which also stored the normalized value in FP32 although
// nothing reads it. The two recurrences do the same arithmetic in the same order per value head and column, so the
// output, the state after it and the FP16 output must be the same bits. STRATA_GDN_CHUNKED=1's recurrence in chunks
// (copied below too) sums in another order: it must stay within 1e-4 of gdn_rec_kh_kernel (largest |difference| over
// largest |value|, the output and the state), also at T = 8192 and 32768; up to 8192 both are also measured against
// the recurrence in FP64. STRATA_GDN_TC=1's tensor-core scan (gdn_tc.cuh, included below) is held to the same 1e-4,
// also on stress inputs (the state 100x smaller, 50x larger or zero, v 50x larger, slow gates, coherent keys).
//
// Synthetic inputs of the model's shape (q/k L2-normalized per key head as gdn_l2_kernel leaves them), no model.
// gdn_rec_parity the bit checks (T from 1 to 4099, a non-zero state as after an earlier chunk)
// gdn_rec_parity --long and the tensor-core scan at T = 51820 (needs ~7 GB)
// gdn_rec_parity --selftest-frag the tensor-core scan's fragment helpers alone, one warp each, against FP64
// gdn_rec_parity --bench-tc the tensor-core series alone (gdn_rec_kh_kernel and the scan alternating, nothing else:
// for A / B runs of two builds back to back)
// gdn_rec_parity --bench and the timings at T = 2048, 8192 and 32768 (one layer, all 48 value heads); the two
// variants' launches alternate, since a card under load steps its clock down after a
// fraction of a second (see native_grouped_parity); then both at 1 to 6 blocks per
// SM, as on cards with more or fewer SMs (scale), and both with the engine's grids on
// 32 to all of this card's SMs, the others held by a sleeping kernel (fewer_sms)
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <vector>
namespace {
constexpr int S = 128, HK = 16, HV = 48, C = 10240; // as kernels.cu: head size, key heads, value heads, q|k|v width
constexpr int RG = 4, RPG = S / RG;
constexpr int CB = 32, NCB = S / CB;
void ck(cudaError_t e, const char* what) {
if (e != cudaSuccess) {
std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
std::exit(2);
}
}
__device__ __forceinline__ float warp_sum(float v) {
#pragma unroll
for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
return v;
}
__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }
__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }
// ------------------------------------------------------------------ the engine's kernels (src/prefill/kernels.cu)
// gdn_rec_cols_kernel with the next token's inputs (q/k rows, v, gate, beta) loaded into registers while this token
// computes (software pipelining). The same arithmetic in the same order: the same bits, and the same CB-column split.
// STRATA_GDN_PIPELINE=0: gdn_rec_cols_kernel.
__global__ void __launch_bounds__(CB * RG) gdn_rec_cols_pipe_kernel(float* __restrict__ state, const float* __restrict__ h,
const float* __restrict__ gate,
const float* __restrict__ beta,
float* __restrict__ oc_out, int64_t T) {
constexpr int NT = CB * RG, LPT = S / NT; // threads, q/k rows loaded per thread
__shared__ float sk[S], sq[S], red[RG][CB];
const int head = blockIdx.x / NCB, cb = blockIdx.x % NCB;
const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
const int qh = head % HK;
float s[RPG];
float* base = state + ((size_t) (rg * RPG) * HV + head) * S + col;
const size_t rs = (size_t) HV * S;
#pragma unroll
for (int r = 0; r < RPG; ++r) s[r] = base[r * rs];
float nq[LPT], nk[LPT], nv = 0.0f, ng = 0.0f, nb = 0.0f;
auto fetch = [&](int64_t t) {
const float* ht = h + t * C;
#pragma unroll
for (int u = 0; u < LPT; ++u) { nq[u] = ht[qh * S + tid + u * NT]; nk[u] = ht[HK * S + qh * S + tid + u * NT]; }
nv = ht[2 * HK * S + head * S + col];
ng = gate[t * HV + head];
nb = beta[t * HV + head];
};
if (T > 0) fetch(0);
for (int64_t t = 0; t < T; ++t) {
float cq[LPT], ck[LPT];
#pragma unroll
for (int u = 0; u < LPT; ++u) { cq[u] = nq[u]; ck[u] = nk[u]; }
const float cv = nv, cg = ng, cbt = nb;
__syncthreads();
#pragma unroll
for (int u = 0; u < LPT; ++u) { sq[tid + u * NT] = cq[u]; sk[tid + u * NT] = ck[u]; }
__syncthreads();
if (t + 1 < T) fetch(t + 1);
const float g = __expf(cg);
float kv = 0.0f;
#pragma unroll
for (int r = 0; r < RPG; ++r) kv = fmaf(s[r], sk[rg * RPG + r], kv);
red[rg][c] = kv;
__syncthreads();
const float kv_col = red[0][c] + red[1][c] + red[2][c] + red[3][c];
const float delta = (cv - g * kv_col) * cbt;
float o = 0.0f;
#pragma unroll
for (int r = 0; r < RPG; ++r) {
s[r] = fmaf(g, s[r], sk[rg * RPG + r] * delta);
o = fmaf(s[r], sq[rg * RPG + r], o);
}
__syncthreads();
red[rg][c] = o;
__syncthreads();
if (rg == 0) oc_out[t * HV * S + head * S + col] = (red[0][c] + red[1][c] + red[2][c] + red[3][c]) * rsqrtf((float) S);
}
#pragma unroll
for (int r = 0; r < RPG; ++r) base[r * rs] = s[r];
}
// The recurrence with one thread for the three value heads that share a key head (head % HK): column c of heads
// qh, qh + 16 and qh + 32, row group rg. gdn_rec_cols_pipe_kernel spends its time in shared memory, not in
// arithmetic: every thread of a warp needs the same 32 q and k values per token (the k twice), and a warp receives one
// such broadcast value per clock however wide the load. Here every q/k value a thread loads feeds three heads, and a
// token's k row goes into registers once for both of its uses. The inputs come in blocks of GDN_TB tokens, copied to
// shared memory by cp.async while the block before computes (one token ahead is shorter than a load from L2 takes),
// and the two cross-row-group sums have their own arrays, so a token needs 2 __syncthreads instead of 5: the second
// one of a token orders every read of rkv before the next token's writes, the next token's first one every read of ro
// before the writes after it. 64 blocks instead of 192. Per value head and column the same arithmetic in the same
// order: the same bits (src/prefill/gdn_rec_parity.cu checks them and times the variants: 1.41x on a 4080 Super).
// sm_80+ (gdn_keyhead_ok); STRATA_GDN_KEYHEAD=0: gdn_rec_cols_pipe_kernel.
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
#define STRATA_GDN_CP_ASYNC 0 // Turing builds: plain copies (never launched there, see gdn_keyhead_ok)
#else
#define STRATA_GDN_CP_ASYNC 1
#endif
__device__ __forceinline__ void gdn_cp4(float* smem, const float* gmem) {
#if STRATA_GDN_CP_ASYNC
asm volatile("cp.async.ca.shared.global [%0], [%1], 4;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
#else
*smem = *gmem;
#endif
}
__device__ __forceinline__ void gdn_cp16(float* smem, const float* gmem) {
#if STRATA_GDN_CP_ASYNC
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((unsigned) __cvta_generic_to_shared(smem)), "l"(gmem));
#else
*reinterpret_cast<float4*>(smem) = *reinterpret_cast<const float4*>(gmem);
#endif
}
__device__ __forceinline__ void gdn_cp_commit() {
#if STRATA_GDN_CP_ASYNC
asm volatile("cp.async.commit_group;\n" ::);
#endif
}
__device__ __forceinline__ void gdn_cp_wait_prev() { // every group but the newest has landed
#if STRATA_GDN_CP_ASYNC
asm volatile("cp.async.wait_group 1;\n" ::);
#endif
}
constexpr int GDN_TB = 8, VPK = HV / HK; // tokens per staged block, value heads per key head
__global__ void __launch_bounds__(CB * RG) gdn_rec_kh_kernel(float* __restrict__ state, const float* __restrict__ h,
const float* __restrict__ gate,
const float* __restrict__ beta,
float* __restrict__ oc_out, int64_t T) {
constexpr int TB = GDN_TB, NT = CB * RG, QKP = S / 4, VP = CB / 4; // threads, 16-byte pieces of a q/k row, of v
__shared__ __align__(16) float sq[2][TB][S];
__shared__ __align__(16) float sk[2][TB][S];
__shared__ __align__(16) float sv[2][TB][VPK][CB];
__shared__ float sg[2][TB][VPK], sb[2][TB][VPK], rkv[VPK][RG][CB], ro[VPK][RG][CB];
const int qh = blockIdx.x / NCB, cb = blockIdx.x % NCB;
const int c = threadIdx.x, rg = threadIdx.y, tid = rg * CB + c, col = cb * CB + c;
float s[VPK][RPG];
const size_t rs = (size_t) HV * S;
#pragma unroll
for (int j = 0; j < VPK; ++j) {
const float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
#pragma unroll
for (int r = 0; r < RPG; ++r) s[j][r] = base[r * rs];
}
const int64_t nblk = (T + TB - 1) / TB;
auto stage = [&](int64_t k) { // tokens [k * TB, k * TB + TB) into buffer k & 1
const int bb = (int) (k & 1);
const int64_t t0 = k * TB;
for (int p = tid; p < TB * 2 * QKP; p += NT) {
const int i = p / (2 * QKP), w = p % (2 * QKP), isk = w / QKP, jj = (w % QKP) * 4;
if (t0 + i < T)
gdn_cp16(isk ? &sk[bb][i][jj] : &sq[bb][i][jj], h + (t0 + i) * C + (isk ? HK * S : 0) + qh * S + jj);
}
for (int p = tid; p < TB * VPK * VP; p += NT) {
const int i = p / (VPK * VP), w = p % (VPK * VP), j = w / VP, jj = (w % VP) * 4;
if (t0 + i < T)
gdn_cp16(&sv[bb][i][j][jj], h + (t0 + i) * C + 2 * HK * S + (qh + j * HK) * S + cb * CB + jj);
}
for (int p = tid; p < 2 * TB * VPK; p += NT) {
const int isb = p / (TB * VPK), w = p % (TB * VPK), i = w / VPK, j = w % VPK;
if (t0 + i < T)
gdn_cp4(isb ? &sb[bb][i][j] : &sg[bb][i][j], (isb ? beta : gate) + (t0 + i) * HV + qh + j * HK);
}
};
if (nblk > 0) stage(0);
gdn_cp_commit();
for (int64_t k = 0; k < nblk; ++k) {
// buffer (k + 1) & 1 was read by block k - 1, whose last token's second __syncthreads every thread has passed
if (k + 1 < nblk) stage(k + 1);
gdn_cp_commit();
gdn_cp_wait_prev();
__syncthreads();
const int bb = (int) (k & 1);
const int n = (int) ((T - k * TB) < TB ? (T - k * TB) : TB);
for (int i = 0; i < n; ++i) {
const int64_t t = k * TB + i;
float kc[RPG];
#pragma unroll
for (int r = 0; r < RPG; ++r) kc[r] = sk[bb][i][rg * RPG + r];
float g[VPK], kv[VPK], delta[VPK], o[VPK];
#pragma unroll
for (int j = 0; j < VPK; ++j) { g[j] = __expf(sg[bb][i][j]); kv[j] = 0.0f; o[j] = 0.0f; }
#pragma unroll
for (int r = 0; r < RPG; ++r)
#pragma unroll
for (int j = 0; j < VPK; ++j) kv[j] = fmaf(s[j][r], kc[r], kv[j]);
#pragma unroll
for (int j = 0; j < VPK; ++j) rkv[j][rg][c] = kv[j];
__syncthreads();
#pragma unroll
for (int j = 0; j < VPK; ++j) {
const float kv_col = rkv[j][0][c] + rkv[j][1][c] + rkv[j][2][c] + rkv[j][3][c];
delta[j] = (sv[bb][i][j][c] - g[j] * kv_col) * sb[bb][i][j];
}
#pragma unroll
for (int r = 0; r < RPG; ++r) {
const float qr = sq[bb][i][rg * RPG + r];
#pragma unroll
for (int j = 0; j < VPK; ++j) {
s[j][r] = fmaf(g[j], s[j][r], kc[r] * delta[j]);
o[j] = fmaf(s[j][r], qr, o[j]);
}
}
#pragma unroll
for (int j = 0; j < VPK; ++j) ro[j][rg][c] = o[j];
__syncthreads();
if (rg < VPK) // row group j writes head j's output
oc_out[t * HV * S + (qh + rg * HK) * S + col] =
(ro[rg][0][c] + ro[rg][1][c] + ro[rg][2][c] + ro[rg][3][c]) * rsqrtf((float) S);
}
}
#pragma unroll
for (int j = 0; j < VPK; ++j) {
float* base = state + ((size_t) (rg * RPG) * HV + qh + j * HK) * S + col;
#pragma unroll
for (int r = 0; r < RPG; ++r) base[r * rs] = s[j][r];
}
}
// src/prefill/kernels.cu's chunked recurrence (STRATA_GDN_CHUNKED=1), copied as it is there
// STRATA_GDN_CHUNKED=1 (opt-in): the recurrence in chunks of GCH tokens (the WY form of the gated delta rule, as
// flash-linear-attention computes it), all in FP32. Per value head, with g_t = exp(gate_t), gamma_t the gate's sum
// from the chunk's start to t (inclusive) and S0 the state before the chunk:
// A[t][i] = beta_t exp(gamma_t - gamma_i) k_t.k_i (i < t), T = (I + A)^-1 (unit lower triangular),
// Y_t = beta_t (v_t - exp(gamma_t) S0^T k_t), D = T Y (the delta rule's corrections, one per token),
// o_t = exp(gamma_t) S0^T q_t + sum_{i <= t} exp(gamma_t - gamma_i) (q_t.k_i) D_i,
// S = exp(gamma_last) S0 + sum_t exp(gamma_last - gamma_t) k_t D_t^T,
// the same state and outputs as the token-by-token kernels up to FP32 rounding (another order of the sums).
// gdn_chunk_prep_kernel builds T, P = exp(gamma_t - gamma_i) q_t.k_i (i <= t) and gamma for every chunk of a
// super-block at once (a block per chunk and key head: its three value heads share k and q); gdn_chunk_scan_kernel then
// walks the super-block's chunks per (value head, 16 columns): the state slice in registers, the products as small
// register-tiled matmuls from shared memory. Every exponent is <= 0 (gates are negative), so nothing overflows.
constexpr int GCH = 32; // tokens per chunk (the prep's solve is a lane per token)
constexpr int GSB = 2048; // tokens per super-block: the prep's scratch is GSB / GCH chunks of T, P and gamma
constexpr int GDV = 16; // value columns per scan block
constexpr int GKP = S + 4; // the scan's padded q / k rows (8 rows of a warp land on 8 banks)
constexpr int GTP = GCH + 1; // padded T / P rows
static_assert(GCH == 32, "gdn_chunk_prep_kernel: a lane per token of the chunk");
constexpr size_t kChunkPrepSmem = sizeof(float) * (2 * GCH * (S + 1) + 2 * GCH * GTP); // A over k / q
static_assert(VPK * GCH * GTP <= 2 * GCH * (S + 1), "gdn_chunk_prep_kernel: A fits where k and q were");
constexpr size_t kChunkScanSmem = sizeof(float) * (3 * GCH * GKP + VPK * (S * GDV + 3 * GCH * GDV));
__global__ void __launch_bounds__(256) gdn_chunk_prep_kernel(const float* __restrict__ h, const float* __restrict__ gate,
const float* __restrict__ beta, float* __restrict__ tm,
float* __restrict__ pm, float* __restrict__ gm, int64_t t0,
int64_t T) {
extern __shared__ __align__(16) float gsm[];
float* sk = gsm; // [GCH][S + 1]
float* sq = sk + GCH * (S + 1); // [GCH][S + 1]
float* skk = sq + GCH * (S + 1); // [GCH][GTP] k_t.k_i
float* sqk = skk + GCH * GTP; // [GCH][GTP] q_t.k_i
float* sa = gsm; // [VPK][GCH][GTP] A, then T below the diagonal (over k and q:
// read by then, so two blocks share an SM)
const int qh = blockIdx.y, ch = blockIdx.x, tid = threadIdx.x, lane = tid & 31, warp = tid >> 5;
const int64_t tb = t0 + (int64_t) ch * GCH;
const int n = (int) min((int64_t) GCH, T - tb);
{ // k and q as float4, every load issued before the first store
constexpr int NP = GCH * S / 4 / 256; // float4 a thread, each of k and q
float4 kv[NP], qv[NP];
#pragma unroll
for (int m = 0; m < NP; ++m) {
const int p = tid + 256 * m, t = p / (S / 4), r = (p % (S / 4)) * 4;
kv[m] = t < n ? *reinterpret_cast<const float4*>(h + (tb + t) * C + HK * S + qh * S + r) : make_float4(0.f, 0.f, 0.f, 0.f);
qv[m] = t < n ? *reinterpret_cast<const float4*>(h + (tb + t) * C + qh * S + r) : make_float4(0.f, 0.f, 0.f, 0.f);
}
#pragma unroll
for (int m = 0; m < NP; ++m) {
const int p = tid + 256 * m, t = p / (S / 4), r = (p % (S / 4)) * 4;
float* dk = sk + t * (S + 1) + r;
float* dq = sq + t * (S + 1) + r;
dk[0] = kv[m].x; dk[1] = kv[m].y; dk[2] = kv[m].z; dk[3] = kv[m].w;
dq[0] = qv[m].x; dq[1] = qv[m].y; dq[2] = qv[m].z; dq[3] = qv[m].w;
}
}
__syncthreads();
{ // lane i, rows warp + 8m: k_t.k_i and q_t.k_i on eight independent sums (rows 129 apart: 32 banks)
constexpr int RW = GCH / 8;
float kk[RW] = {}, qk[RW] = {};
for (int r = 0; r < S; ++r) {
const float ki = sk[lane * (S + 1) + r];
#pragma unroll
for (int m = 0; m < RW; ++m) {
kk[m] = fmaf(sk[(warp + 8 * m) * (S + 1) + r], ki, kk[m]);
qk[m] = fmaf(sq[(warp + 8 * m) * (S + 1) + r], ki, qk[m]);
}
}
#pragma unroll
for (int m = 0; m < RW; ++m) {
const int t = warp + 8 * m;
skk[t * GTP + lane] = lane <= t ? kk[m] : 0.0f;
sqk[t * GTP + lane] = lane <= t ? qk[m] : 0.0f;
}
}
__syncthreads();
if (warp >= VPK) return;
const int vh = qh + warp * HK, i = lane; // warp j: value head j of the key head; lane i: token i (column i)
float* a = sa + warp * GCH * GTP;
const float b = i < n ? beta[(tb + i) * HV + vh] : 0.0f;
float gam = i < n ? gate[(tb + i) * HV + vh] : 0.0f; // past the end: gate 0, beta 0 (no effect on S)
#pragma unroll
for (int o = 1; o < 32; o <<= 1) {
const float u = __shfl_up_sync(0xffffffffu, gam, o);
if (lane >= o) gam += u;
}
for (int t = 0; t < GCH; ++t) {
const float gt = __shfl_sync(0xffffffffu, gam, t), bt = __shfl_sync(0xffffffffu, b, t);
a[t * GTP + i] = i < t ? bt * expf(gt - gam) * skk[t * GTP + i] : (i == t ? 1.0f : 0.0f);
}
__syncwarp();
// T = (I + A)^-1 by forward substitution, row by row in place: T[t][i] = -sum_{k < t} A[t][k] T[k][i] over the
// finished rows (diagonal 1, zeros above it: the terms k < i add an exact 0, so the same bits as summing from k = i)
// - a trip the whole warp shares, so the loads go ahead of the sums
for (int t = 1; t < GCH; ++t) {
float v = 0.0f;
#pragma unroll 8
for (int k = 0; k < t; ++k) v = fmaf(-a[t * GTP + k], a[k * GTP + i], v);
__syncwarp();
if (i < t) a[t * GTP + i] = v;
__syncwarp();
}
float* T_ = tm + ((int64_t) ch * HV + vh) * GCH * GCH;
float* P_ = pm + ((int64_t) ch * HV + vh) * GCH * GCH;
for (int t = 0; t < GCH; ++t) {
const float gt = __shfl_sync(0xffffffffu, gam, t);
T_[t * GCH + i] = a[t * GTP + i];
P_[t * GCH + i] = i <= t ? expf(gt - gam) * sqk[t * GTP + i] : 0.0f;
}
gm[((int64_t) ch * HV + vh) * GCH + i] = gam;
}
__device__ __forceinline__ void gdn_pf_l2(const float* p) { asm volatile("prefetch.global.L2 [%0];" ::"l"(p)); }
__device__ __forceinline__ void gdn_cp_wait_all() {
#if STRATA_GDN_CP_ASYNC
asm volatile("cp.async.wait_all;\n" ::);
#endif
}
// A block per (key head, 16 value columns): 128 threads for each of the key head's three value heads, which share the
// staged k and q rows. 128 blocks, one an SM, in one wave (each walks every chunk: a second wave would double the
// time). A thread owns one token's row of the products (4 columns) and holds that token's T and P rows in registers,
// loaded at the chunk's start so they land during the first product; the next chunk's k (a second buffer), q (after
// this chunk's last read of q), v, gamma and beta load while this chunk computes. Three barriers a chunk.
__global__ void __launch_bounds__(128 * VPK) gdn_chunk_scan_kernel(float* __restrict__ state, const float* __restrict__ h,
const float* __restrict__ beta, const float* __restrict__ tm,
const float* __restrict__ pm, const float* __restrict__ gm,
float* __restrict__ oc_out, int64_t t0, int nch, int64_t T) {
extern __shared__ __align__(16) float gsm[];
constexpr int NT = 128 * VPK, HEAD = S * GDV + 3 * GCH * GDV; // threads; a head's shared floats
float* kbuf = gsm; // [2][GCH][GKP] k, double-buffered
float* sq = kbuf + 2 * GCH * GKP; // [GCH][GKP]
const int qh = blockIdx.x, c0 = blockIdx.y * GDV, j = threadIdx.x / 128, tid = threadIdx.x % 128;
const int vh = qh + j * HK;
float* ss = sq + GCH * GKP + j * HEAD; // [S][GDV] the state slice at the chunk's start
float* sy = ss + S * GDV; // [GCH][GDV] Y
float* sd = sy + GCH * GDV; // [GCH][GDV] D
float* sdd = sd + GCH * GDV; // [GCH][GDV] exp(gamma_last - gamma_t) D
const int rq = tid >> 2, vq = tid & 3; // the update's tile: rows 4rq..4rq+3; both tiles: columns 4vq..4vq+3
const int tt = tid >> 2, vv = 4 * vq; // the products' tile: token tt
const size_t rs = (size_t) HV * S;
auto stage_kq = [&](int64_t c_t0, int n, float* dst, bool isq) { // k or q rows of a chunk, cp.async
for (int p = threadIdx.x; p < GCH * (S / 4); p += NT) {
const int t = p / (S / 4), r = (p % (S / 4)) * 4;
if (t < n) gdn_cp16(dst + t * GKP + r, h + (c_t0 + t) * C + (isq ? 0 : HK * S) + qh * S + r);
else *reinterpret_cast<float4*>(dst + t * GKP + r) = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
}
};
// this tile's v, gamma, beta and the chunk's last gamma (registers)
auto load_vgb = [&](int ch, int64_t c_t0, int n, float4& v4, float& gam, float& bet, float& glast) {
v4 = tt < n ? __ldg(reinterpret_cast<const float4*>(h + (c_t0 + tt) * C + 2 * HK * S + vh * S + c0 + vv))
: make_float4(0.0f, 0.0f, 0.0f, 0.0f);
const float* g = gm + ((int64_t) ch * HV + vh) * GCH;
gam = __ldg(g + tt);
glast = __ldg(g + GCH - 1);
bet = tt < n ? __ldg(beta + (c_t0 + tt) * HV + vh) : 0.0f;
};
float s[4][4];
#pragma unroll
for (int x = 0; x < 4; ++x) {
#pragma unroll
for (int y = 0; y < 4; ++y) s[x][y] = state[(size_t) (4 * rq + x) * rs + (size_t) vh * S + c0 + vv + y];
*reinterpret_cast<float4*>(&ss[(4 * rq + x) * GDV + vv]) = make_float4(s[x][0], s[x][1], s[x][2], s[x][3]);
}
const float out_scale = rsqrtf((float) S);
float4 v4;
float gam, bet, glast;
{
const int n0 = (int) min((int64_t) GCH, T - t0);
stage_kq(t0, n0, kbuf, false);
stage_kq(t0, n0, sq, true);
gdn_cp_commit();
load_vgb(0, t0, n0, v4, gam, bet, glast);
gdn_cp_wait_all();
__syncthreads();
}
for (int ch = 0; ch < nch; ++ch) {
const int64_t c_t0 = t0 + (int64_t) ch * GCH;
const int n = (int) min((int64_t) GCH, T - c_t0);
const bool more = ch + 1 < nch;
const int64_t n_t0 = c_t0 + GCH;
const int nn = more ? (int) min((int64_t) GCH, T - n_t0) : 0;
const float* sk = kbuf + (ch & 1) * GCH * GKP;
// this token's T and P rows into registers now: they land while the first product runs
float tr[GCH], pr[GCH];
{
const float4* T4 = reinterpret_cast<const float4*>(tm + (((int64_t) ch * HV + vh) * GCH + tt) * GCH);
const float4* P4 = reinterpret_cast<const float4*>(pm + (((int64_t) ch * HV + vh) * GCH + tt) * GCH);
#pragma unroll
for (int i = 0; i < GCH / 4; ++i) {
const float4 a = __ldg(T4 + i), b = __ldg(P4 + i);
tr[4 * i] = a.x; tr[4 * i + 1] = a.y; tr[4 * i + 2] = a.z; tr[4 * i + 3] = a.w;
pr[4 * i] = b.x; pr[4 * i + 1] = b.y; pr[4 * i + 2] = b.z; pr[4 * i + 3] = b.w;
}
}
float4 nv4 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
float ngam = 0.0f, nbet = 0.0f, nglast = 0.0f;
if (more) { // the next chunk's v, gamma and beta; the one after into L2 (its k and q: after barrier A)
load_vgb(ch + 1, n_t0, nn, nv4, ngam, nbet, nglast);
const int64_t f_t0 = n_t0 + GCH;
const int t = threadIdx.x >> 3, l = threadIdx.x & 7;
if (t < GCH && f_t0 + t < T) gdn_pf_l2(h + (f_t0 + t) * C + (l < 4 ? HK * S : 0) + qh * S + 32 * (l & 3));
if (tid < GCH) {
gdn_pf_l2(tm + (((int64_t) (ch + 1) * HV + vh) * GCH + tid) * GCH);
gdn_pf_l2(pm + (((int64_t) (ch + 1) * HV + vh) * GCH + tid) * GCH);
}
}
// X = K S0 and Q S0 for token tt, then Y = beta (V - exp(gamma) X)
float x[4] = {0.f, 0.f, 0.f, 0.f}, qs[4] = {0.f, 0.f, 0.f, 0.f};
#pragma unroll 4
for (int r = 0; r < S; r += 4) {
const float4 k4 = *reinterpret_cast<const float4*>(&sk[tt * GKP + r]);
const float4 q4 = *reinterpret_cast<const float4*>(&sq[tt * GKP + r]);
const float kk[4] = {k4.x, k4.y, k4.z, k4.w}, qq[4] = {q4.x, q4.y, q4.z, q4.w};
#pragma unroll
for (int jj = 0; jj < 4; ++jj) {
const float4 s4 = *reinterpret_cast<const float4*>(&ss[(r + jj) * GDV + vv]);
x[0] = fmaf(kk[jj], s4.x, x[0]); x[1] = fmaf(kk[jj], s4.y, x[1]);
x[2] = fmaf(kk[jj], s4.z, x[2]); x[3] = fmaf(kk[jj], s4.w, x[3]);
qs[0] = fmaf(qq[jj], s4.x, qs[0]); qs[1] = fmaf(qq[jj], s4.y, qs[1]);
qs[2] = fmaf(qq[jj], s4.z, qs[2]); qs[3] = fmaf(qq[jj], s4.w, qs[3]);
}
}
const float e = expf(gam);
*reinterpret_cast<float4*>(&sy[tt * GDV + vv]) = make_float4(bet * (v4.x - e * x[0]), bet * (v4.y - e * x[1]),
bet * (v4.z - e * x[2]), bet * (v4.w - e * x[3]));
__syncthreads(); // (A) Y complete; every read of q done
if (more) { // the next chunk's k into the other buffer, its q into the single q buffer
stage_kq(n_t0, nn, kbuf + ((ch + 1) & 1) * GCH * GKP, false);
stage_kq(n_t0, nn, sq, true);
gdn_cp_commit();
}
// D = T Y (T is zero above its diagonal), and D decayed to the chunk's end for the update
float d[4] = {0.f, 0.f, 0.f, 0.f};
#pragma unroll
for (int i = 0; i < GCH; ++i) {
const float4 y4 = *reinterpret_cast<const float4*>(&sy[i * GDV + vv]);
d[0] = fmaf(tr[i], y4.x, d[0]); d[1] = fmaf(tr[i], y4.y, d[1]);
d[2] = fmaf(tr[i], y4.z, d[2]); d[3] = fmaf(tr[i], y4.w, d[3]);
}
const float f = expf(glast - gam);
*reinterpret_cast<float4*>(&sd[tt * GDV + vv]) = make_float4(d[0], d[1], d[2], d[3]);
*reinterpret_cast<float4*>(&sdd[tt * GDV + vv]) = make_float4(f * d[0], f * d[1], f * d[2], f * d[3]);
__syncthreads(); // (B) D complete
// o = exp(gamma) Q S0 + P D (P is zero above its diagonal), the output
float o[4] = {e * qs[0], e * qs[1], e * qs[2], e * qs[3]};
#pragma unroll
for (int i = 0; i < GCH; ++i) {
const float4 d4 = *reinterpret_cast<const float4*>(&sd[i * GDV + vv]);
o[0] = fmaf(pr[i], d4.x, o[0]); o[1] = fmaf(pr[i], d4.y, o[1]);
o[2] = fmaf(pr[i], d4.z, o[2]); o[3] = fmaf(pr[i], d4.w, o[3]);
}
if (tt < n)
*reinterpret_cast<float4*>(oc_out + (c_t0 + tt) * HV * S + vh * S + c0 + vv) =
make_float4(o[0] * out_scale, o[1] * out_scale, o[2] * out_scale, o[3] * out_scale);
// S = exp(gamma_last) S0 + K^T (decayed D)
const float gL = expf(glast);
#pragma unroll
for (int xx = 0; xx < 4; ++xx)
#pragma unroll
for (int y = 0; y < 4; ++y) s[xx][y] *= gL;
#pragma unroll 4
for (int t = 0; t < GCH; ++t) {
const float4 k4 = *reinterpret_cast<const float4*>(&sk[t * GKP + 4 * rq]);
const float4 d4 = *reinterpret_cast<const float4*>(&sdd[t * GDV + vv]);
const float kx[4] = {k4.x, k4.y, k4.z, k4.w}, dy[4] = {d4.x, d4.y, d4.z, d4.w};
#pragma unroll
for (int xx = 0; xx < 4; ++xx)
#pragma unroll
for (int y = 0; y < 4; ++y) s[xx][y] = fmaf(kx[xx], dy[y], s[xx][y]);
}
#pragma unroll
for (int xx = 0; xx < 4; ++xx)
*reinterpret_cast<float4*>(&ss[(4 * rq + xx) * GDV + vv]) = make_float4(s[xx][0], s[xx][1], s[xx][2], s[xx][3]);
v4 = nv4; gam = ngam; bet = nbet; glast = nglast;
gdn_cp_wait_all();
__syncthreads(); // (C) the new ss, the next k and q landed; every read of sd, sdd and this k buffer done
}
#pragma unroll
for (int xx = 0; xx < 4; ++xx)
#pragma unroll
for (int y = 0; y < 4; ++y) state[(size_t) (4 * rq + xx) * rs + (size_t) vh * S + c0 + vv + y] = s[xx][y];
}
// The scratch, one a device: allocated on first use at its largest (GSB / GCH chunks of T, P and gamma: 25.6 MB) and
// kept (a device runs its prompt on one stream). A stream-ordered allocation and free per call, even from a pool that
// keeps its memory, cost host time the GPU waited for (a 2K prompt's recurrence took 78 ms against 34 ms for
// gdn_rec_kh_kernel).
float* gdn_chunk_scratch() {
static float* bufs[64] = {};
int dev = 0;
if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return nullptr; }
if (bufs[dev] == nullptr) {
const size_t n = (size_t) (GSB / GCH) * HV * (2 * GCH * GCH + GCH);
if (cudaMalloc((void**) &bufs[dev], n * sizeof(float)) != cudaSuccess) {
cudaGetLastError();
bufs[dev] = nullptr;
}
}
return bufs[dev];
}
// The whole recurrence of T tokens in super-blocks of GSB: a prep and a scan launch each, the scratch (T, P and gamma
// of a super-block's chunks: up to GSB / GCH x 48 heads x (2 x 32 x 32 + 32) floats, 25.6 MB) from gdn_chunk_scratch.
// y: the [T][HV][S] output. An error return means nothing was launched (the caller takes another kernel): the scratch
// could not be had, or the card has fewer than 128 SMs (a second wave of the scan would double it) or too little
// shared memory for it (Turing).
cudaError_t gdn_rec_chunked(float* state, const float* h, const float* gate, const float* beta, float* y, int64_t T,
cudaStream_t s) {
if (T <= 0) return cudaSuccess;
{ // sm_80+ (cp.async), the scan's 128 blocks one an SM in one wave, and its 93.7 KB of shared memory (else:
// another kernel)
int dev = 0, major = 0, sms = 0, smem = 0;
if (cudaGetDevice(&dev) != cudaSuccess ||
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess ||
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess ||
cudaDeviceGetAttribute(&smem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev) != cudaSuccess) {
cudaGetLastError();
return cudaErrorNotSupported;
}
// as in kernels.cu: STRATA_GDN_CHUNKED=2 skips the SM-count test (to measure a card under 128 SMs)
static const bool any_sms = [] { const char* v = std::getenv("STRATA_GDN_CHUNKED"); return v != nullptr && std::atoi(v) >= 2; }();
if (major < 8 || (!any_sms && sms < HK * (S / GDV)) || (size_t) smem < kChunkScanSmem)
return cudaErrorNotSupported;
}
float* scratch = gdn_chunk_scratch();
if (scratch == nullptr) return cudaErrorMemoryAllocation;
const int nb = (int) ((std::min<int64_t>(GSB, T) + GCH - 1) / GCH);
const size_t tsz = (size_t) nb * HV * GCH * GCH;
float *tm = scratch, *pm = tm + tsz, *gm = pm + tsz;
{ // once a device
static bool attrs[64] = {};
int dev = 0;
cudaGetDevice(&dev);
if (dev >= 0 && dev < 64 && !attrs[dev]) {
cudaFuncSetAttribute(gdn_chunk_prep_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) kChunkPrepSmem);
cudaFuncSetAttribute(gdn_chunk_scan_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) kChunkScanSmem);
attrs[dev] = true;
}
}
for (int64_t t0 = 0; t0 < T; t0 += GSB) {
const int nch = (int) ((std::min<int64_t>(GSB, T - t0) + GCH - 1) / GCH);
gdn_chunk_prep_kernel<<<dim3((unsigned) nch, HK), 256, kChunkPrepSmem, s>>>(h, gate, beta, tm, pm, gm, t0, T);
gdn_chunk_scan_kernel<<<dim3(HK, S / GDV), 128 * VPK, kChunkScanSmem, s>>>(state, h, beta, tm, pm, gm, y, t0, nch, T);
}
return cudaSuccess; // launch errors surface at the caller's check; an error above means nothing ran
}
// src/prefill/kernels.cu's tensor-core scan (STRATA_GDN_TC=1): the same text, included there too
#include "gdn_tc.cuh"
// src/prefill/kernels.cu's gdn_rec_tc, copied as it is there (with the raw device attributes, as gdn_rec_chunked above)
cudaError_t gdn_rec_tc(float* state, const float* h, const float* gate, const float* beta, float* y, int64_t T,
cudaStream_t s) {
if (T <= 0) return cudaSuccess;
{
int dev = 0, major = 0, sms = 0, smem = 0;
if (cudaGetDevice(&dev) != cudaSuccess ||
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) != cudaSuccess ||
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess ||
cudaDeviceGetAttribute(&smem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev) != cudaSuccess) {
cudaGetLastError();
return cudaErrorNotSupported;
}
if (major < 8 || sms < HV || (size_t) smem < kTcScanSmem<false>) return cudaErrorNotSupported;
}
const GtcScratch sc = gdn_tc_scratch();
if (sc.ti == nullptr) return cudaErrorMemoryAllocation;
{ // once a device
static bool attrs[64] = {};
int dev = 0;
cudaGetDevice(&dev);
if (dev >= 0 && dev < 64 && !attrs[dev]) {
cudaFuncSetAttribute(gdn_tc_scan_kernel<false>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) kTcScanSmem<false>);
attrs[dev] = true;
}
}
for (int64_t t0 = 0; t0 < T; t0 += GSB) {
const int nch = (int) ((std::min<int64_t>(GSB, T - t0) + GCH - 1) / GCH);
gdn_tc_prep_kernel<<<dim3((unsigned) nch, HK), 256, kTcPrepSmem, s>>>(h, gate, beta, sc.ti, sc.te, sc.sc, sc.kq, sc.pi, t0, T);
gdn_tc_scan_kernel<false><<<HV, GTC_CW * 32, kTcScanSmem<false>, s>>>(state, h, sc.ti, sc.te, sc.sc, sc.kq, sc.pi, y, t0, nch, T);
}
return cudaSuccess;
}
// the output norm over a head's 128 columns, into the FP16 copy the out projection reads (the FP32 output before the
// norm stays in its scratch buffer)
__global__ void __launch_bounds__(S) gdn_out_norm_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
float eps, const float* __restrict__ y,
uint16_t* __restrict__ y16) {
__shared__ float wsum[4];
const int64_t t = blockIdx.x;
const int head = blockIdx.y, col = threadIdx.x;
const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
const float oc = y[at];
float sp = warp_sum(oc * oc);
if ((col & 31) == 0) wsum[col >> 5] = sp;
__syncthreads();
const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
y16[at] = hf(v);
}
// ------------------------------------------------------------------ the output norm before (0.1.31)
__global__ void __launch_bounds__(S) gdn_out_norm_old_kernel(const float* __restrict__ z, const float* __restrict__ gamma,
float eps, float* __restrict__ y,
uint16_t* __restrict__ y16) {
__shared__ float wsum[4];
const int64_t t = blockIdx.x;
const int head = blockIdx.y, col = threadIdx.x;
const size_t at = (size_t) t * HV * S + (size_t) head * S + col;
const float oc = y[at];
float sp = warp_sum(oc * oc);
if ((col & 31) == 0) wsum[col >> 5] = sp;
__syncthreads();
const float ss = wsum[0] + wsum[1] + wsum[2] + wsum[3];
const float v = oc * rsqrtf(ss / (float) S + eps) * gamma[col] * sigm(z[t * HV * S + head * S + col]);
y[at] = v;
y16[at] = hf(v);
}
// ------------------------------------------------------------------ synthetic inputs
__device__ __forceinline__ uint32_t mix(uint64_t x) {
x ^= x >> 33; x *= 0xff51afd7ed558ccdULL; x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53ULL; x ^= x >> 33;
return (uint32_t) x;
}
__device__ __forceinline__ float uni(uint64_t key) { return ((mix(key) >> 8) + 0.5f) * (1.0f / 16777216.0f); }
__device__ __forceinline__ float gauss(uint64_t key) {
const float u1 = uni(key * 2 + 1), u2 = uni(key * 2 + 2);
return sqrtf(-2.0f * logf(u1)) * cosf(6.2831853f * u2);
}
__global__ void fill_gauss(float* x, int64_t n, float scale, uint64_t seed, float offset = 0.0f) {
for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
x[i] = offset + scale * gauss(seed * 0x9E3779B97F4A7C15ULL + (uint64_t) i);
}
// the q and k rows L2-normalized per key head, as gdn_l2_kernel leaves them for the recurrence
__global__ void l2_rows(float* h, float eps) {
float* x = h + (int64_t) blockIdx.y * C + (int64_t) blockIdx.x * S;
const float v = x[threadIdx.x];
__shared__ float part[4];
const float sq = warp_sum(v * v);
if ((threadIdx.x & 31) == 0) part[threadIdx.x >> 5] = sq;
__syncthreads();
x[threadIdx.x] = v * rsqrtf(part[0] + part[1] + part[2] + part[3] + eps);
}
// gate = log of the decay (mostly near 1, some strong; slow: -0.001 u, so the state grows for long), beta in (0, 1)
__global__ void fill_gates(float* gate, float* beta, int64_t n, uint64_t seed, bool slow = false) {
for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) {
const float u = uni(seed * 0x9E3779B97F4A7C15ULL + 7 * (uint64_t) i);
gate[i] = slow ? -0.001f * u : -0.02f - 0.8f * u * u * u;
beta[i] = 1.0f / (1.0f + expf(-gauss(seed * 0xD1B54A32D192ED03ULL + (uint64_t) i)));
}
}
// coherent keys: every token's k row = a key head's k_0 + 0.05 x the row that was there (before l2_rows): the
// chunk's T = (I + A)^-1 then carries large entries, the reason the prep scales its T images by a power of two of their own
__global__ void coherent_k(float* h, uint64_t seed) {
float* x = h + (int64_t) blockIdx.y * C + HK * S + (int64_t) blockIdx.x * S;
x[threadIdx.x] = gauss(seed * 0x9E3779B97F4A7C15ULL + (uint64_t) (blockIdx.x * S + threadIdx.x)) + 0.05f * x[threadIdx.x];
}
__global__ void scale_v(float* h, int64_t T, float scale) {
for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < T * HV * S; i += (int64_t) gridDim.x * blockDim.x)
h[(i / (HV * S)) * C + 2 * HK * S + i % (HV * S)] *= scale;
}
__global__ void count_diff(const uint32_t* a, const uint32_t* b, int64_t n, unsigned long long* out) {
unsigned long long d = 0;
for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
d += a[i] != b[i];
if (d) atomicAdd(out, d);
}
// ------------------------------------------------------------------ holding SMs (fewer_sms)
__device__ __forceinline__ uint64_t gtime() {
uint64_t t;
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
return t;
}
__device__ __forceinline__ int sm_id() {
unsigned id;
asm volatile("mov.u32 %0, %%smid;" : "=r"(id));
return (int) id;
}
__device__ __forceinline__ void nap(unsigned ns) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
__nanosleep(ns);
#else
(void) ns;
#endif
}
// limits that a run that works never reaches (a launch here takes milliseconds); they keep a run that does not work
// from waiting forever, and the host drops its time
constexpr uint64_t kHoldLimitNs = 1000000000ull, kWaitLimitNs = 500000000ull;
// One block per SM: it asks for all the shared memory a block may have (launched with that much dynamic shared
// memory), so no block of another kernel fits beside it. It notes its SM and sleeps until release says gen.
__global__ void hold_sm(unsigned long long* sitting, const volatile unsigned* release, unsigned gen, int* where,
unsigned* late) {
where[blockIdx.x] = sm_id();
__threadfence();
atomicAdd(sitting, 1ull);
const uint64_t t0 = gtime();
while (*release != gen) {
if (gtime() - t0 > kHoldLimitNs) { atomicAdd(late, 1u); return; }
nap(20000);
}
}
// on the timed launch's stream, before it: returns once all holding blocks so far sit on their SMs
__global__ void wait_sitting(const volatile unsigned long long* sitting, unsigned long long all, unsigned* late) {
const uint64_t t0 = gtime();
while (*sitting < all) {
if (gtime() - t0 > kWaitLimitNs) { atomicAdd(late, 1u); return; }
nap(1000);
}
}
__global__ void release_sms(volatile unsigned* release, unsigned gen) { *release = gen; }
// Where the blocks of a grid land: launched like the recurrence it stands for (the same grid and block size, and as
// much dynamic shared memory as lets as many blocks share an SM), every block notes its SM and stays until all of them
// have landed, so they all sit at once as the recurrence's blocks do for the whole chunk.
__global__ void land(int* where, unsigned* landed, unsigned* apart) {
extern __shared__ unsigned char land_smem[]; // only its size matters
if (threadIdx.x != 0 || threadIdx.y != 0) return;
where[blockIdx.x] = sm_id();
atomicAdd(landed, 1u);
const uint64_t t0 = gtime();
while (*(volatile unsigned*) landed < gridDim.x) {
if (gtime() - t0 > kWaitLimitNs) { atomicAdd(apart, 1u); return; } // they did not all fit at once
nap(1000);
}
(void) land_smem;
}
template <typename T> T* dalloc(size_t n) {
T* p = nullptr;
ck(cudaMalloc((void**) &p, n * sizeof(T) + 16), "cudaMalloc");
return p;
}
unsigned long long diff_words(const void* a, const void* b, size_t bytes) {
static unsigned long long* d = dalloc<unsigned long long>(1);
ck(cudaMemset(d, 0, sizeof(*d)), "memset");
count_diff<<<1024, 256>>>((const uint32_t*) a, (const uint32_t*) b, (int64_t) (bytes / 4), d);
unsigned long long h = 0;
ck(cudaMemcpy(&h, d, sizeof(h), cudaMemcpyDeviceToHost), "diff");
return h;
}
// The model's shapes with gaussian values; the defaults are the bit checks' inputs (the state as after an earlier
// chunk, scale 0.05). The others stress the tensor-core scan's f16 range handling: the state 100x smaller, 50x larger
// or zero (a fresh conversation), v 50x larger, slow gates, coherent keys.
struct Inputs {
int64_t T = 0;
float *h = nullptr, *gate = nullptr, *beta = nullptr, *z = nullptr, *gamma = nullptr, *state0 = nullptr;
explicit Inputs(int64_t t, uint64_t seed, float state_scale = 0.05f, float v_scale = 1.0f, bool slow_gates = false,
bool coherent_keys = false)
: T(t) {
h = dalloc<float>((size_t) T * C);
gate = dalloc<float>((size_t) T * HV);
beta = dalloc<float>((size_t) T * HV);
z = dalloc<float>((size_t) T * HV * S);
gamma = dalloc<float>(S);
state0 = dalloc<float>((size_t) S * HV * S);
fill_gauss<<<4096, 256>>>(h, T * C, 1.0f, seed);
if (coherent_keys) coherent_k<<<dim3(HK, (unsigned) T), S>>>(h, seed + 5);
l2_rows<<<dim3(2 * HK, (unsigned) T), S>>>(h, 1e-6f);
if (v_scale != 1.0f) scale_v<<<4096, 256>>>(h, T, v_scale);
fill_gates<<<1024, 256>>>(gate, beta, T * HV, seed + 1, slow_gates);
fill_gauss<<<4096, 256>>>(z, T * HV * S, 1.0f, seed + 2);
fill_gauss<<<1, 128>>>(gamma, S, 0.1f, seed + 3, 1.0f);
fill_gauss<<<1024, 256>>>(state0, (int64_t) S * HV * S, state_scale, seed + 4);
ck(cudaDeviceSynchronize(), "inputs");
}
Inputs(const Inputs&) = delete;
Inputs& operator=(const Inputs&) = delete;
~Inputs() { cudaFree(h); cudaFree(gate); cudaFree(beta); cudaFree(z); cudaFree(gamma); cudaFree(state0); }
};
enum Rec { kBefore, kKeyHead, kRecCount };
void run_rec(Rec v, float* state, const Inputs& in, float* oc, cudaStream_t s = 0) {
if (v == kBefore)
gdn_rec_cols_pipe_kernel<<<HV * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
else
gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG), 0, s>>>(state, in.h, in.gate, in.beta, oc, in.T);
}
int check_bits(int64_t T, uint64_t seed) {
Inputs in(T, seed);
const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
float* state[kRecCount];
float* oc[kRecCount];
for (int v = 0; v < kRecCount; ++v) {
state[v] = dalloc<float>(st);
oc[v] = dalloc<float>(oc_n);
ck(cudaMemcpy(state[v], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
run_rec((Rec) v, state[v], in, oc[v]);
}
ck(cudaDeviceSynchronize(), "recurrence");
int fails = 0;
const unsigned long long d_oc = diff_words(oc[kBefore], oc[kKeyHead], oc_n * 4),
d_st = diff_words(state[kBefore], state[kKeyHead], st * 4);
if (d_oc || d_st) {
std::printf("FAIL T=%lld recurrence: %llu of %zu outputs and %llu of %zu state values differ\n", (long long) T,
d_oc, oc_n, d_st, st);
++fails;
}
// the output norm: before (in place, with the FP32 store) against now (the FP16 store only), on the same output
uint16_t* y16a = dalloc<uint16_t>(oc_n);
uint16_t* y16b = dalloc<uint16_t>(oc_n);
float* yb = dalloc<float>(oc_n);
ck(cudaMemcpy(yb, oc[kBefore], oc_n * 4, cudaMemcpyDeviceToDevice), "y copy");
gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, yb, y16b);
gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc[kBefore], y16a);
ck(cudaDeviceSynchronize(), "norm");
const unsigned long long d16 = diff_words(y16a, y16b, oc_n * 2);
if (d16) {
std::printf("FAIL T=%lld output norm: %llu of %zu FP16 pairs differ\n", (long long) T, d16, oc_n / 2);
++fails;
}
std::printf("T=%-6lld %s\n", (long long) T, fails ? "DIFFERENT" : "the same bits (output, state, FP16 output)");
for (int v = 0; v < kRecCount; ++v) { cudaFree(state[v]); cudaFree(oc[v]); }
cudaFree(y16a); cudaFree(y16b); cudaFree(yb);
return fails;
}
// The chunked recurrence against gdn_rec_kh_kernel (other bits: the largest |difference| over the largest |value|, the
// output and the state after the chunk); a failure above 1e-4 (FP32 sums in another order stay near 1e-6).
double max_rel(const float* a, const float* b, size_t n) {
std::vector<float> x(n), y(n);
ck(cudaMemcpy(x.data(), a, n * 4, cudaMemcpyDeviceToHost), "copy");
ck(cudaMemcpy(y.data(), b, n * 4, cudaMemcpyDeviceToHost), "copy");
double md = 0.0, mv = 0.0;
for (size_t i = 0; i < n; ++i) {
md = std::max(md, (double) std::fabs(x[i] - y[i]));
mv = std::max(mv, (double) std::fabs(x[i]));
}
return mv > 0.0 ? md / mv : md;
}
// The recurrence in FP64 (a thread per value head and column, the state in local memory): the reference both FP32
// recurrences are measured against
__global__ void gdn_rec_f64_kernel(double* __restrict__ state, const float* __restrict__ h, const float* __restrict__ gate,
const float* __restrict__ beta, double* __restrict__ out, int64_t T) {
const int head = blockIdx.x, col = threadIdx.x, qh = head % HK;
double s[S];
for (int r = 0; r < S; ++r) s[r] = state[((size_t) r * HV + head) * S + col];
for (int64_t t = 0; t < T; ++t) {
const float* ht = h + t * C;
const double g = exp((double) gate[t * HV + head]), b = beta[t * HV + head];
double kv = 0.0;
for (int r = 0; r < S; ++r) kv += s[r] * ht[HK * S + qh * S + r];
const double delta = (ht[2 * HK * S + head * S + col] - g * kv) * b;
double o = 0.0;
for (int r = 0; r < S; ++r) {
s[r] = g * s[r] + ht[HK * S + qh * S + r] * delta;
o += s[r] * ht[qh * S + r];
}
out[(t * HV + head) * S + col] = o / sqrt((double) S);
}
for (int r = 0; r < S; ++r) state[((size_t) r * HV + head) * S + col] = s[r];
}
__global__ void to_f64(const float* x, double* y, int64_t n) {
for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x) y[i] = x[i];
}
double max_rel_f64(const float* a, const double* ref, size_t n) { // max |a - ref| / max |ref|
std::vector<float> x(n);
std::vector<double> y(n);
ck(cudaMemcpy(x.data(), a, n * 4, cudaMemcpyDeviceToHost), "copy");
ck(cudaMemcpy(y.data(), ref, n * 8, cudaMemcpyDeviceToHost), "copy");
double md = 0.0, mv = 0.0;
for (size_t i = 0; i < n; ++i) {
md = std::max(md, std::fabs(x[i] - y[i]));
mv = std::max(mv, std::fabs(y[i]));
}
return mv > 0.0 ? md / mv : md;
}
// a recurrence of gdn_rec_chunked's shape (the FP32 chunked pair, the tensor-core scan) on the inputs given
using RecLaunch = cudaError_t (*)(float*, const float*, const float*, const float*, float*, int64_t, cudaStream_t);
int check_variant(const Inputs& in, const char* label, RecLaunch launch, const char* needs) {
const int64_t T = in.T;
const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
float* s0 = dalloc<float>(st);
float* s1 = dalloc<float>(st);
float* o0 = dalloc<float>(oc_n);
float* o1 = dalloc<float>(oc_n);
ck(cudaMemcpy(s0, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
ck(cudaMemcpy(s1, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state copy");
gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG)>>>(s0, in.h, in.gate, in.beta, o0, in.T);
const cudaError_t e = launch(s1, in.h, in.gate, in.beta, o1, in.T, 0);
if (e == cudaErrorNotSupported) {
std::printf(" T=%-6lld %s: not on this card (%s)\n", (long long) T, label, needs);
cudaFree(s0); cudaFree(s1); cudaFree(o0); cudaFree(o1);
return 0;
}
ck(e, label);
ck(cudaDeviceSynchronize(), label);
const double eo = max_rel(o0, o1, oc_n), es = max_rel(s0, s1, st);
const bool bad = !(eo < 1e-4) || !(es < 1e-4);
std::printf("%s T=%-6lld %s: output %.2e, state %.2e (max |diff| / max |value|)\n", bad ? "FAIL" : " ",
(long long) T, label, eo, es);
if (T <= 8192) { // both against FP64 (its output: 8 bytes a value, so not at 32K)
double* sd = dalloc<double>(st);
double* od = dalloc<double>(oc_n);
to_f64<<<1024, 256>>>(in.state0, sd, (int64_t) st);
gdn_rec_f64_kernel<<<HV, S>>>(sd, in.h, in.gate, in.beta, od, in.T);
ck(cudaDeviceSynchronize(), "FP64 run");
std::printf(" against FP64: gdn_rec_kh_kernel output %.2e, state %.2e; this one output %.2e, state %.2e\n",
max_rel_f64(o0, od, oc_n), max_rel_f64(s0, sd, st), max_rel_f64(o1, od, oc_n), max_rel_f64(s1, sd, st));
cudaFree(sd); cudaFree(od);
}
cudaFree(s0); cudaFree(s1); cudaFree(o0); cudaFree(o1);
return bad ? 1 : 0;
}
int check_chunked(int64_t T, uint64_t seed) {
Inputs in(T, seed);
return check_variant(in, "chunked (STRATA_GDN_CHUNKED)", gdn_rec_chunked, "sm_80+, 128 SMs, 93.7 KB a block");
}
constexpr const char* kTcNeeds = "sm_80+, 48 SMs, 122 KB a block";
int check_tc(int64_t T, uint64_t seed) {
Inputs in(T, seed);
return check_variant(in, "tc (STRATA_GDN_TC=1)", gdn_rec_tc, kTcNeeds);
}
// the tensor-core scan on the stress inputs (Inputs): its f16 range handling has a case for each
int check_tc_stress(int64_t T, uint64_t seed) {
struct Case { const char* label; float state_scale, v_scale; bool slow_gates, coherent_keys; };
const Case cases[] = {{"tc (STRATA_GDN_TC=1), state0 x 0.01", 5e-4f, 1.0f, false, false},
{"tc (STRATA_GDN_TC=1), state0 x 50", 2.5f, 1.0f, false, false},
{"tc (STRATA_GDN_TC=1), state0 = 0", 0.0f, 1.0f, false, false},
{"tc (STRATA_GDN_TC=1), v x 50", 0.05f, 50.0f, false, false},
{"tc (STRATA_GDN_TC=1), slow gates", 0.05f, 1.0f, true, false},
{"tc (STRATA_GDN_TC=1), coherent keys", 0.05f, 1.0f, false, true}};
int fails = 0;
for (const Case& c : cases) {
Inputs in(T, seed++, c.state_scale, c.v_scale, c.slow_gates, c.coherent_keys);
fails += check_variant(in, c.label, gdn_rec_tc, kTcNeeds);
}
return fails;
}
// ------------------------------------------------------------------ the tensor-core scan's fragment self-test (--selftest-frag)
// One warp each, random inputs, the reference in double on the host: (b) a 16x128 . 128x32 product through
// gtc_prod_state (the f16 split with the state's 2^es and k's 2^8, ldmatrix), (c) a 16x32 . 32x128 update through
// gtc_update_state (ldmatrix.trans, the 2^ed scale, the accumulator's pre / post scale), (d) a 16x32 . 32x32 product
// through gtc_prod_dt on a full random T of wide range (the prep's 2^et on the B images), (e) the triangle-skipped
// tile sets of gtc_prod_dt against the full ones on triangular T / P: the same bits, since the skipped tiles add
// exact zeros. Most bugs of a tensor-core kernel are fragment-layout bugs: these catch them before the kernel does.
// ((a), the C -> A tf32 shuffle, went with the tf32 form of D^T.)
__device__ void st_stage_k(uint16_t* kh, uint16_t* kl, const float* k) { // k: 32 x 128 f32 -> the hi / lo images x 2^8
for (int p = threadIdx.x; p < GCH * S / 4; p += 32) {
const int t = p / (S / 4), r = (p % (S / 4)) * 4;
gtc_split_store4(kh + t * GTC_KP + r, kl + t * GTC_KP + r, *reinterpret_cast<const float4*>(k + t * S + r), 256.0f);
}
__syncwarp();
}
// a 16-row matrix (row stride ld) as 8-column tiles in the C layout: tile j, lane (gid, tig) holds (gid, 8j + 2tig),
// (gid, 8j + 2tig + 1), (gid + 8, 8j + 2tig), (gid + 8, 8j + 2tig + 1)
template <int NT> __device__ void st_load_c(float (&c)[NT][4], const float* m, int ld) {
const int lane = threadIdx.x & 31, gid = lane >> 2, tig = lane & 3;
#pragma unroll
for (int j = 0; j < NT; ++j) {
c[j][0] = m[gid * ld + 8 * j + 2 * tig];
c[j][1] = m[gid * ld + 8 * j + 2 * tig + 1];
c[j][2] = m[(gid + 8) * ld + 8 * j + 2 * tig];
c[j][3] = m[(gid + 8) * ld + 8 * j + 2 * tig + 1];
}
}
template <int NT> __device__ void st_store_c(const float (&c)[NT][4], float* m, int ld, float scale) {
const int lane = threadIdx.x & 31, gid = lane >> 2, tig = lane & 3;
#pragma unroll
for (int j = 0; j < NT; ++j) {
m[gid * ld + 8 * j + 2 * tig] = c[j][0] * scale;
m[gid * ld + 8 * j + 2 * tig + 1] = c[j][1] * scale;
m[(gid + 8) * ld + 8 * j + 2 * tig] = c[j][2] * scale;
m[(gid + 8) * ld + 8 * j + 2 * tig + 1] = c[j][3] * scale;
}
}
template <int NT> __device__ float st_max_abs(const float (&c)[NT][4]) {
float m = 0.0f;
#pragma unroll
for (int j = 0; j < NT; ++j)
#pragma unroll
for (int q = 0; q < 4; ++q) m = fmaxf(m, fabsf(c[j][q]));
return gtc_warp_max(m);
}
__global__ void st_state_kernel(const float* a, const float* k, float* out) { // out (16 x 32) = a (16 x 128) k^T
__shared__ __align__(16) uint16_t kh[GCH * GTC_KP], kl[GCH * GTC_KP];
st_stage_k(kh, kl, k);
float s[16][4];
st_load_c<16>(s, a, S);
const int es = gtc_scale_exp(st_max_abs<16>(s));
float xt[4][4] = {}, qs[4][4] = {};
gtc_prod_state(s, gtc_pow2(es), kh, kl, kh, kl, xt, qs);
st_store_c<4>(xt, out, GCH, gtc_pow2(-(8 + es)));
}
__global__ void st_update_kernel(const float* s0, const float* dd, const float* k, float eL, float* out) { // out (16 x 128) = eL s0 + dd (16 x 32) k
__shared__ __align__(16) uint16_t kh[GCH * GTC_KP], kl[GCH * GTC_KP];
st_stage_k(kh, kl, k);
float s[16][4], dt[4][4], ddt[4][4];
st_load_c<16>(s, s0, S);
st_load_c<4>(dt, dd, GCH);
const int ed = gtc_scale_exp(st_max_abs<4>(dt));
const float sD = gtc_pow2(ed), sPre = eL * gtc_pow2(ed + 8);
#pragma unroll
for (int j = 0; j < 4; ++j)
#pragma unroll
for (int q = 0; q < 4; ++q) ddt[j][q] = dt[j][q] * sD;
#pragma unroll
for (int j = 0; j < 16; ++j)
#pragma unroll
for (int q = 0; q < 4; ++q) s[j][q] *= sPre;
gtc_update_state(s, ddt, kh, kl);
st_store_c<16>(s, out, S, gtc_pow2(-(ed + 8)));
}
// out (16 x 32) = d p^T: p's images x 2^eb (the scan's P carry 2^8, its T the prep's 2^et), d x 2^ed from its maximum
template <bool SKIP> __global__ void st_dt_kernel(const float* d, const float* p, int eb, float* out) {
__shared__ __align__(16) uint16_t ph[GCH * GTC_PP], pl[GCH * GTC_PP];
for (int q = threadIdx.x; q < GCH * GCH / 4; q += 32) {
const int t = q / (GCH / 4), i = (q % (GCH / 4)) * 4;
gtc_split_store4(ph + t * GTC_PP + i, pl + t * GTC_PP + i, *reinterpret_cast<const float4*>(p + t * GCH + i), gtc_pow2(eb));
}
__syncwarp();
float dt[4][4], ot[4][4] = {};
st_load_c<4>(dt, d, GCH);
const int ed = gtc_scale_exp(st_max_abs<4>(dt));
gtc_prod_dt<SKIP>(dt, gtc_pow2(ed), ph, pl, ot);
st_store_c<4>(ot, out, GCH, gtc_pow2(-(ed + eb)));
}
int st_scale_exp(const std::vector<float>& m) { // the prep's gtc_scale_exp on the host: max |m| into [2^14, 2^15)
float mx = 0.0f;
for (float x : m) mx = std::max(mx, std::fabs(x));
if (mx == 0.0f) return 0;
int e = 0;
std::frexp(mx, &e); // mx = f 2^e, f in [0.5, 1): floor(log2 mx) = e - 1
e = 14 - (e - 1);
return e < -40 ? -40 : (e > 40 ? 40 : e);
}
struct StRng { // xorshift64, host side
uint64_t s;
explicit StRng(uint64_t seed) : s(seed) {}
float uni() { s ^= s << 13; s ^= s >> 7; s ^= s << 17; return ((float) (s >> 40) + 0.5f) / 16777216.0f; }
float gauss() { return std::sqrt(-2.0f * std::log(uni())) * std::cos(6.2831853f * uni()); }
float logmag(float lo, float hi) { return (uni() < 0.5f ? -1.0f : 1.0f) * std::pow(10.0f, lo + (hi - lo) * uni()); }
};
float* st_upload(const std::vector<float>& v) {
float* d = dalloc<float>(v.size());
ck(cudaMemcpy(d, v.data(), v.size() * 4, cudaMemcpyHostToDevice), "upload");
return d;
}
std::vector<float> st_download(const float* d, size_t n) {
std::vector<float> v(n);
ck(cudaDeviceSynchronize(), "self-test kernel");
ck(cudaMemcpy(v.data(), d, n * 4, cudaMemcpyDeviceToHost), "download");
return v;
}
double st_max_rel(const std::vector<float>& got, const std::vector<double>& ref) {
double md = 0.0, mv = 0.0;
for (size_t i = 0; i < ref.size(); ++i) {
md = std::max(md, std::fabs((double) got[i] - ref[i]));
mv = std::max(mv, std::fabs(ref[i]));
}
return mv > 0.0 ? md / mv : md;
}
int selftest_frag() {
std::printf("--selftest-frag: the tensor-core scan's fragment helpers, one warp each\n");
int fails = 0;
StRng rng(0x5e1f7e57ULL);
auto unit_rows = [&](std::vector<float>& k, int rows, int cols) { // gaussian rows, L2-normalized
for (int t = 0; t < rows; ++t) {
double ss = 0.0;
for (int r = 0; r < cols; ++r) { k[t * cols + r] = rng.gauss(); ss += (double) k[t * cols + r] * k[t * cols + r]; }
for (int r = 0; r < cols; ++r) k[t * cols + r] = (float) (k[t * cols + r] / std::sqrt(ss));
}
};
{ // (b) X^T = S^T K^T: f16x3 with the state's 2^es and k's 2^8, B by ldmatrix
std::vector<float> a(16 * S), k(GCH * S);
for (float& x : a) x = rng.logmag(-3.0f, 2.0f);
unit_rows(k, GCH, S);
std::vector<double> ref(16 * GCH);
for (int c = 0; c < 16; ++c)
for (int i = 0; i < GCH; ++i) {
double v = 0.0;
for (int r = 0; r < S; ++r) v += (double) a[c * S + r] * k[i * S + r];
ref[c * GCH + i] = v;
}
float* da = st_upload(a);
float* dk = st_upload(k);
float* dout = dalloc<float>(16 * GCH);
st_state_kernel<<<1, 32>>>(da, dk, dout);
const double rel = st_max_rel(st_download(dout, 16 * GCH), ref);
const bool bad = !(rel <= 2e-6);
std::printf(" (b) X^T = S^T K^T (16x128 . 128x32, f16x3, 2^es 2^8, ldmatrix): max |diff| / max |ref| %.2e %s\n", rel, bad ? "FAIL" : "ok");
fails += bad;
cudaFree(da); cudaFree(dk); cudaFree(dout);
}
{ // (c) S^T = eL S^T + Dd^T K: ldmatrix.trans, 2^ed, the accumulator scaled before and after
std::vector<float> s0(16 * S), dd(16 * GCH), k(GCH * S);
for (float& x : s0) x = rng.logmag(-3.0f, 2.0f);
for (float& x : dd) x = rng.logmag(-3.0f, 2.0f);
unit_rows(k, GCH, S);
const float eL = 0.25f + 0.75f * rng.uni();
std::vector<double> ref(16 * S);
for (int c = 0; c < 16; ++c)
for (int r = 0; r < S; ++r) {
double v = (double) eL * s0[c * S + r];
for (int i = 0; i < GCH; ++i) v += (double) dd[c * GCH + i] * k[i * S + r];
ref[c * S + r] = v;
}
float* ds = st_upload(s0);
float* dd_ = st_upload(dd);
float* dk = st_upload(k);
float* dout = dalloc<float>(16 * S);
st_update_kernel<<<1, 32>>>(ds, dd_, dk, eL, dout);
const double rel = st_max_rel(st_download(dout, 16 * S), ref);
const bool bad = !(rel <= 2e-6);
std::printf(" (c) S^T = eL S^T + Dd^T K (16x32 . 32x128, f16x3, ldmatrix.trans, 2^ed pre / post scale): %.2e %s\n", rel, bad ? "FAIL" : "ok");
fails += bad;
cudaFree(ds); cudaFree(dd_); cudaFree(dk); cudaFree(dout);
}
{ // (d) D^T = Y^T T^T on f16x3 through gtc_prod_dt, a full random T of wide range with the prep's 2^et
std::vector<float> y(16 * GCH), t(GCH * GCH);
for (float& x : y) x = rng.logmag(-3.0f, 2.0f);
for (float& x : t) x = rng.logmag(-2.0f, 3.0f);
std::vector<double> ref(16 * GCH);
for (int c = 0; c < 16; ++c)
for (int i = 0; i < GCH; ++i) {
double v = 0.0;
for (int ip = 0; ip < GCH; ++ip) v += (double) y[c * GCH + ip] * t[i * GCH + ip];
ref[c * GCH + i] = v;
}
float* dy = st_upload(y);
float* dt_ = st_upload(t);
float* dout = dalloc<float>(16 * GCH);
st_dt_kernel<false><<<1, 32>>>(dy, dt_, st_scale_exp(t), dout);
const double rel = st_max_rel(st_download(dout, 16 * GCH), ref);
const bool bad = !(rel <= 2e-6);
std::printf(" (d) D^T = Y^T T^T (16x32 . 32x32, f16x3, Y x 2^ey, T x 2^et): %.2e %s\n", rel, bad ? "FAIL" : "ok");
fails += bad;
cudaFree(dy); cudaFree(dt_); cudaFree(dout);
}
{ // (e) the triangle-skipped tile sets against the full ones: T unit lower triangular, P lower triangular
std::vector<float> y(16 * GCH), t(GCH * GCH), d(16 * GCH), p(GCH * GCH);
for (float& x : y) x = rng.logmag(-3.0f, 2.0f);
for (float& x : d) x = rng.logmag(-3.0f, 2.0f);
for (int i = 0; i < GCH; ++i)
for (int j = 0; j < GCH; ++j) {
t[i * GCH + j] = j < i ? rng.logmag(-2.0f, 3.0f) : (j == i ? 1.0f : 0.0f);
p[i * GCH + j] = j <= i ? 2.0f * rng.uni() - 1.0f : 0.0f;
}
std::vector<double> ref_d(16 * GCH), ref_o(16 * GCH);
for (int c = 0; c < 16; ++c)
for (int i = 0; i < GCH; ++i) {
double v = 0.0, w = 0.0;
for (int ip = 0; ip < GCH; ++ip) { v += (double) y[c * GCH + ip] * t[i * GCH + ip]; w += (double) d[c * GCH + ip] * p[i * GCH + ip]; }
ref_d[c * GCH + i] = v;
ref_o[c * GCH + i] = w;
}
float* dy = st_upload(y);
float* dt_ = st_upload(t);
float* dd_ = st_upload(d);
float* dp = st_upload(p);
float* full = dalloc<float>(16 * GCH);
float* skip = dalloc<float>(16 * GCH);
st_dt_kernel<false><<<1, 32>>>(dy, dt_, st_scale_exp(t), full);
st_dt_kernel<true><<<1, 32>>>(dy, dt_, st_scale_exp(t), skip);
const std::vector<float> f1 = st_download(full, 16 * GCH), s1 = st_download(skip, 16 * GCH);
const unsigned long long diff1 = std::memcmp(f1.data(), s1.data(), 16 * GCH * 4) != 0 ? diff_words(full, skip, 16 * GCH * 4) : 0;
const double rel1 = st_max_rel(s1, ref_d);
st_dt_kernel<false><<<1, 32>>>(dd_, dp, 8, full);
st_dt_kernel<true><<<1, 32>>>(dd_, dp, 8, skip);
const std::vector<float> f2 = st_download(full, 16 * GCH), s2 = st_download(skip, 16 * GCH);
const unsigned long long diff2 = std::memcmp(f2.data(), s2.data(), 16 * GCH * 4) != 0 ? diff_words(full, skip, 16 * GCH * 4) : 0;
const double rel2 = st_max_rel(s2, ref_o);
const bool bad = diff1 != 0 || diff2 != 0 || !(rel1 <= 2e-6) || !(rel2 <= 2e-6);
std::printf(" (e) triangle skipping: D^T 10 of 16 tiles %llu words differ (vs FP64 %.2e), o^T 6 of 8 tiles %llu words differ (vs FP64 %.2e) %s\n",
diff1, rel1, diff2, rel2, bad ? "FAIL" : "ok");
fails += bad;
cudaFree(dy); cudaFree(dt_); cudaFree(dd_); cudaFree(dp); cudaFree(full); cudaFree(skip);
}
std::printf("--selftest-frag: %d failures\n", fails);
return fails;
}
float median(std::vector<float> v) {
std::sort(v.begin(), v.end());
return v.empty() ? 0.0f : v[v.size() / 2];
}
// tc_only (--bench-tc): the tensor-core series alone, for A / B runs of two builds back to back
void bench(int64_t T, bool tc_only = false) {
Inputs in(T, 0xB3);
const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
float* state = dalloc<float>(st);
float* oc = dalloc<float>(oc_n);
uint16_t* y16 = dalloc<uint16_t>(oc_n);
cudaEvent_t e0, e1;
ck(cudaEventCreate(&e0), "event");
ck(cudaEventCreate(&e1), "event");
auto timed = [&](auto&& launch) {
ck(cudaMemcpy(state, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
cudaEventRecord(e0);
launch();
cudaEventRecord(e1);
ck(cudaEventSynchronize(e1), "timed");
float ms = 0.0f;
cudaEventElapsedTime(&ms, e0, e1);
return ms;
};
const int warm = 3, rounds = T >= 16384 ? 10 : 20;
std::vector<float> t_rec[kRecCount], t_norm[2];
for (int i = 0; !tc_only && i < warm + rounds; ++i) {
for (int j = 0; j < kRecCount; ++j) {
const Rec v = (Rec) ((i + j) % kRecCount); // a different variant first in each round
const float ms = timed([&] { run_rec(v, state, in, oc); });
if (i >= warm) t_rec[v].push_back(ms);
}
}
for (int i = 0; !tc_only && i < warm + rounds; ++i) {
for (int j = 0; j < 2; ++j) {
const int v = (i + j) % 2;
const float ms = timed([&] {
if (v == 0) gdn_out_norm_old_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
else gdn_out_norm_kernel<<<dim3((unsigned) T, HV), S>>>(in.z, in.gamma, 1e-6f, oc, y16);
});
if (i >= warm) t_norm[v].push_back(ms);
}
}
std::vector<float> t_kc[2]; // gdn_rec_kh_kernel and the chunked recurrence, alternating
const bool can_chunk = !tc_only && gdn_rec_chunked(state, in.h, in.gate, in.beta, oc, in.T, 0) == cudaSuccess &&
cudaDeviceSynchronize() == cudaSuccess;
for (int i = 0; can_chunk && i < warm + rounds; ++i)
for (int j = 0; j < 2; ++j) {
const int v = (i + j) % 2;
const float ms = timed([&] {
if (v == 0) gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG)>>>(state, in.h, in.gate, in.beta, oc, in.T);
else ck(gdn_rec_chunked(state, in.h, in.gate, in.beta, oc, in.T, 0), "chunked");
});
if (i >= warm) t_kc[v].push_back(ms);
}
std::vector<float> t_tc[2]; // gdn_rec_kh_kernel and the tensor-core scan, alternating
const bool can_tc = gdn_rec_tc(state, in.h, in.gate, in.beta, oc, in.T, 0) == cudaSuccess &&
cudaDeviceSynchronize() == cudaSuccess;
for (int i = 0; can_tc && i < warm + rounds; ++i)
for (int j = 0; j < 2; ++j) {
const int v = (i + j) % 2;
const float ms = timed([&] {
if (v == 0) gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG)>>>(state, in.h, in.gate, in.beta, oc, in.T);
else ck(gdn_rec_tc(state, in.h, in.gate, in.beta, oc, in.T, 0), "tc");
});
if (i >= warm) t_tc[v].push_back(ms);
}
const float r0 = median(t_rec[kBefore]), r1 = median(t_rec[kKeyHead]), n0 = median(t_norm[0]), n1 = median(t_norm[1]);
std::printf(" T = %lld tokens, one layer (48 value heads), medians of %d alternating runs:\n", (long long) T, rounds);
if (tc_only) {
const float kh = median(t_tc[0]), tc = median(t_tc[1]);
std::printf(" recurrence, gdn_rec_kh_kernel %8.2f ms (%.3f us per token)\n", kh, 1000.0f * kh / (float) T);
std::printf(" recurrence, tensor-core chunked (STRATA_GDN_TC=1) %8.2f ms (%.3f us per token) %.2fx gdn_rec_kh_kernel\n",
tc, 1000.0f * tc / (float) T, kh / tc);
cudaEventDestroy(e0);
cudaEventDestroy(e1);
cudaFree(state); cudaFree(oc); cudaFree(y16);
return;
}
std::printf(" recurrence, before (gdn_rec_cols_pipe_kernel) %8.2f ms (%.3f us per token)\n", r0,
1000.0f * r0 / (float) T);
std::printf(" recurrence, gdn_rec_kh_kernel %8.2f ms (%.3f us per token) %.2fx\n", r1,
1000.0f * r1 / (float) T, r0 / r1);
if (can_chunk) {
const float kh = median(t_kc[0]), kc = median(t_kc[1]);
std::printf(" recurrence, chunked (STRATA_GDN_CHUNKED=1) %8.2f ms (%.3f us per token) %.2fx gdn_rec_kh_kernel\n",
kc, 1000.0f * kc / (float) T, kh / kc);
}
if (can_tc) {
const float kh = median(t_tc[0]), tc = median(t_tc[1]);
std::printf(" recurrence, tensor-core chunked (STRATA_GDN_TC=1) %8.2f ms (%.3f us per token) %.2fx gdn_rec_kh_kernel\n",
tc, 1000.0f * tc / (float) T, kh / tc);
}
std::printf(" output norm, before (FP32 and FP16 store) %8.2f ms\n", n0);
std::printf(" output norm, FP16 store only %8.2f ms %.2fx\n", n1, n0 / n1);
std::printf(" 36 DeltaNet layers per %lld-token block: recurrence + norm %.0f ms -> %.0f ms (-%.0f ms)\n",
(long long) T, 36.0f * (r0 + n0), 36.0f * (r1 + n1), 36.0f * (r0 + n0 - r1 - n1));
cudaEventDestroy(e0);
cudaEventDestroy(e1);
cudaFree(state); cudaFree(oc); cudaFree(y16);
}
// gdn_rec_kh_kernel against the chunked recurrence on short chunks (the chunked one's launches, host side included:
// the GPU is idle at each start), where kernels.cu's kGdnChunkedMin comes from
void chunked_sweep() {
std::printf(" short chunks, gdn_rec_kh_kernel -> chunked (medians of 21 alternating runs):\n");
for (const int64_t T : {32, 64, 96, 128, 192, 256, 512, 1024}) {
Inputs in(T, 0xC5);
const size_t st = (size_t) S * HV * S;
float* state = dalloc<float>(st);
float* oc = dalloc<float>((size_t) T * HV * S);
cudaEvent_t e0, e1;
ck(cudaEventCreate(&e0), "event");
ck(cudaEventCreate(&e1), "event");
const bool can = gdn_rec_chunked(state, in.h, in.gate, in.beta, oc, in.T, 0) == cudaSuccess &&
cudaDeviceSynchronize() == cudaSuccess;
std::vector<float> t[2];
for (int i = 0; can && i < 5 + 21; ++i)
for (int j = 0; j < 2; ++j) {
const int v = (i + j) % 2;
ck(cudaMemcpy(state, in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
cudaEventRecord(e0);
if (v == 0) gdn_rec_kh_kernel<<<HK * NCB, dim3(CB, RG)>>>(state, in.h, in.gate, in.beta, oc, in.T);
else ck(gdn_rec_chunked(state, in.h, in.gate, in.beta, oc, in.T, 0), "chunked");
cudaEventRecord(e1);
ck(cudaEventSynchronize(e1), "timed");
float ms = 0.0f;
cudaEventElapsedTime(&ms, e0, e1);
if (i >= 5) t[v].push_back(ms);
}
if (can) {
const float kh = median(t[0]), kc = median(t[1]);
std::printf(" T = %-5lld %8.1f us -> %8.1f us %.2fx\n", (long long) T, 1000.0f * kh, 1000.0f * kc, kh / kc);
} else {
std::printf(" chunked: not on this card\n");
}
cudaEventDestroy(e0); cudaEventDestroy(e1);
cudaFree(state); cudaFree(oc);
if (!can) return;
}
}
// The two kernels at a given number of blocks per SM, as on cards with more or fewer SMs than this one: the kernel
// before puts ceil(192 / SMs) of its blocks on an SM, gdn_rec_kh_kernel ceil(64 / SMs), and the busiest SM sets the
// pace (every block walks the whole chunk). b blocks per SM here: b rounds of one block per SM, each round one launch
// of SMs blocks (the first heads; split in launches of at most the engine's grid), every launch on its own stream with
// its own state and output, all started together, so the block scheduler puts one block of each round on every SM.
// The kernels are the ones above, unchanged.
void scale(int64_t T) {
int dev = 0, sms = 0, per_sm[kRecCount] = {};
ck(cudaGetDevice(&dev), "device");
ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
constexpr int kMaxB = 6, kMaxLaunches = 6;
const int full[kRecCount] = {HV * NCB, HK * NCB}; // the grids as the engine launches them
auto parts = [&](int v) { return (sms + full[v] - 1) / full[v]; }; // launches per round
Inputs in(T, 0xC7);
const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
float* state[kMaxLaunches];
float* oc[kMaxLaunches];
cudaStream_t stream[kMaxLaunches];
cudaEvent_t e0, e1, done[kMaxLaunches];
ck(cudaEventCreate(&e0), "event");
ck(cudaEventCreate(&e1), "event");
for (int c = 0; c < kMaxLaunches; ++c) {
state[c] = dalloc<float>(st);
oc[c] = dalloc<float>(oc_n);
ck(cudaStreamCreateWithFlags(&stream[c], cudaStreamNonBlocking), "stream");
ck(cudaEventCreate(&done[c]), "event");
}
struct Case { int v, b; };
std::vector<Case> cases;
for (int v = 0; v < kRecCount; ++v)
for (int b = 1; b <= std::min(kMaxB, per_sm[v]); ++b)
if (b * parts(v) <= kMaxLaunches) cases.push_back({v, b});
auto run = [&](const Case& cs) {
for (int c = 0; c < kMaxLaunches; ++c) ck(cudaMemcpy(state[c], in.state0, st * 4, cudaMemcpyDeviceToDevice), "state");
ck(cudaDeviceSynchronize(), "sync");
const int np = parts(cs.v), n = cs.b * np;
cudaEventRecord(e0, stream[0]);
for (int c = 0; c < n; ++c) {
const int k = c % np, grid = sms / np + (k < sms % np ? 1 : 0); // round c / np, part k
if (c > 0) cudaStreamWaitEvent(stream[c], e0, 0);
if (cs.v == kBefore)
gdn_rec_cols_pipe_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
else
gdn_rec_kh_kernel<<<grid, dim3(CB, RG), 0, stream[c]>>>(state[c], in.h, in.gate, in.beta, oc[c], T);
cudaEventRecord(done[c], stream[c]);
}
for (int c = 1; c < n; ++c) cudaStreamWaitEvent(stream[0], done[c], 0);
cudaEventRecord(e1, stream[0]);
ck(cudaEventSynchronize(e1), "scale");
float ms = 0.0f;
cudaEventElapsedTime(&ms, e0, e1);
return ms;
};
const int warm = 2, rounds = 9;
std::vector<std::vector<float>> t(cases.size());
for (int i = 0; i < warm + rounds; ++i)
for (size_t j = 0; j < cases.size(); ++j) {
const size_t k = (i + j) % cases.size(); // a different case first in each round
const float ms = run(cases[k]);
if (i >= warm) t[k].push_back(ms);
}
auto us = [&](int v, int b) { // per token, or < 0 if not measured
for (size_t k = 0; k < cases.size(); ++k)
if (cases[k].v == v && cases[k].b == b) return 1000.0f * median(t[k]) / (float) T;
return -1.0f;
};
std::printf(" per token at a given number of blocks per SM (%lld tokens, medians of %d alternating runs; this card has "
"%d SMs):\n", (long long) T, rounds, sms);
std::printf(" blocks per SM before gdn_rec_kh_kernel\n");
for (int b = 1; b <= kMaxB; ++b) {
const float u0 = us(kBefore, b), u1 = us(kKeyHead, b);
if (u0 < 0 && u1 < 0) continue;
char a0[32] = "-", a1[32] = "-";
if (u0 >= 0) std::snprintf(a0, sizeof a0, "%.3f us", u0);
if (u1 >= 0) std::snprintf(a1, sizeof a1, "%.3f us", u1);
std::printf(" %8d %-11s %s\n", b, a0, a1);
}
std::printf(" so on a card with N SMs (the kernel before puts ceil(192 / N) blocks on an SM, gdn_rec_kh_kernel "
"ceil(64 / N), at most %d):\n", per_sm[kKeyHead]);
const int bands[][2] = {{192, 0}, {96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
for (const auto& bd : bands) {
const int nsm = bd[0], b0 = (192 + nsm - 1) / nsm, b1 = (64 + nsm - 1) / nsm;
const float u0 = us(kBefore, b0), u1 = us(kKeyHead, b1);
char range[32];
if (bd[1]) std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
else std::snprintf(range, sizeof range, "%d+ SMs", bd[0]);
if (u0 < 0 || u1 < 0)
std::printf(" %-12s before %d / new %d per SM: not measured on this card\n", range, b0, b1);
else
std::printf(" %-12s before %d / new %d per SM: %.3f -> %.3f us per token, %.2fx\n", range, b0, b1, u0, u1,
u0 / u1);
}
for (int c = 0; c < kMaxLaunches; ++c) {
cudaFree(state[c]);
cudaFree(oc[c]);
cudaStreamDestroy(stream[c]);
cudaEventDestroy(done[c]);
}
cudaEventDestroy(e0);
cudaEventDestroy(e1);
}
// The engine's grids on N of this card's SMs. scale() puts as many blocks on every SM as the busiest SM of a card with
// N SMs gets; on that card only some SMs get that many (192 blocks on 80 SMs: 3 on 32 SMs, 2 on the other 48), the
// rest of the card has less to do, and the estimate came out 4-9 % above the real launch on two cards. Here the
// recurrences run as the engine launches them, while hold_sm keeps all but N SMs: its blocks go out first, a small
// kernel on the timed launch's stream waits until they all sit, and one after the timed launch lets them go. land()
// shows for every N where the two grids land: none on a held SM, and how many blocks share the busiest SM. The
// recurrences are the ones above, unchanged; L2, memory and clock stay this card's.
void fewer_sms(int64_t T) {
int dev = 0, sms = 0, optin = 0, smem_sm = 0, reserved = 0, hold_per_sm = 0, per_sm[kRecCount] = {};
ck(cudaGetDevice(&dev), "device");
ck(cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev), "SMs");
ck(cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev), "shared memory");
ck(cudaDeviceGetAttribute(&smem_sm, cudaDevAttrMaxSharedMemoryPerMultiprocessor, dev), "shared memory");
ck(cudaDeviceGetAttribute(&reserved, cudaDevAttrReservedSharedMemoryPerBlock, dev), "shared memory");
for (const void* f : {(const void*) hold_sm, (const void*) land}) {
ck(cudaFuncSetAttribute(f, cudaFuncAttributeMaxDynamicSharedMemorySize, optin), "shared memory");
ck(cudaFuncSetAttribute(f, cudaFuncAttributePreferredSharedMemoryCarveout, (int) cudaSharedmemCarveoutMaxShared),
"carveout");
}
// Loaded now rather than at a first launch: with lazy loading (CUDA 12's default) a kernel's first launch may wait
// until the device is idle, and so for hold_sm, which waits for release_sms.
cudaFuncAttributes fa[kRecCount], fx;
ck(cudaFuncGetAttributes(&fa[kBefore], gdn_rec_cols_pipe_kernel), "attributes");
ck(cudaFuncGetAttributes(&fa[kKeyHead], gdn_rec_kh_kernel), "attributes");
for (const void* f : {(const void*) hold_sm, (const void*) wait_sitting, (const void*) release_sms, (const void*) land})
ck(cudaFuncGetAttributes(&fx, f), "attributes");
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&hold_per_sm, hold_sm, 1, optin), "occupancy");
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kBefore], gdn_rec_cols_pipe_kernel, CB * RG, 0), "occupancy");
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm[kKeyHead], gdn_rec_kh_kernel, CB * RG, 0), "occupancy");
const int grid[kRecCount] = {HV * NCB, HK * NCB}; // as the engine launches them
std::printf(" the engine's grids on N of this card's %d SMs (%lld tokens; the other SMs held by a sleeping kernel, see "
"fewer_sms):\n", sms, (long long) T);
const int left = smem_sm - optin - reserved; // shared memory a holding block leaves on its SM
if (hold_per_sm != 1 || (int) fa[kBefore].sharedSizeBytes + reserved <= left ||
(int) fa[kKeyHead].sharedSizeBytes + reserved <= left) {
std::printf(" not on this card: a block of a recurrence would fit beside a holding block (%d bytes left)\n",
left);
return;
}
// land(): as much dynamic shared memory as still lets as many of its blocks share an SM as of the recurrence
int land_smem[kRecCount];
for (int v = 0; v < kRecCount; ++v) {
int lo = 0, hi = optin;
while (lo < hi) {
const int mid = (lo + hi + 1) / 2;
int nb = 0;
ck(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&nb, land, CB * RG, mid), "occupancy");
if (nb >= per_sm[v]) lo = mid;
else hi = mid - 1;
}
land_smem[v] = lo;
}
// N: all SMs, and below that the SM counts of cards and the band edges, as long as both grids fit at once
std::vector<int> ns = {sms};
for (const int n : {170, 144, 132, 128, 112, 108, 96, 95, 84, 82, 80, 76, 72, 70, 66, 64, 63, 60, 56, 52, 48, 47, 46, 44,
40, 39, 38, 36, 34, 32})
if (n < sms && (int64_t) n * per_sm[kBefore] >= grid[kBefore] && (int64_t) n * per_sm[kKeyHead] >= grid[kKeyHead])
ns.push_back(n);
Inputs in(T, 0xD5);
const size_t st = (size_t) S * HV * S, oc_n = (size_t) T * HV * S;
float* state = dalloc<float>(st);
float* oc = dalloc<float>(oc_n);
unsigned long long* sitting = dalloc<unsigned long long>(1);
unsigned* flags = dalloc<unsigned>(4);
unsigned *release = flags, *late = flags + 1, *landed = flags + 2, *apart = flags + 3;
int* where = dalloc<int>((size_t) sms + grid[kBefore]); // the holding blocks' SMs, then land()'s
ck(cudaMemset(sitting, 0, sizeof(*sitting)), "memset");
ck(cudaMemset(flags, 0, 4 * sizeof(unsigned)), "memset");
cudaStream_t s_hold, s_run;
ck(cudaStreamCreateWithFlags(&s_hold, cudaStreamNonBlocking), "stream");
ck(cudaStreamCreateWithFlags(&s_run, cudaStreamNonBlocking), "stream");
cudaEvent_t e0, e1;
ck(cudaEventCreate(&e0), "event");
ck(cudaEventCreate(&e1), "event");
unsigned long long all = 0; // holding blocks launched so far: sitting counts every one that sat down
unsigned gen = 0;
auto hold = [&](int n) { // all but n SMs, before the next launch on s_run
if (n >= sms) return;
++gen;
all += (unsigned long long) (sms - n);
hold_sm<<<sms - n, 1, optin, s_hold>>>(sitting, release, gen, where, late);
ck(cudaGetLastError(), "hold_sm");
const cudaError_t q = cudaStreamQuery(s_hold); // hands the launch to the GPU now (Windows collects launches)
if (q != cudaErrorNotReady) ck(q, "hold_sm");
(void) cudaGetLastError(); // not ready is no error
wait_sitting<<<1, 1, 0, s_run>>>(sitting, all, late);
};
auto let_go = [&](int n) {
if (n < sms) release_sms<<<1, 1, 0, s_run>>>(release, gen);
};
auto settle = [&](const char* what) {
ck(cudaStreamSynchronize(s_run), what);
ck(cudaStreamSynchronize(s_hold), what);
};
auto count = [&](const unsigned* c) { // after settle()
unsigned h = 0;
ck(cudaMemcpy(&h, c, sizeof(h), cudaMemcpyDeviceToHost), "count");
return h;
};
auto done = [&] {
cudaEventDestroy(e0);
cudaEventDestroy(e1);
cudaStreamDestroy(s_hold);
cudaStreamDestroy(s_run);
cudaFree(state); cudaFree(oc); cudaFree(sitting); cudaFree(flags); cudaFree(where);
};
// where the grids land, once per N
struct Landing { bool held = true; int busiest[kRecCount] = {}, on[kRecCount] = {}; };
std::vector<Landing> lands(ns.size());
for (size_t i = 0; i < ns.size(); ++i) {
const int n = ns[i];
for (int v = 0; v < kRecCount; ++v) {
bool sat = false;
unsigned apart0 = 0;
for (int attempt = 0; attempt < 2 && !sat; ++attempt) {
const unsigned late0 = count(late);
apart0 = count(apart);
ck(cudaMemsetAsync(landed, 0, sizeof(unsigned), s_run), "memset");
hold(n);
land<<<grid[v], dim3(CB, RG), land_smem[v], s_run>>>(where + sms, landed, apart);
ck(cudaGetLastError(), "land");
let_go(n);
settle("land");
sat = count(late) == late0;
}
if (!sat) { // twice: the holding kernel and the launches beside it do not run at the same time here
std::printf(" not on this card: the other SMs could not be held (the kernel holding them and the "
"launches beside it did not run at the same time)\n");
done();
return;
}
std::vector<int> w((size_t) sms + grid[v]);
ck(cudaMemcpy(w.data(), where, w.size() * sizeof(int), cudaMemcpyDeviceToHost), "where");
std::map<int, int> held, blocks;
for (int b = 0; b < (n < sms ? sms - n : 0); ++b) ++held[w[b]];
for (int b = 0; b < grid[v]; ++b) ++blocks[w[sms + b]];
bool ok = count(apart) == apart0 && (int) held.size() == (n < sms ? sms - n : 0) && (int) blocks.size() <= n;
int most = 0, on = 0;
for (const auto& sb : blocks) {
if (held.count(sb.first)) ok = false;
if (sb.second > most) { most = sb.second; on = 0; }
if (sb.second == most) ++on;
}
lands[i].held = lands[i].held && ok;
lands[i].busiest[v] = most;
lands[i].on[v] = on;
}
}
// the timings: every N and recurrence once per round, a different one first in each round
struct Case { size_t i; Rec v; };
std::vector<Case> cases;
for (size_t i = 0; i < ns.size(); ++i)
if (lands[i].held)
for (int v = 0; v < kRecCount; ++v) cases.push_back({i, (Rec) v});
auto run = [&](const Case& cs) { // ms, or < 0 if the SMs were not held for the whole launch
const int n = ns[cs.i];
const unsigned late0 = count(late);
ck(cudaMemcpyAsync(state, in.state0, st * 4, cudaMemcpyDeviceToDevice, s_run), "state");
hold(n);
ck(cudaEventRecord(e0, s_run), "event");
run_rec(cs.v, state, in, oc, s_run);
ck(cudaEventRecord(e1, s_run), "event");
let_go(n);
settle("fewer SMs");
if (count(late) != late0) return -1.0f;
float ms = 0.0f;
ck(cudaEventElapsedTime(&ms, e0, e1), "event");
return ms;
};
const int warm = 2, rounds = 15, max_dropped = 5;
std::vector<std::vector<float>> t(cases.size());
int dropped = 0;
for (int r = 0; r < warm + rounds && dropped <= max_dropped; ++r)
for (size_t j = 0; j < cases.size() && dropped <= max_dropped; ++j) {
const size_t k = (r + j) % cases.size();
const float ms = run(cases[k]);
if (ms < 0.0f) ++dropped;
else if (r >= warm) t[k].push_back(ms);
}
if (dropped > max_dropped) { // each such run waits for a limit of a second or so
std::printf(" stopped: in %d runs the other SMs were not held for the whole launch\n", dropped);
done();
return;
}
auto us = [&](size_t i, int v) { // per token, or < 0 if not measured
for (size_t k = 0; k < cases.size(); ++k)
if (cases[k].i == i && cases[k].v == v && !t[k].empty()) return 1000.0f * median(t[k]) / (float) T;
return -1.0f;
};
auto blocks_on = [](char* out, size_t size, int most, int on) {
std::snprintf(out, size, "%d block%s on %d SM%s", most, most == 1 ? "" : "s", on, on == 1 ? "" : "s");
};
std::printf(" (medians of %d alternating runs; L2, memory and clock stay this card's)\n", rounds);
std::printf(" N before: busiest SM new: busiest SM us per token, before -> new\n");
for (size_t i = 0; i < ns.size(); ++i) {
if (!lands[i].held) {
std::printf(" %3d the other SMs were not held (a block landed on one, or a wait ran out)\n", ns[i]);
continue;
}
char b0[40], b1[40];
blocks_on(b0, sizeof b0, lands[i].busiest[kBefore], lands[i].on[kBefore]);
blocks_on(b1, sizeof b1, lands[i].busiest[kKeyHead], lands[i].on[kKeyHead]);
const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
if (u0 < 0.0f || u1 < 0.0f) std::printf(" %3d %-20s %-20s not measured\n", ns[i], b0, b1);
else std::printf(" %3d %-20s %-20s %.3f -> %.3f %.2fx\n", ns[i], b0, b1, u0, u1, u0 / u1);
}
if (dropped) std::printf(" (%d runs dropped: the other SMs were not held for the whole launch)\n", dropped);
std::printf(" so on a card with N SMs, as measured here (the kernel before puts ceil(192 / N) blocks on the busiest SM, "
"gdn_rec_kh_kernel ceil(64 / N)):\n");
const int bands[][2] = {{96, 191}, {64, 95}, {48, 63}, {39, 47}, {32, 38}};
for (const auto& bd : bands) {
const int b0 = (192 + bd[0] - 1) / bd[0], b1 = (64 + bd[0] - 1) / bd[0];
float lo = 0.0f, hi = 0.0f;
int cnt = 0, nmin = 0, nmax = 0;
for (size_t i = 0; i < ns.size(); ++i) {
const float u0 = us(i, kBefore), u1 = us(i, kKeyHead);
if (ns[i] < bd[0] || ns[i] > bd[1] || u0 < 0.0f || u1 < 0.0f) continue;
const float x = u0 / u1;
if (cnt == 0 || x < lo) lo = x;
if (cnt == 0 || x > hi) hi = x;
if (cnt == 0 || ns[i] < nmin) nmin = ns[i];
if (cnt == 0 || ns[i] > nmax) nmax = ns[i];
++cnt;
}
char range[32];
std::snprintf(range, sizeof range, "%d-%d SMs", bd[0], bd[1]);
if (cnt > 0)
std::printf(" %-12s before %d / new %d per SM: %.2fx to %.2fx (%d to %d SMs)\n", range, b0, b1, lo, hi, nmin,
nmax);
else if (bd[0] > sms)
std::printf(" %-12s before %d / new %d per SM: more SMs than this card has (see the estimate above)\n", range,
b0, b1);
else
std::printf(" %-12s before %d / new %d per SM: not measured\n", range, b0, b1);
}
done();
}
} // namespace
int main(int argc, char** argv) {
bool do_bench = false, do_self = false, do_long = false, do_bench_tc = false;
for (int i = 1; i < argc; ++i) {
if (std::strcmp(argv[i], "--bench") == 0) do_bench = true;
if (std::strcmp(argv[i], "--selftest-frag") == 0) do_self = true;
if (std::strcmp(argv[i], "--long") == 0) do_long = true;
if (std::strcmp(argv[i], "--bench-tc") == 0) do_bench_tc = true;
}
cudaDeviceProp p{};
ck(cudaGetDeviceProperties(&p, 0), "device");
std::printf("gdn_rec_parity on %s (sm_%d%d)\n", p.name, p.major, p.minor);
int fails = 0;
if (do_self) { // the tensor-core helpers alone (one warp each, any sm_80+ card; the device code below sm_80 traps)
if (p.major < 8) { std::printf("--selftest-frag: the tensor-core helpers need sm_80 or newer (skipped)\n"); return 77; }
fails = selftest_frag();
std::printf("gdn_rec_parity: %d failures\n", fails);
return fails ? 1 : 0;
}
if (do_bench_tc) { // the tensor-core timings alone
for (const int64_t T : {(int64_t) 2048, (int64_t) 8192, (int64_t) 32768}) bench(T, true);
return 0;
}
const int64_t Ts[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 33, 257, 1000, 4099};
uint64_t seed = 11;
for (const int64_t T : Ts) fails += check_bits(T, seed++);
for (const int64_t T : Ts) fails += check_chunked(T, seed++);
for (const int64_t T : {(int64_t) 8192, (int64_t) 32768}) fails += check_chunked(T, seed++);
for (const int64_t T : Ts) fails += check_tc(T, seed++);
for (const int64_t T : {(int64_t) 8192, (int64_t) 32768}) fails += check_tc(T, seed++);
for (const int64_t T : {(int64_t) 4099, (int64_t) 32768}) { fails += check_tc_stress(T, seed); seed += 8; }
if (do_long) fails += check_tc(51820, seed++); // the engine's longest prompt (about 7 GB of inputs and outputs)
if (do_bench) {
std::printf("--bench: the prompt path's DeltaNet recurrence\n");
bench(2048);
bench(8192);
bench(32768);
chunked_sweep();
scale(8192);
fewer_sms(8192);
}
std::printf("gdn_rec_parity: %d failures\n", fails);
return fails ? 1 : 0;
}