Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group rdna34 (f-145-pp-rdna34)

Niko1221b277412eb196
diff --cc src/kernels/cuda/fused_gr.cu
index 8adc141e2,e62315fca..38b7535b6
--- a/src/kernels/cuda/fused_gr.cu
+++ b/src/kernels/cuda/fused_gr.cu
@@@ -2408,20 -2332,447 +2410,463 @@@ void fused_gr_check()
}
+namespace {
+__global__ void hc_q8_bf16_kernel(const int8_t* __restrict__ q, const float* __restrict__ sc, uint16_t* __restrict__ out,
+ long long n) {
+ const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ const float v = sc[i / 32] * (float) q[i];
+ uint32_t b = __float_as_uint(v);
+ b += 0x7fffu + ((b >> 16) & 1u); // round to nearest even (the values are finite)
+ out[i] = (uint16_t) (b >> 16);
+}
+} // namespace
+
+void hc_q8_to_bf16(const void* codes, const float* scales, uint16_t* out, long long n, void* stream) {
+ hc_q8_bf16_kernel<<<(unsigned) ((n + 255) / 256), 256, 0, (cudaStream_t) stream>>>((const int8_t*) codes, scales, out, n);
+}
+
+ // ===================================== the read as ONE persistent kernel (opt-in STRATA_HC_PERSIST=1, HIP) ==============
+ // 80 blocks that must all be resident (a start-up run per T decides; larger T keep the default read) and two grid
+ // barriers on an atomic counter. Block b: (A) stages R' * w_norm for columns [128 b, 128 b + 128) (one stream) and writes
+ // that slice's sum of squares and the UNSCALED partial dots of the 320 down + 4 inject rows (the v3 identity
+ // w . xn = sum_c rs[c] (w[:, c] . x'[c])); barrier; (B) rs per stream, then rows (k, r) = b + 80 j reduced to lo / inject;
+ // barrier; (C) the up projection read straight from VRAM, a thread pair per row, the epilogue as gr_up_kernel.
+ // ANOTHER SUMMATION ORDER than the default read (not bitwise; the init check bounds the difference). Only the verifier's hyper-connection reads use it: they run on one stream per card (both pipeline
+ // verifiers share it), so two of these kernels never wait on each other; the drafter's reads keep the default kernels.
+ #if defined(__HIPCC__)
+ namespace {
+ namespace hcp {
+ // Measured on gfx1030 (bench/results/2026-10-12-hc-persist-gfx1030): the gain over the three-kernel read is (1) the down
+ // projection split by column slices over all 80 blocks (the default read runs 41 blocks staging the activations through
+ // LDS) and (2) one launch with two grid barriers instead of three launches and the gaps between them. Prefetching the
+ // up weights (into registers or by touching them into L2 / Infinity Cache) and the epilogue inputs measured no gain on
+ // gfx1030 and is gone: the 13 MB are read once from VRAM whichever way.
+ constexpr int NB = 80, SL = D / NB, SPS = N / SL, UPC = N / NB, NR = LR + HC;
+ static_assert(UPC * HC * 2 == THREADS, "a thread pair per up row");
+ struct PArgs {
+ FusedGrArgs a[kFusedGrMaxT];
+ float* part; // [T][NR][NB]
+ float* ssp; // [T][NB]
+ unsigned* bar; // [0] count, [32] generation, [48] abort (a barrier that timed out)
+ unsigned long long timeout_ticks; // s_memrealtime ticks (100 MHz) a barrier waits before it aborts the read
+ double* partd; // STRATA_HC_PERSIST=3 (a reference only): the partial dots and sums of squares in double
+ double* sspd;
+ };
+ // Returns false when the read was aborted: a block waited longer than `ticks` (the 80 blocks were not all resident - the
+ // start-up check finds that out by running every T with a short timeout) or another block aborted. Then every block
+ // leaves the kernel; the outputs are not written completely and bar[48] stays set until the host clears the words.
+ __device__ __forceinline__ bool grid_sync(unsigned* bar, unsigned long long ticks) {
+ #if defined(__GFX11__) || defined(__GFX12__)
+ // gfx11 and gfx12 have no s_memrealtime (clang: "needs target feature s-memrealtime"), so no barrier clock: this read is not
+ // offered there (fused_gr_persist_init refuses it; an R9700 measured it neutral to 1% slower in #2226), and a launch that gets
+ // here anyway aborts at once instead of waiting for a barrier it cannot time.
+ (void) ticks;
+ if (threadIdx.x == 0) __hip_atomic_store(&bar[48], 1u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+ return false;
+ #else
+ __shared__ int s_abort;
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ int ab = 0;
+ const unsigned gen = __hip_atomic_load(&bar[32], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+ __threadfence();
+ const unsigned arrived = __hip_atomic_fetch_add(&bar[0], 1u, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_AGENT);
+ if (arrived == NB - 1) {
+ __hip_atomic_store(&bar[0], 0u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+ __hip_atomic_fetch_add(&bar[32], 1u, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT);
+ } else {
+ const unsigned long long t0 = __builtin_amdgcn_s_memrealtime();
+ while (__hip_atomic_load(&bar[32], __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT) == gen) {
+ if (__hip_atomic_load(&bar[48], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) != 0u) { ab = 1; break; }
+ if (__builtin_amdgcn_s_memrealtime() - t0 > ticks) {
+ __hip_atomic_store(&bar[48], 1u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+ ab = 1;
+ break;
+ }
+ __builtin_amdgcn_s_sleep(1);
+ }
+ }
+ if (__hip_atomic_load(&bar[48], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) != 0u) ab = 1;
+ s_abort = ab;
+ }
+ __syncthreads();
+ __threadfence();
+ return s_abort == 0;
+ #endif
+ }
+ // STRATA_HC_PERSIST=3 (a reference only, slow): every dot and sum accumulated in double
+ __device__ __forceinline__ double dot8d(const uint4 w, const float* x) {
+ double acc = 0.0;
+ const unsigned u[4] = {w.x, w.y, w.z, w.w};
+ #pragma unroll
+ for (int q = 0; q < 4; ++q) {
+ acc += (double) __uint_as_float(u[q] << 16) * (double) x[2 * q];
+ acc += (double) __uint_as_float(u[q] & 0xffff0000u) * (double) x[2 * q + 1];
+ }
+ return acc;
+ }
+ template <int T, bool DP = false>
+ __global__ void __launch_bounds__(THREADS) persist_kernel(PArgs f) {
+ using A = typename std::conditional<DP, double, float>::type;
+ __shared__ __align__(16) float xs[T][SL];
+ __shared__ __align__(16) float lo[T][LR];
+ __shared__ float s_rs[T][HC];
+ __shared__ float g[T][HC][UPC];
+ const int tid = threadIdx.x, lane = tid & 31, b = blockIdx.x;
+ const FusedGrArgs& a0 = f.a[0];
+ // (A) block b: columns [128 b, 128 b + 128) of D (one stream): R' * w_norm into LDS, the slice's sum of squares, and
+ // the UNSCALED partial dots of the 320 down + 4 inject rows (w . xn = sum_c rs[c] (w[:, c] . x'[c]))
+ const int c_b = b / SPS, col0 = b * SL;
+ if (tid < T * (SL / 4)) {
+ const int k = tid / (SL / 4), o4 = tid - k * (SL / 4);
+ const FusedGrArgs& a = f.a[k];
+ const int i = col0 + o4 * 4, d = i - c_b * N;
+ float4 r = *reinterpret_cast<const float4*>(a.R + i);
+ if (a.apply) {
+ const float gw = 2.0f * sigmoidf_(a.inj_prev[c_b] / (float) HC);
+ const float4 bo = *reinterpret_cast<const float4*>(a.bo_prev + d);
+ r.x = fmaf(bo.x, gw, r.x); r.y = fmaf(bo.y, gw, r.y); r.z = fmaf(bo.z, gw, r.z); r.w = fmaf(bo.w, gw, r.w);
+ }
+ const float4 gn = *reinterpret_cast<const float4*>(a.w_norm + i);
+ if constexpr (DP) {
+ double ss = (double) r.x * r.x + (double) r.y * r.y + (double) r.z * r.z + (double) r.w * r.w;
+ #pragma unroll
+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor(ss, o);
+ if (lane == 0) f.sspd[k * NB + b] = ss;
+ } else {
+ const float ss = warp_sum(r.x * r.x + r.y * r.y + r.z * r.z + r.w * r.w); // warp k = token k
+ if (lane == 0) f.ssp[k * NB + b] = ss;
+ }
+ *reinterpret_cast<float4*>(&xs[k][o4 * 4]) = make_float4(r.x * gn.x, r.y * gn.y, r.z * gn.z, r.w * gn.w);
+ }
+ __syncthreads();
+ const int hw = tid >> 4, hl = tid & 15;
+ const bool inj = a0.w_inject != nullptr;
+ const int nrows = inj ? NR : LR;
+ #pragma unroll 3
+ for (int r = hw; r < nrows; r += 16) {
+ const uint16_t* wrow = r < LR ? a0.w_down + (size_t) r * D : a0.w_inject + (size_t) (r - LR) * D;
+ const uint4 w = __ldg(reinterpret_cast<const uint4*>(wrow + col0) + hl);
+ A acc[T];
+ #pragma unroll
+ for (int k = 0; k < T; ++k) {
+ A v;
+ if constexpr (DP) v = dot8d(w, &xs[k][hl * 8]); else v = dot8(w, &xs[k][hl * 8]);
+ #pragma unroll
+ for (int o = 8; o > 0; o >>= 1) v += __shfl_xor(v, o);
+ acc[k] = v;
+ }
+ #pragma unroll
+ for (int k = 0; k < T; ++k)
+ if (hl == k) {
+ if constexpr (DP) f.partd[((size_t) k * NR + r) * NB + b] = acc[k];
+ else f.part[((size_t) k * NR + r) * NB + b] = (float) acc[k];
+ }
+ }
+ if (!grid_sync(f.bar, f.timeout_ticks)) return;
+ // (B) rs per stream (every block), then rows (k, r) = b + 80 j reduced to lo / inject by the blocks in turn
+ if (tid < T * HC) {
+ const int k = tid / HC, c = tid - k * HC;
+ float rr;
+ if constexpr (DP) {
+ double sd = 0.0;
+ for (int j = 0; j < SPS; ++j) sd += f.sspd[k * NB + c * SPS + j];
+ rr = (float) (1.0 / sqrt(sd / (double) N + (double) f.a[k].eps));
+ } else {
+ float s = 0.0f;
+ for (int j = 0; j < SPS; ++j) s += f.ssp[k * NB + c * SPS + j];
+ rr = rsqrtf(s / (float) N + f.a[k].eps);
+ }
+ s_rs[k][c] = rr;
+ if (b == 0) f.a[k].rs[c] = rr;
+ }
+ __syncthreads();
+ for (int idx = b + NB * tid; idx < T * nrows; idx += NB * THREADS) {
+ const int k = idx / nrows, r = idx - k * nrows;
+ A sum = 0;
+ if constexpr (DP) {
+ const double* p1 = f.partd + ((size_t) k * NR + r) * NB;
+ for (int c = 0; c < HC; ++c) {
+ double p = 0.0;
+ for (int j = 0; j < SPS; ++j) p += p1[c * SPS + j];
+ sum += (double) s_rs[k][c] * p;
+ }
+ } else {
+ const float4* p4 = reinterpret_cast<const float4*>(f.part + ((size_t) k * NR + r) * NB);
+ #pragma unroll
+ for (int c = 0; c < HC; ++c) {
+ float p = 0.0f;
+ #pragma unroll
+ for (int j = 0; j < SPS / 4; ++j) {
+ const float4 v = p4[c * (SPS / 4) + j];
+ p += (v.x + v.y) + (v.z + v.w);
+ }
+ sum = fmaf(s_rs[k][c], p, sum);
+ }
+ }
+ if (r < LR) {
+ const A x = sum / (A) HC;
+ f.a[k].lo[r] = (float) (x / ((A) 1 + (DP ? (A) exp((double) -x) : (A) __expf((float) -x))));
+ } else {
+ f.a[k].inject_out[r - LR] = (float) sum;
+ }
+ }
+ if (!grid_sync(f.bar, f.timeout_ticks)) return;
+ for (int idx = tid; idx < T * LR; idx += THREADS) {
+ const int k = idx / LR, r = idx - k * LR;
+ lo[k][r] = f.a[k].lo[r];
+ }
+ __syncthreads();
+ // (C) the up projection, a thread pair per row (block b: columns 32 b .. 32 b + 31 of all four streams), then the
+ // epilogue as gr_up_kernel
+ {
+ const int d0 = b * UPC;
+ const int ur = tid >> 1, uh = tid & 1, uc = ur / UPC, udd = ur - uc * UPC, ui = uc * N + d0 + udd;
+ const uint4* w4u = reinterpret_cast<const uint4*>(a0.w_up + (size_t) ui * LR) + uh * 20;
+ A v[T];
+ #pragma unroll
+ for (int k = 0; k < T; ++k) v[k] = 0;
+ #pragma unroll 1
+ for (int j0 = 0; j0 < 20; j0 += 5) { // five chunks a batch (all twenty at once costs registers)
+ uint4 w[5];
+ #pragma unroll
+ for (int q = 0; q < 5; ++q) w[q] = __ldg(w4u + j0 + q);
+ #pragma unroll
+ for (int q = 0; q < 5; ++q)
+ #pragma unroll
+ for (int k = 0; k < T; ++k) {
+ if constexpr (DP) v[k] += dot8d(w[q], &lo[k][uh * 160] + (j0 + q) * 8);
+ else v[k] += dot8(w[q], &lo[k][uh * 160] + (j0 + q) * 8);
+ }
+ }
+ float acc[T];
+ #pragma unroll
+ for (int k = 0; k < T; ++k) acc[k] = (float) (v[k] + __shfl_xor(v[k], 1));
+ if (uh == 0) {
+ const float wn = a0.w_norm[ui];
+ #pragma unroll
+ for (int k = 0; k < T; ++k) {
+ const FusedGrArgs& a = f.a[k];
+ float rv = a.R[ui];
+ if (a.apply) {
+ rv = fmaf(a.bo_prev[d0 + udd], 2.0f * sigmoidf_(a.inj_prev[uc] / (float) HC), rv);
+ a.R_out[ui] = rv;
+ }
+ g[k][uc][udd] = rv * wn * s_rs[k][uc] * sigmoidf_(acc[k]);
+ }
+ }
+ }
+ __syncthreads();
+ for (int idx = tid; idx < T * UPC; idx += THREADS) {
+ const int k = idx / UPC, col = idx - k * UPC;
+ float s = 0.0f;
+ #pragma unroll
+ for (int c = 0; c < HC; ++c) s += g[k][c][col];
+ f.a[k].mixed[b * UPC + col] = s / (float) HC;
+ }
+ }
+ struct DevState {
+ float* part = nullptr;
+ float* ssp = nullptr;
+ double* partd = nullptr;
+ double* sspd = nullptr;
+ unsigned* bar = nullptr;
+ int max_t = 0; // 0: off on this card
+ bool tried = false;
+ };
+ DevState g_dev[64];
+ using KFn = void (*)(PArgs);
+ template <bool DP>
+ KFn kernel_t(int T) {
+ switch (T) {
+ case 1: return persist_kernel<1, DP>;
+ case 2: return persist_kernel<2, DP>;
+ case 3: return persist_kernel<3, DP>;
+ case 4: return persist_kernel<4, DP>;
+ case 5: return persist_kernel<5, DP>;
+ case 6: return persist_kernel<6, DP>;
+ case 7: return persist_kernel<7, DP>;
+ case 8: return persist_kernel<8, DP>;
+ }
+ return nullptr;
+ }
+ bool dp_on() {
+ static const bool dp = [] { const char* v = std::getenv("STRATA_HC_PERSIST"); return v != nullptr && std::atoi(v) == 3; }();
+ return dp;
+ }
+ KFn kernel_for(int T) { return dp_on() ? kernel_t<true>(T) : kernel_t<false>(T); }
+ // s_memrealtime runs at 100 MHz on RDNA: a barrier gives up after 2 s in a read (never in a working run; it only turns a
+ // hang into a wrong read) and after 50 ms in the start-up check (where a timeout means "not resident here")
+ constexpr unsigned long long kTicksRun = 200000000ULL, kTicksCheck = 5000000ULL;
+ void launch(const FusedGrArgs* a, int n_tok, const DevState& st, cudaStream_t s, unsigned long long ticks) {
+ PArgs f;
+ for (int t = 0; t < n_tok; ++t) f.a[t] = a[t];
+ f.part = st.part; f.ssp = st.ssp; f.bar = st.bar; f.partd = st.partd; f.sspd = st.sspd; f.timeout_ticks = ticks;
+ kernel_for(n_tok)<<<NB, THREADS, 0, s>>>(f);
+ }
+ bool env_on() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_HC_PERSIST"); return v != nullptr && std::atoi(v) != 0; }();
+ return on;
+ }
+ } // namespace hcp
+ } // namespace
+
+ void fused_gr_persist_init() {
+ if (!hcp::env_on()) return;
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 64 || hcp::g_dev[dev].tried) return;
+ hcp::DevState& st = hcp::g_dev[dev];
+ st.tried = true;
+ cudaDeviceProp prop{};
+ cudaGetDeviceProperties(&prop, dev);
+ if (prop.warpSize != 32) {
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: the kernel is written for 32-lane waves "
+ "(this card runs %d)\n", dev, prop.warpSize);
+ return;
+ }
+ if (strata::kernels::gfx_arch_is_gfx12(prop.gcnArchName) || strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName)) {
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: gfx11 / gfx12 have no s_memrealtime for the barrier "
+ "timeout (written and measured for gfx1030; an R9700 saw no gain from it, #2226)\n", dev);
+ return;
+ }
+ const int mp = prop.multiProcessorCount; // only for the confirmation line (WGPs on RDNA)
+ bool ok = cudaMalloc((void**) &st.part, (size_t) kFusedGrMaxT * hcp::NR * hcp::NB * sizeof(float)) == cudaSuccess &&
+ cudaMalloc((void**) &st.ssp, (size_t) kFusedGrMaxT * hcp::NB * sizeof(float)) == cudaSuccess &&
+ cudaMalloc((void**) &st.bar, 64 * sizeof(unsigned)) == cudaSuccess &&
+ cudaMalloc((void**) &st.partd, (size_t) kFusedGrMaxT * hcp::NR * hcp::NB * sizeof(double)) == cudaSuccess &&
+ cudaMalloc((void**) &st.sspd, (size_t) kFusedGrMaxT * hcp::NB * sizeof(double)) == cudaSuccess &&
+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned)) == cudaSuccess;
+ if (!ok) {
+ cudaGetLastError();
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: workspace allocation failed\n", dev);
+ return;
+ }
+ // the check: random weights and inputs with the pending write and the inject rows, against the default read; the
+ // persistent read sums in another order, so it is bounded, not bitwise. Every T is RUN with a 50 ms barrier timeout; a
+ // timeout means "not resident here" and larger T keep the default read. No occupancy API in front of it: it promised
+ // more than the card holds on 30 WGPs under HSA_CU_MASK (a hang without the timeout) and less than it holds with
+ // the HIP 7.1 runtime (1 block per WGP on 40 WGPs, which turned the read off although it runs resident).
+ std::mt19937 rng(11);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ auto bf = [](float x) { uint32_t u; std::memcpy(&u, &x, 4); return (uint16_t) ((u + 0x7fff + ((u >> 16) & 1)) >> 16); };
+ const int TM = kFusedGrMaxT;
+ std::vector<float> R((size_t) TM * D), bo((size_t) TM * N), inj((size_t) TM * HC), wn(D);
+ for (auto& x : R) x = nd(rng);
+ for (auto& x : bo) x = 0.5f * nd(rng);
+ for (auto& x : inj) x = nd(rng);
+ for (auto& x : wn) x = 1.0f + 0.3f * nd(rng);
+ std::vector<uint16_t> wd((size_t) LR * D), wu((size_t) D * LR), wi((size_t) HC * D);
+ for (auto& x : wd) x = bf(0.02f * nd(rng));
+ for (auto& x : wu) x = bf(0.05f * nd(rng));
+ for (auto& x : wi) x = bf(0.02f * nd(rng));
+ float *dR, *dbo, *dinj, *dwn, *dxn, *dlo, *drs, *dio, *dmix;
+ uint16_t *dwd, *dwu, *dwi;
+ cudaMalloc((void**) &dR, R.size() * 4); cudaMalloc((void**) &dbo, bo.size() * 4);
+ cudaMalloc((void**) &dinj, inj.size() * 4); cudaMalloc((void**) &dwn, wn.size() * 4);
+ cudaMalloc((void**) &dxn, (size_t) TM * D * 4); cudaMalloc((void**) &dlo, (size_t) TM * LR * 4);
+ cudaMalloc((void**) &drs, (size_t) TM * HC * 4); cudaMalloc((void**) &dio, (size_t) TM * HC * 4);
+ cudaMalloc((void**) &dmix, (size_t) TM * N * 4);
+ cudaMalloc((void**) &dwd, wd.size() * 2); cudaMalloc((void**) &dwu, wu.size() * 2); cudaMalloc((void**) &dwi, wi.size() * 2);
+ cudaMemcpy(dbo, bo.data(), bo.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dinj, inj.data(), inj.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dwn, wn.data(), wn.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dwd, wd.data(), wd.size() * 2, cudaMemcpyHostToDevice);
+ cudaMemcpy(dwu, wu.data(), wu.size() * 2, cudaMemcpyHostToDevice);
+ cudaMemcpy(dwi, wi.data(), wi.size() * 2, cudaMemcpyHostToDevice);
+ double worst = 0.0;
+ int bad_t = 0, timeouts = 0;
+ // 0: the read agrees with the default read; 1: a barrier timed out (not resident); 2: wrong or a device error
+ auto check = [&](int T) -> int {
+ FusedGrArgs a[kFusedGrMaxT];
+ for (int t = 0; t < T; ++t) {
+ a[t] = FusedGrArgs{};
+ a[t].R = dR + (size_t) t * D; a[t].R_out = dR + (size_t) t * D; a[t].apply = true;
+ a[t].bo_prev = dbo + (size_t) t * N; a[t].inj_prev = dinj + (size_t) t * HC;
+ a[t].w_norm = dwn; a[t].w_down = dwd; a[t].w_up = dwu; a[t].w_inject = dwi; a[t].eps = 1e-6f;
+ a[t].lo = dlo + (size_t) t * LR; a[t].rs = drs + (size_t) t * HC;
+ a[t].inject_out = dio + (size_t) t * HC; a[t].mixed = dmix + (size_t) t * N;
+ }
+ std::vector<float> out[2];
+ for (int which = 0; which < 2; ++which) {
+ cudaMemcpy(dR, R.data(), R.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemset(dmix, 0xff, (size_t) TM * N * 4);
+ cudaMemset(dio, 0xff, (size_t) TM * HC * 4);
+ cudaDeviceSynchronize();
+ if (which == 0) fused_gr_read_multi(a, T, dxn, nullptr);
+ else hcp::launch(a, T, st, nullptr, hcp::kTicksCheck);
+ if (cudaDeviceSynchronize() != cudaSuccess) { cudaGetLastError(); return 2; }
+ if (which == 1) {
+ unsigned abort_word = 0;
+ cudaMemcpy(&abort_word, st.bar + 48, sizeof abort_word, cudaMemcpyDeviceToHost);
+ if (abort_word != 0u) {
+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned));
+ cudaDeviceSynchronize();
+ return 1;
+ }
+ }
+ out[which].resize((size_t) T * (N + HC + D));
+ cudaMemcpy(out[which].data(), dmix, (size_t) T * N * 4, cudaMemcpyDeviceToHost);
+ cudaMemcpy(out[which].data() + (size_t) T * N, dio, (size_t) T * HC * 4, cudaMemcpyDeviceToHost);
+ cudaMemcpy(out[which].data() + (size_t) T * (N + HC), dR, (size_t) T * D * 4, cudaMemcpyDeviceToHost);
+ }
+ double mref = 0.0, md = 0.0;
+ bool finite = true;
+ for (size_t i = 0; i < out[0].size(); ++i) {
+ finite = finite && std::isfinite(out[1][i]);
+ mref = std::fmax(mref, std::fabs((double) out[0][i]));
+ md = std::fmax(md, std::fabs((double) out[1][i] - out[0][i]));
+ }
+ const double rel = md / (mref > 0 ? mref : 1.0);
+ worst = std::fmax(worst, rel);
+ return finite && rel < 1e-4 ? 0 : 2;
+ };
+ int max_t = 0;
+ for (int T = 1; T <= kFusedGrMaxT && ok; ++T) {
+ const int r = check(T);
+ if (r == 1) ++timeouts; else if (r == 2) { bad_t = T; ok = false; }
+ if (r != 0) break;
+ max_t = T;
+ }
+ cudaFree(dR); cudaFree(dbo); cudaFree(dinj); cudaFree(dwn); cudaFree(dxn); cudaFree(dlo); cudaFree(drs);
+ cudaFree(dio); cudaFree(dmix); cudaFree(dwd); cudaFree(dwu); cudaFree(dwi);
+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned));
+ cudaDeviceSynchronize();
+ cudaGetLastError();
+ if (!ok) {
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: the check failed at %d tokens "
+ "(max relative difference %.3g)\n", dev, bad_t, worst);
+ return;
+ }
+ if (max_t == 0) {
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: 80 blocks are not resident at once on %d "
+ "multiprocessors (the check timed out)\n", dev, mp);
+ return;
+ }
+ st.max_t = max_t;
+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1: the verifier's hyper-connection read runs as one "
+ "persistent kernel (80 blocks on %d multiprocessors, up to %d tokens; %d token counts timed out in "
+ "the check; checked against the default read, max relative difference %.2g)\n", dev, mp, max_t,
+ timeouts, worst);
+ }
+
+ bool fused_gr_read_persist(const FusedGrArgs* a, int n_tok, void* stream) {
+ if (!hcp::env_on() || n_tok < 1) return false;
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 64) return false;
+ const hcp::DevState& st = hcp::g_dev[dev];
+ if (n_tok > st.max_t) return false;
+ if (a[0].q8_down != nullptr || a[0].q8_up != nullptr) return false; // STRATA_HC_Q8: the default read
+ for (int t = 0; t < n_tok; ++t)
+ if (a[t].q8_mixed != nullptr) return false; // STRATA_QFUSE: the default read
+ hcp::launch(a, n_tok, st, (cudaStream_t) stream, hcp::kTicksRun);
+ return true;
+ }
+ #else
+ void fused_gr_persist_init() {}
+ bool fused_gr_read_persist(const FusedGrArgs*, int, void*) { return false; }
+ #endif
+
} // namespace strata::kernels
diff --cc src/kernels/cuda/qsa_select.cu
index 02e6c750a,5fd67f95b..75bd1fa6e
--- a/src/kernels/cuda/qsa_select.cu
+++ b/src/kernels/cuda/qsa_select.cu
@@@ -887,114 -868,106 +887,214 @@@ __device__ __forceinline__ void block_s
}
}
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit, QsaScoreRows srow = {}) {
+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);
+}
+
+// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the
+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).
+struct ScoreSlotKeys {
+ const float* pooled[MQ];
+ const float* dead[MQ];
+};
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit) {
+ const unsigned z = blockIdx.y;
+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory
+ const float* pooled = keys.pooled[0];
+ const float* dead = keys.dead[0];
+#pragma unroll
+ for (unsigned i = 1; i < MQ; ++i)
+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }
+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,
+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,
+ QsaScoreRows{});
+}
+
+// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue
+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a
+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory
+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two
+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices
+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products
+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered
+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the
+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the
+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.
+template <int NQ>
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out) {
+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];
+ if (threadIdx.x < nq) {
+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];
+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];
+ }
+ __syncthreads();
+ int64_t top = 0;
+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)
+ const int lane = threadIdx.x & 31;
+ float4 qr[NQ][IDX_HEADS];
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi)
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)
+ : make_float4(0.f, 0.f, 0.f, 0.f);
+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;
+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) {
+ if (qi >= nq) break;
+ const int64_t n_bid = s_nbid[qi];
+ if (b > n_bid) continue;
+ const float4 k4 = (b == n_bid) ? kd : kp;
+ float d[IDX_HEADS];
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float4 q4 = qr[qi][h];
+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ }
+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head
+ float r2[2];
+#pragma unroll
+ for (int i = 0; i < 2; ++i)
+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);
+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);
+ r += __shfl_xor_sync(0xffffffffu, r, 4);
+ r += __shfl_xor_sync(0xffffffffu, r, 2);
+ r += __shfl_xor_sync(0xffffffffu, r, 1);
+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel
+ const float s1 = __shfl_sync(0xffffffffu, r, 8);
+ const float s2 = __shfl_sync(0xffffffffu, r, 16);
+ const float s3 = __shfl_sync(0xffffffffu, r, 24);
+ if (lane == 0) {
+ float score = 0.0f;
+ score += r > 0.0f ? r : 0.0f;
+ score += s1 > 0.0f ? s1 : 0.0f;
+ score += s2 > 0.0f ? s2 : 0.0f;
+ score += s3 > 0.0f ? s3 : 0.0f;
+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;
+ out[qi * max_blocks + b] = score;
+ }
+ }
+ }
+}
+
+ #if defined(__HIPCC__)
+ // Optional gfx1100 port of PR1825 (14bf3215): same FP32 expression and reduction tree.
+ // Decode and the original top-k stay unchanged. No additional persistent weight/state allocation.
+ constexpr int TS_Q = 64; // queries per CTA (8 per wave, four lanes per query)
+ constexpr int TS_KB = 32; // key blocks per shared-memory tile
+ constexpr int TS_BR = 512; // key blocks per CTA
+
+ __device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {
+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression
+ }
+
+ __global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int64_t nq,
+ int64_t max_blocks, int64_t reach,
+ float* __restrict__ out) {
+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];
+ __shared__ float so[TS_Q][TS_KB + 1];
+ __shared__ int64_t s_nbid[TS_Q];
+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's
+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet
+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer
+ // shared-memory loads per FMA. The MIO-pipe motivation was measured by PR1825 on NVIDIA.
+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};
+ const int t = threadIdx.x, lane = t & 31;
+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter
+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;
+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;
+ float4 qv[IDX_HEADS][8];
+ {
+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ #pragma unroll
+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];
+ }
+ __syncthreads();
+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here
+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;
+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;
+ int64_t b_hi = b_lo + TS_BR;
+ if (b_hi > reach) b_hi = reach;
+ if (b_hi > top) b_hi = top;
+ if (b_hi > max_blocks) b_hi = max_blocks;
+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {
+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);
+ __syncthreads(); // the previous tile's reads of ks and so are done
+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)
+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];
+ __syncthreads();
+ for (int j = 0; j < nb; ++j) {
+ float4 kk[8];
+ #pragma unroll
+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];
+ float e[IDX_HEADS];
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) +
+ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) +
+ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) +
+ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));
+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3
+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)
+ }
+ if (j4 == 0) {
+ float score = 0.0f;
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;
+ so[ql][j] = score;
+ }
+ }
+ __syncthreads();
+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {
+ const int q = i / nb, j = i % nb;
+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];
+ }
+ }
+ }
+
+ // The exact scorer runs on gfx1100 (the PR's own card) when STRATA_SCORES_THREAD=1, and on gfx12 (RDNA4) unless STRATA_SCORES_THREAD=0:
+ // there it is bitwise equal to the warp scorer on an R9700 (qsa_scores_gfx1100_parity --full, 35 cases; exact_pp residual dumps at 70K and
+ // 131K tokens) and 3.6-4.4x faster in the scorer stage, +8% (131K) to +15% (230K) on a prompt read. Other wave32 parts have no measurement.
+ // Returns 0 (not offered), 1 (gfx1100: only on request), 2 (gfx12: the default).
+ int scores_thread_arch() {
+ int device = -1;
+ if (cudaGetDevice(&device) != cudaSuccess) { cudaGetLastError(); return 0; }
+ static thread_local int cached_device = -1;
+ static thread_local int arch = 0;
+ if (device != cached_device) {
+ cudaDeviceProp properties{};
+ arch = 0;
+ if (cudaGetDeviceProperties(&properties, device) == cudaSuccess)
+ arch = gfx_arch_is_gfx12(properties.gcnArchName) ? 2 : gfx_arch_is(properties.gcnArchName, "gfx1100") ? 1 : 0;
+ cached_device = device;
+ cudaGetLastError();
+ }
+ return arch;
+ }
+ #endif
+
#if !defined(__HIPCC__)
// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or
// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
@@@ -1516,14 -1459,10 +1645,15 @@@ void qsa_block_topk(const float* scores
const int64_t reach = counted && active_blocks < max_blocks ? active_blocks : max_blocks;
const int64_t fit = (int64_t) TK_T * (counted ? TK_PER_MAX : TK_PER);
#if defined(__HIPCC__)
- constexpr int64_t kRegMinBlocks = 7168; // gfx1201: below ~28K cells the 1,024-thread kernel's fixed cost loses to the ref
+ // gfx1201: below ~28K cells (7,168 blocks) the 1,024-thread kernel's fixed cost loses to the ref. STRATA_TOPK_REG_MIN_BLOCKS
+ // moves the line (0: the register kernel always); gfx1151's table sets 4096, the crossover with the cooperative scan
+ static const int64_t kRegMinBlocks = [] {
+ const char* v = std::getenv("STRATA_TOPK_REG_MIN_BLOCKS");
+ return v != nullptr ? (int64_t) std::atoll(v) : (int64_t) 7168;
+ }();
const bool too_small = counted && reach < kRegMinBlocks;
- if (old || too_small || reach > fit) {
+ // campaign3: `reach > fit` no longer forces the ref radix - the wide kernel below takes it (same ids).
+ if (old || too_small) {
#else
constexpr int64_t kWideMinBlocks = 4608; // RTX 3060: a prompt batch below ~18K cells is faster on the ref
const bool too_small = reach > fit && active_blocks > 0 && active_blocks < kWideMinBlocks;