Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group rdna34 (f-145-pp-rdna34)
Niko1221b277412eb196
diff --cc src/kernels/cuda/fused_gr.cuindex 8adc141e2,e62315fca..38b7535b6--- a/src/kernels/cuda/fused_gr.cu+++ b/src/kernels/cuda/fused_gr.cu@@@ -2408,20 -2332,447 +2410,463 @@@ void fused_gr_check()}+namespace {+__global__ void hc_q8_bf16_kernel(const int8_t* __restrict__ q, const float* __restrict__ sc, uint16_t* __restrict__ out,+ long long n) {+ const long long i = (long long) blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ const float v = sc[i / 32] * (float) q[i];+ uint32_t b = __float_as_uint(v);+ b += 0x7fffu + ((b >> 16) & 1u); // round to nearest even (the values are finite)+ out[i] = (uint16_t) (b >> 16);+}+} // namespace++void hc_q8_to_bf16(const void* codes, const float* scales, uint16_t* out, long long n, void* stream) {+ hc_q8_bf16_kernel<<<(unsigned) ((n + 255) / 256), 256, 0, (cudaStream_t) stream>>>((const int8_t*) codes, scales, out, n);+}++ // ===================================== the read as ONE persistent kernel (opt-in STRATA_HC_PERSIST=1, HIP) ==============+ // 80 blocks that must all be resident (a start-up run per T decides; larger T keep the default read) and two grid+ // barriers on an atomic counter. Block b: (A) stages R' * w_norm for columns [128 b, 128 b + 128) (one stream) and writes+ // that slice's sum of squares and the UNSCALED partial dots of the 320 down + 4 inject rows (the v3 identity+ // w . xn = sum_c rs[c] (w[:, c] . x'[c])); barrier; (B) rs per stream, then rows (k, r) = b + 80 j reduced to lo / inject;+ // barrier; (C) the up projection read straight from VRAM, a thread pair per row, the epilogue as gr_up_kernel.+ // ANOTHER SUMMATION ORDER than the default read (not bitwise; the init check bounds the difference). Only the verifier's hyper-connection reads use it: they run on one stream per card (both pipeline+ // verifiers share it), so two of these kernels never wait on each other; the drafter's reads keep the default kernels.+ #if defined(__HIPCC__)+ namespace {+ namespace hcp {+ // Measured on gfx1030 (bench/results/2026-10-12-hc-persist-gfx1030): the gain over the three-kernel read is (1) the down+ // projection split by column slices over all 80 blocks (the default read runs 41 blocks staging the activations through+ // LDS) and (2) one launch with two grid barriers instead of three launches and the gaps between them. Prefetching the+ // up weights (into registers or by touching them into L2 / Infinity Cache) and the epilogue inputs measured no gain on+ // gfx1030 and is gone: the 13 MB are read once from VRAM whichever way.+ constexpr int NB = 80, SL = D / NB, SPS = N / SL, UPC = N / NB, NR = LR + HC;+ static_assert(UPC * HC * 2 == THREADS, "a thread pair per up row");+ struct PArgs {+ FusedGrArgs a[kFusedGrMaxT];+ float* part; // [T][NR][NB]+ float* ssp; // [T][NB]+ unsigned* bar; // [0] count, [32] generation, [48] abort (a barrier that timed out)+ unsigned long long timeout_ticks; // s_memrealtime ticks (100 MHz) a barrier waits before it aborts the read+ double* partd; // STRATA_HC_PERSIST=3 (a reference only): the partial dots and sums of squares in double+ double* sspd;+ };+ // Returns false when the read was aborted: a block waited longer than `ticks` (the 80 blocks were not all resident - the+ // start-up check finds that out by running every T with a short timeout) or another block aborted. Then every block+ // leaves the kernel; the outputs are not written completely and bar[48] stays set until the host clears the words.+ __device__ __forceinline__ bool grid_sync(unsigned* bar, unsigned long long ticks) {+ #if defined(__GFX11__) || defined(__GFX12__)+ // gfx11 and gfx12 have no s_memrealtime (clang: "needs target feature s-memrealtime"), so no barrier clock: this read is not+ // offered there (fused_gr_persist_init refuses it; an R9700 measured it neutral to 1% slower in #2226), and a launch that gets+ // here anyway aborts at once instead of waiting for a barrier it cannot time.+ (void) ticks;+ if (threadIdx.x == 0) __hip_atomic_store(&bar[48], 1u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);+ return false;+ #else+ __shared__ int s_abort;+ __syncthreads();+ if (threadIdx.x == 0) {+ int ab = 0;+ const unsigned gen = __hip_atomic_load(&bar[32], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);+ __threadfence();+ const unsigned arrived = __hip_atomic_fetch_add(&bar[0], 1u, __ATOMIC_ACQ_REL, __HIP_MEMORY_SCOPE_AGENT);+ if (arrived == NB - 1) {+ __hip_atomic_store(&bar[0], 0u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);+ __hip_atomic_fetch_add(&bar[32], 1u, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_AGENT);+ } else {+ const unsigned long long t0 = __builtin_amdgcn_s_memrealtime();+ while (__hip_atomic_load(&bar[32], __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT) == gen) {+ if (__hip_atomic_load(&bar[48], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) != 0u) { ab = 1; break; }+ if (__builtin_amdgcn_s_memrealtime() - t0 > ticks) {+ __hip_atomic_store(&bar[48], 1u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);+ ab = 1;+ break;+ }+ __builtin_amdgcn_s_sleep(1);+ }+ }+ if (__hip_atomic_load(&bar[48], __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) != 0u) ab = 1;+ s_abort = ab;+ }+ __syncthreads();+ __threadfence();+ return s_abort == 0;+ #endif+ }+ // STRATA_HC_PERSIST=3 (a reference only, slow): every dot and sum accumulated in double+ __device__ __forceinline__ double dot8d(const uint4 w, const float* x) {+ double acc = 0.0;+ const unsigned u[4] = {w.x, w.y, w.z, w.w};+ #pragma unroll+ for (int q = 0; q < 4; ++q) {+ acc += (double) __uint_as_float(u[q] << 16) * (double) x[2 * q];+ acc += (double) __uint_as_float(u[q] & 0xffff0000u) * (double) x[2 * q + 1];+ }+ return acc;+ }+ template <int T, bool DP = false>+ __global__ void __launch_bounds__(THREADS) persist_kernel(PArgs f) {+ using A = typename std::conditional<DP, double, float>::type;+ __shared__ __align__(16) float xs[T][SL];+ __shared__ __align__(16) float lo[T][LR];+ __shared__ float s_rs[T][HC];+ __shared__ float g[T][HC][UPC];+ const int tid = threadIdx.x, lane = tid & 31, b = blockIdx.x;+ const FusedGrArgs& a0 = f.a[0];+ // (A) block b: columns [128 b, 128 b + 128) of D (one stream): R' * w_norm into LDS, the slice's sum of squares, and+ // the UNSCALED partial dots of the 320 down + 4 inject rows (w . xn = sum_c rs[c] (w[:, c] . x'[c]))+ const int c_b = b / SPS, col0 = b * SL;+ if (tid < T * (SL / 4)) {+ const int k = tid / (SL / 4), o4 = tid - k * (SL / 4);+ const FusedGrArgs& a = f.a[k];+ const int i = col0 + o4 * 4, d = i - c_b * N;+ float4 r = *reinterpret_cast<const float4*>(a.R + i);+ if (a.apply) {+ const float gw = 2.0f * sigmoidf_(a.inj_prev[c_b] / (float) HC);+ const float4 bo = *reinterpret_cast<const float4*>(a.bo_prev + d);+ r.x = fmaf(bo.x, gw, r.x); r.y = fmaf(bo.y, gw, r.y); r.z = fmaf(bo.z, gw, r.z); r.w = fmaf(bo.w, gw, r.w);+ }+ const float4 gn = *reinterpret_cast<const float4*>(a.w_norm + i);+ if constexpr (DP) {+ double ss = (double) r.x * r.x + (double) r.y * r.y + (double) r.z * r.z + (double) r.w * r.w;+ #pragma unroll+ for (int o = 16; o > 0; o >>= 1) ss += __shfl_xor(ss, o);+ if (lane == 0) f.sspd[k * NB + b] = ss;+ } else {+ const float ss = warp_sum(r.x * r.x + r.y * r.y + r.z * r.z + r.w * r.w); // warp k = token k+ if (lane == 0) f.ssp[k * NB + b] = ss;+ }+ *reinterpret_cast<float4*>(&xs[k][o4 * 4]) = make_float4(r.x * gn.x, r.y * gn.y, r.z * gn.z, r.w * gn.w);+ }+ __syncthreads();+ const int hw = tid >> 4, hl = tid & 15;+ const bool inj = a0.w_inject != nullptr;+ const int nrows = inj ? NR : LR;+ #pragma unroll 3+ for (int r = hw; r < nrows; r += 16) {+ const uint16_t* wrow = r < LR ? a0.w_down + (size_t) r * D : a0.w_inject + (size_t) (r - LR) * D;+ const uint4 w = __ldg(reinterpret_cast<const uint4*>(wrow + col0) + hl);+ A acc[T];+ #pragma unroll+ for (int k = 0; k < T; ++k) {+ A v;+ if constexpr (DP) v = dot8d(w, &xs[k][hl * 8]); else v = dot8(w, &xs[k][hl * 8]);+ #pragma unroll+ for (int o = 8; o > 0; o >>= 1) v += __shfl_xor(v, o);+ acc[k] = v;+ }+ #pragma unroll+ for (int k = 0; k < T; ++k)+ if (hl == k) {+ if constexpr (DP) f.partd[((size_t) k * NR + r) * NB + b] = acc[k];+ else f.part[((size_t) k * NR + r) * NB + b] = (float) acc[k];+ }+ }+ if (!grid_sync(f.bar, f.timeout_ticks)) return;+ // (B) rs per stream (every block), then rows (k, r) = b + 80 j reduced to lo / inject by the blocks in turn+ if (tid < T * HC) {+ const int k = tid / HC, c = tid - k * HC;+ float rr;+ if constexpr (DP) {+ double sd = 0.0;+ for (int j = 0; j < SPS; ++j) sd += f.sspd[k * NB + c * SPS + j];+ rr = (float) (1.0 / sqrt(sd / (double) N + (double) f.a[k].eps));+ } else {+ float s = 0.0f;+ for (int j = 0; j < SPS; ++j) s += f.ssp[k * NB + c * SPS + j];+ rr = rsqrtf(s / (float) N + f.a[k].eps);+ }+ s_rs[k][c] = rr;+ if (b == 0) f.a[k].rs[c] = rr;+ }+ __syncthreads();+ for (int idx = b + NB * tid; idx < T * nrows; idx += NB * THREADS) {+ const int k = idx / nrows, r = idx - k * nrows;+ A sum = 0;+ if constexpr (DP) {+ const double* p1 = f.partd + ((size_t) k * NR + r) * NB;+ for (int c = 0; c < HC; ++c) {+ double p = 0.0;+ for (int j = 0; j < SPS; ++j) p += p1[c * SPS + j];+ sum += (double) s_rs[k][c] * p;+ }+ } else {+ const float4* p4 = reinterpret_cast<const float4*>(f.part + ((size_t) k * NR + r) * NB);+ #pragma unroll+ for (int c = 0; c < HC; ++c) {+ float p = 0.0f;+ #pragma unroll+ for (int j = 0; j < SPS / 4; ++j) {+ const float4 v = p4[c * (SPS / 4) + j];+ p += (v.x + v.y) + (v.z + v.w);+ }+ sum = fmaf(s_rs[k][c], p, sum);+ }+ }+ if (r < LR) {+ const A x = sum / (A) HC;+ f.a[k].lo[r] = (float) (x / ((A) 1 + (DP ? (A) exp((double) -x) : (A) __expf((float) -x))));+ } else {+ f.a[k].inject_out[r - LR] = (float) sum;+ }+ }+ if (!grid_sync(f.bar, f.timeout_ticks)) return;+ for (int idx = tid; idx < T * LR; idx += THREADS) {+ const int k = idx / LR, r = idx - k * LR;+ lo[k][r] = f.a[k].lo[r];+ }+ __syncthreads();+ // (C) the up projection, a thread pair per row (block b: columns 32 b .. 32 b + 31 of all four streams), then the+ // epilogue as gr_up_kernel+ {+ const int d0 = b * UPC;+ const int ur = tid >> 1, uh = tid & 1, uc = ur / UPC, udd = ur - uc * UPC, ui = uc * N + d0 + udd;+ const uint4* w4u = reinterpret_cast<const uint4*>(a0.w_up + (size_t) ui * LR) + uh * 20;+ A v[T];+ #pragma unroll+ for (int k = 0; k < T; ++k) v[k] = 0;+ #pragma unroll 1+ for (int j0 = 0; j0 < 20; j0 += 5) { // five chunks a batch (all twenty at once costs registers)+ uint4 w[5];+ #pragma unroll+ for (int q = 0; q < 5; ++q) w[q] = __ldg(w4u + j0 + q);+ #pragma unroll+ for (int q = 0; q < 5; ++q)+ #pragma unroll+ for (int k = 0; k < T; ++k) {+ if constexpr (DP) v[k] += dot8d(w[q], &lo[k][uh * 160] + (j0 + q) * 8);+ else v[k] += dot8(w[q], &lo[k][uh * 160] + (j0 + q) * 8);+ }+ }+ float acc[T];+ #pragma unroll+ for (int k = 0; k < T; ++k) acc[k] = (float) (v[k] + __shfl_xor(v[k], 1));+ if (uh == 0) {+ const float wn = a0.w_norm[ui];+ #pragma unroll+ for (int k = 0; k < T; ++k) {+ const FusedGrArgs& a = f.a[k];+ float rv = a.R[ui];+ if (a.apply) {+ rv = fmaf(a.bo_prev[d0 + udd], 2.0f * sigmoidf_(a.inj_prev[uc] / (float) HC), rv);+ a.R_out[ui] = rv;+ }+ g[k][uc][udd] = rv * wn * s_rs[k][uc] * sigmoidf_(acc[k]);+ }+ }+ }+ __syncthreads();+ for (int idx = tid; idx < T * UPC; idx += THREADS) {+ const int k = idx / UPC, col = idx - k * UPC;+ float s = 0.0f;+ #pragma unroll+ for (int c = 0; c < HC; ++c) s += g[k][c][col];+ f.a[k].mixed[b * UPC + col] = s / (float) HC;+ }+ }+ struct DevState {+ float* part = nullptr;+ float* ssp = nullptr;+ double* partd = nullptr;+ double* sspd = nullptr;+ unsigned* bar = nullptr;+ int max_t = 0; // 0: off on this card+ bool tried = false;+ };+ DevState g_dev[64];+ using KFn = void (*)(PArgs);+ template <bool DP>+ KFn kernel_t(int T) {+ switch (T) {+ case 1: return persist_kernel<1, DP>;+ case 2: return persist_kernel<2, DP>;+ case 3: return persist_kernel<3, DP>;+ case 4: return persist_kernel<4, DP>;+ case 5: return persist_kernel<5, DP>;+ case 6: return persist_kernel<6, DP>;+ case 7: return persist_kernel<7, DP>;+ case 8: return persist_kernel<8, DP>;+ }+ return nullptr;+ }+ bool dp_on() {+ static const bool dp = [] { const char* v = std::getenv("STRATA_HC_PERSIST"); return v != nullptr && std::atoi(v) == 3; }();+ return dp;+ }+ KFn kernel_for(int T) { return dp_on() ? kernel_t<true>(T) : kernel_t<false>(T); }+ // s_memrealtime runs at 100 MHz on RDNA: a barrier gives up after 2 s in a read (never in a working run; it only turns a+ // hang into a wrong read) and after 50 ms in the start-up check (where a timeout means "not resident here")+ constexpr unsigned long long kTicksRun = 200000000ULL, kTicksCheck = 5000000ULL;+ void launch(const FusedGrArgs* a, int n_tok, const DevState& st, cudaStream_t s, unsigned long long ticks) {+ PArgs f;+ for (int t = 0; t < n_tok; ++t) f.a[t] = a[t];+ f.part = st.part; f.ssp = st.ssp; f.bar = st.bar; f.partd = st.partd; f.sspd = st.sspd; f.timeout_ticks = ticks;+ kernel_for(n_tok)<<<NB, THREADS, 0, s>>>(f);+ }+ bool env_on() {+ static const bool on = [] { const char* v = std::getenv("STRATA_HC_PERSIST"); return v != nullptr && std::atoi(v) != 0; }();+ return on;+ }+ } // namespace hcp+ } // namespace++ void fused_gr_persist_init() {+ if (!hcp::env_on()) return;+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 64 || hcp::g_dev[dev].tried) return;+ hcp::DevState& st = hcp::g_dev[dev];+ st.tried = true;+ cudaDeviceProp prop{};+ cudaGetDeviceProperties(&prop, dev);+ if (prop.warpSize != 32) {+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: the kernel is written for 32-lane waves "+ "(this card runs %d)\n", dev, prop.warpSize);+ return;+ }+ if (strata::kernels::gfx_arch_is_gfx12(prop.gcnArchName) || strata::kernels::gfx_arch_is_gfx11_wmma(prop.gcnArchName)) {+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: gfx11 / gfx12 have no s_memrealtime for the barrier "+ "timeout (written and measured for gfx1030; an R9700 saw no gain from it, #2226)\n", dev);+ return;+ }+ const int mp = prop.multiProcessorCount; // only for the confirmation line (WGPs on RDNA)+ bool ok = cudaMalloc((void**) &st.part, (size_t) kFusedGrMaxT * hcp::NR * hcp::NB * sizeof(float)) == cudaSuccess &&+ cudaMalloc((void**) &st.ssp, (size_t) kFusedGrMaxT * hcp::NB * sizeof(float)) == cudaSuccess &&+ cudaMalloc((void**) &st.bar, 64 * sizeof(unsigned)) == cudaSuccess &&+ cudaMalloc((void**) &st.partd, (size_t) kFusedGrMaxT * hcp::NR * hcp::NB * sizeof(double)) == cudaSuccess &&+ cudaMalloc((void**) &st.sspd, (size_t) kFusedGrMaxT * hcp::NB * sizeof(double)) == cudaSuccess &&+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned)) == cudaSuccess;+ if (!ok) {+ cudaGetLastError();+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: workspace allocation failed\n", dev);+ return;+ }+ // the check: random weights and inputs with the pending write and the inject rows, against the default read; the+ // persistent read sums in another order, so it is bounded, not bitwise. Every T is RUN with a 50 ms barrier timeout; a+ // timeout means "not resident here" and larger T keep the default read. No occupancy API in front of it: it promised+ // more than the card holds on 30 WGPs under HSA_CU_MASK (a hang without the timeout) and less than it holds with+ // the HIP 7.1 runtime (1 block per WGP on 40 WGPs, which turned the read off although it runs resident).+ std::mt19937 rng(11);+ std::normal_distribution<float> nd(0.f, 1.f);+ auto bf = [](float x) { uint32_t u; std::memcpy(&u, &x, 4); return (uint16_t) ((u + 0x7fff + ((u >> 16) & 1)) >> 16); };+ const int TM = kFusedGrMaxT;+ std::vector<float> R((size_t) TM * D), bo((size_t) TM * N), inj((size_t) TM * HC), wn(D);+ for (auto& x : R) x = nd(rng);+ for (auto& x : bo) x = 0.5f * nd(rng);+ for (auto& x : inj) x = nd(rng);+ for (auto& x : wn) x = 1.0f + 0.3f * nd(rng);+ std::vector<uint16_t> wd((size_t) LR * D), wu((size_t) D * LR), wi((size_t) HC * D);+ for (auto& x : wd) x = bf(0.02f * nd(rng));+ for (auto& x : wu) x = bf(0.05f * nd(rng));+ for (auto& x : wi) x = bf(0.02f * nd(rng));+ float *dR, *dbo, *dinj, *dwn, *dxn, *dlo, *drs, *dio, *dmix;+ uint16_t *dwd, *dwu, *dwi;+ cudaMalloc((void**) &dR, R.size() * 4); cudaMalloc((void**) &dbo, bo.size() * 4);+ cudaMalloc((void**) &dinj, inj.size() * 4); cudaMalloc((void**) &dwn, wn.size() * 4);+ cudaMalloc((void**) &dxn, (size_t) TM * D * 4); cudaMalloc((void**) &dlo, (size_t) TM * LR * 4);+ cudaMalloc((void**) &drs, (size_t) TM * HC * 4); cudaMalloc((void**) &dio, (size_t) TM * HC * 4);+ cudaMalloc((void**) &dmix, (size_t) TM * N * 4);+ cudaMalloc((void**) &dwd, wd.size() * 2); cudaMalloc((void**) &dwu, wu.size() * 2); cudaMalloc((void**) &dwi, wi.size() * 2);+ cudaMemcpy(dbo, bo.data(), bo.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dinj, inj.data(), inj.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dwn, wn.data(), wn.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dwd, wd.data(), wd.size() * 2, cudaMemcpyHostToDevice);+ cudaMemcpy(dwu, wu.data(), wu.size() * 2, cudaMemcpyHostToDevice);+ cudaMemcpy(dwi, wi.data(), wi.size() * 2, cudaMemcpyHostToDevice);+ double worst = 0.0;+ int bad_t = 0, timeouts = 0;+ // 0: the read agrees with the default read; 1: a barrier timed out (not resident); 2: wrong or a device error+ auto check = [&](int T) -> int {+ FusedGrArgs a[kFusedGrMaxT];+ for (int t = 0; t < T; ++t) {+ a[t] = FusedGrArgs{};+ a[t].R = dR + (size_t) t * D; a[t].R_out = dR + (size_t) t * D; a[t].apply = true;+ a[t].bo_prev = dbo + (size_t) t * N; a[t].inj_prev = dinj + (size_t) t * HC;+ a[t].w_norm = dwn; a[t].w_down = dwd; a[t].w_up = dwu; a[t].w_inject = dwi; a[t].eps = 1e-6f;+ a[t].lo = dlo + (size_t) t * LR; a[t].rs = drs + (size_t) t * HC;+ a[t].inject_out = dio + (size_t) t * HC; a[t].mixed = dmix + (size_t) t * N;+ }+ std::vector<float> out[2];+ for (int which = 0; which < 2; ++which) {+ cudaMemcpy(dR, R.data(), R.size() * 4, cudaMemcpyHostToDevice);+ cudaMemset(dmix, 0xff, (size_t) TM * N * 4);+ cudaMemset(dio, 0xff, (size_t) TM * HC * 4);+ cudaDeviceSynchronize();+ if (which == 0) fused_gr_read_multi(a, T, dxn, nullptr);+ else hcp::launch(a, T, st, nullptr, hcp::kTicksCheck);+ if (cudaDeviceSynchronize() != cudaSuccess) { cudaGetLastError(); return 2; }+ if (which == 1) {+ unsigned abort_word = 0;+ cudaMemcpy(&abort_word, st.bar + 48, sizeof abort_word, cudaMemcpyDeviceToHost);+ if (abort_word != 0u) {+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned));+ cudaDeviceSynchronize();+ return 1;+ }+ }+ out[which].resize((size_t) T * (N + HC + D));+ cudaMemcpy(out[which].data(), dmix, (size_t) T * N * 4, cudaMemcpyDeviceToHost);+ cudaMemcpy(out[which].data() + (size_t) T * N, dio, (size_t) T * HC * 4, cudaMemcpyDeviceToHost);+ cudaMemcpy(out[which].data() + (size_t) T * (N + HC), dR, (size_t) T * D * 4, cudaMemcpyDeviceToHost);+ }+ double mref = 0.0, md = 0.0;+ bool finite = true;+ for (size_t i = 0; i < out[0].size(); ++i) {+ finite = finite && std::isfinite(out[1][i]);+ mref = std::fmax(mref, std::fabs((double) out[0][i]));+ md = std::fmax(md, std::fabs((double) out[1][i] - out[0][i]));+ }+ const double rel = md / (mref > 0 ? mref : 1.0);+ worst = std::fmax(worst, rel);+ return finite && rel < 1e-4 ? 0 : 2;+ };+ int max_t = 0;+ for (int T = 1; T <= kFusedGrMaxT && ok; ++T) {+ const int r = check(T);+ if (r == 1) ++timeouts; else if (r == 2) { bad_t = T; ok = false; }+ if (r != 0) break;+ max_t = T;+ }+ cudaFree(dR); cudaFree(dbo); cudaFree(dinj); cudaFree(dwn); cudaFree(dxn); cudaFree(dlo); cudaFree(drs);+ cudaFree(dio); cudaFree(dmix); cudaFree(dwd); cudaFree(dwu); cudaFree(dwi);+ cudaMemset(st.bar, 0, 64 * sizeof(unsigned));+ cudaDeviceSynchronize();+ cudaGetLastError();+ if (!ok) {+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: the check failed at %d tokens "+ "(max relative difference %.3g)\n", dev, bad_t, worst);+ return;+ }+ if (max_t == 0) {+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1 is off here: 80 blocks are not resident at once on %d "+ "multiprocessors (the check timed out)\n", dev, mp);+ return;+ }+ st.max_t = max_t;+ std::fprintf(stderr, "strata hc: CUDA%d: STRATA_HC_PERSIST=1: the verifier's hyper-connection read runs as one "+ "persistent kernel (80 blocks on %d multiprocessors, up to %d tokens; %d token counts timed out in "+ "the check; checked against the default read, max relative difference %.2g)\n", dev, mp, max_t,+ timeouts, worst);+ }++ bool fused_gr_read_persist(const FusedGrArgs* a, int n_tok, void* stream) {+ if (!hcp::env_on() || n_tok < 1) return false;+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 64) return false;+ const hcp::DevState& st = hcp::g_dev[dev];+ if (n_tok > st.max_t) return false;+ if (a[0].q8_down != nullptr || a[0].q8_up != nullptr) return false; // STRATA_HC_Q8: the default read+ for (int t = 0; t < n_tok; ++t)+ if (a[t].q8_mixed != nullptr) return false; // STRATA_QFUSE: the default read+ hcp::launch(a, n_tok, st, (cudaStream_t) stream, hcp::kTicksRun);+ return true;+ }+ #else+ void fused_gr_persist_init() {}+ bool fused_gr_read_persist(const FusedGrArgs*, int, void*) { return false; }+ #endif+} // namespace strata::kernelsdiff --cc src/kernels/cuda/qsa_select.cuindex 02e6c750a,5fd67f95b..75bd1fa6e--- a/src/kernels/cuda/qsa_select.cu+++ b/src/kernels/cuda/qsa_select.cu@@@ -887,114 -868,106 +887,214 @@@ __device__ __forceinline__ void block_s}}+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit, QsaScoreRows srow = {}) {+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);+}++// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).+struct ScoreSlotKeys {+ const float* pooled[MQ];+ const float* dead[MQ];+};+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit) {+ const unsigned z = blockIdx.y;+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory+ const float* pooled = keys.pooled[0];+ const float* dead = keys.dead[0];+#pragma unroll+ for (unsigned i = 1; i < MQ; ++i)+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,+ QsaScoreRows{});+}++// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.+template <int NQ>+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out) {+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];+ if (threadIdx.x < nq) {+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];+ }+ __syncthreads();+ int64_t top = 0;+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)+ const int lane = threadIdx.x & 31;+ float4 qr[NQ][IDX_HEADS];+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi)+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)+ : make_float4(0.f, 0.f, 0.f, 0.f);+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi) {+ if (qi >= nq) break;+ const int64_t n_bid = s_nbid[qi];+ if (b > n_bid) continue;+ const float4 k4 = (b == n_bid) ? kd : kp;+ float d[IDX_HEADS];+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float4 q4 = qr[qi][h];+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ }+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head+ float r2[2];+#pragma unroll+ for (int i = 0; i < 2; ++i)+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);+ r += __shfl_xor_sync(0xffffffffu, r, 4);+ r += __shfl_xor_sync(0xffffffffu, r, 2);+ r += __shfl_xor_sync(0xffffffffu, r, 1);+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel+ const float s1 = __shfl_sync(0xffffffffu, r, 8);+ const float s2 = __shfl_sync(0xffffffffu, r, 16);+ const float s3 = __shfl_sync(0xffffffffu, r, 24);+ if (lane == 0) {+ float score = 0.0f;+ score += r > 0.0f ? r : 0.0f;+ score += s1 > 0.0f ? s1 : 0.0f;+ score += s2 > 0.0f ? s2 : 0.0f;+ score += s3 > 0.0f ? s3 : 0.0f;+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;+ out[qi * max_blocks + b] = score;+ }+ }+ }+}++ #if defined(__HIPCC__)+ // Optional gfx1100 port of PR1825 (14bf3215): same FP32 expression and reduction tree.+ // Decode and the original top-k stay unchanged. No additional persistent weight/state allocation.+ constexpr int TS_Q = 64; // queries per CTA (8 per wave, four lanes per query)+ constexpr int TS_KB = 32; // key blocks per shared-memory tile+ constexpr int TS_BR = 512; // key blocks per CTA++ __device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression+ }++ __global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int64_t nq,+ int64_t max_blocks, int64_t reach,+ float* __restrict__ out) {+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];+ __shared__ float so[TS_Q][TS_KB + 1];+ __shared__ int64_t s_nbid[TS_Q];+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer+ // shared-memory loads per FMA. The MIO-pipe motivation was measured by PR1825 on NVIDIA.+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};+ const int t = threadIdx.x, lane = t & 31;+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;+ float4 qv[IDX_HEADS][8];+ {+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ #pragma unroll+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];+ }+ __syncthreads();+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;+ int64_t b_hi = b_lo + TS_BR;+ if (b_hi > reach) b_hi = reach;+ if (b_hi > top) b_hi = top;+ if (b_hi > max_blocks) b_hi = max_blocks;+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);+ __syncthreads(); // the previous tile's reads of ks and so are done+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];+ __syncthreads();+ for (int j = 0; j < nb; ++j) {+ float4 kk[8];+ #pragma unroll+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];+ float e[IDX_HEADS];+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) ++ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) ++ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) ++ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)+ }+ if (j4 == 0) {+ float score = 0.0f;+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;+ so[ql][j] = score;+ }+ }+ __syncthreads();+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {+ const int q = i / nb, j = i % nb;+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];+ }+ }+ }++ // The exact scorer runs on gfx1100 (the PR's own card) when STRATA_SCORES_THREAD=1, and on gfx12 (RDNA4) unless STRATA_SCORES_THREAD=0:+ // there it is bitwise equal to the warp scorer on an R9700 (qsa_scores_gfx1100_parity --full, 35 cases; exact_pp residual dumps at 70K and+ // 131K tokens) and 3.6-4.4x faster in the scorer stage, +8% (131K) to +15% (230K) on a prompt read. Other wave32 parts have no measurement.+ // Returns 0 (not offered), 1 (gfx1100: only on request), 2 (gfx12: the default).+ int scores_thread_arch() {+ int device = -1;+ if (cudaGetDevice(&device) != cudaSuccess) { cudaGetLastError(); return 0; }+ static thread_local int cached_device = -1;+ static thread_local int arch = 0;+ if (device != cached_device) {+ cudaDeviceProp properties{};+ arch = 0;+ if (cudaGetDeviceProperties(&properties, device) == cudaSuccess)+ arch = gfx_arch_is_gfx12(properties.gcnArchName) ? 2 : gfx_arch_is(properties.gcnArchName, "gfx1100") ? 1 : 0;+ cached_device = device;+ cudaGetLastError();+ }+ return arch;+ }+ #endif+#if !defined(__HIPCC__)// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to@@@ -1516,14 -1459,10 +1645,15 @@@ void qsa_block_topk(const float* scoresconst int64_t reach = counted && active_blocks < max_blocks ? active_blocks : max_blocks;const int64_t fit = (int64_t) TK_T * (counted ? TK_PER_MAX : TK_PER);#if defined(__HIPCC__)- constexpr int64_t kRegMinBlocks = 7168; // gfx1201: below ~28K cells the 1,024-thread kernel's fixed cost loses to the ref+ // gfx1201: below ~28K cells (7,168 blocks) the 1,024-thread kernel's fixed cost loses to the ref. STRATA_TOPK_REG_MIN_BLOCKS+ // moves the line (0: the register kernel always); gfx1151's table sets 4096, the crossover with the cooperative scan+ static const int64_t kRegMinBlocks = [] {+ const char* v = std::getenv("STRATA_TOPK_REG_MIN_BLOCKS");+ return v != nullptr ? (int64_t) std::atoll(v) : (int64_t) 7168;+ }();const bool too_small = counted && reach < kRegMinBlocks;- if (old || too_small || reach > fit) {+ // campaign3: `reach > fit` no longer forces the ref radix - the wide kernel below takes it (same ids).+ if (old || too_small) {#elseconstexpr int64_t kWideMinBlocks = 4608; // RTX 3060: a prompt batch below ~18K cells is faster on the refconst bool too_small = reach > fit && active_blocks > 0 && active_blocks < kWideMinBlocks;