Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group sm75 (f-145-pp-sm75)
Niko1221df6262f270fe
diff --cc CMakeLists.txtindex 5b6854f70,ba4556c1a..d900326e3--- a/CMakeLists.txt+++ b/CMakeLists.txt@@@ -432,35 -408,13 +432,41 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)+ # STRATA_QSA_SLOTS: a batch window's scores, top-k and attention as one launch over rows of different slots+ # against one call per row, bitwise (synthetic, no model)+ add_executable(qsa_slots_parity src/kernels/qsa_slots_parity.cpp)+ target_include_directories(qsa_slots_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_slots_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_slots_parity COMMAND qsa_slots_parity)+ add_test(NAME qsa_slots_parity_lanecell COMMAND qsa_slots_parity)+ set_tests_properties(qsa_slots_parity qsa_slots_parity_lanecell PROPERTIES SKIP_RETURN_CODE 77)+ set_tests_properties(qsa_slots_parity_lanecell PROPERTIES ENVIRONMENT "STRATA_ATTN_LANECELL=1")+ # fork: the Daily's decode attention (STRATA_ATTN_DAILY=1: PR #540's chunk kernel on every card and the bounded+ # merge) against upstream's kernels, bitwise; the production entries checked with the variable off and on+ add_executable(qsa_merge_daily_parity src/kernels/qsa_merge_daily_parity.cpp)+ target_include_directories(qsa_merge_daily_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_merge_daily_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_merge_daily_parity_default COMMAND qsa_merge_daily_parity)+ add_test(NAME qsa_merge_daily_parity_daily COMMAND qsa_merge_daily_parity)+ set_tests_properties(qsa_merge_daily_parity_default PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=0"+ SKIP_RETURN_CODE 77)+ set_tests_properties(qsa_merge_daily_parity_daily PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=1"+ SKIP_RETURN_CODE 77)+ # the prompt path's block scores on the thread-per-head kernel (Turing) against the warp kernel, bitwise+ add_executable(qsa_scores_parity src/kernels/qsa_scores_parity.cpp)+ target_include_directories(qsa_scores_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_scores_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_scores_parity COMMAND qsa_scores_parity)+ set_tests_properties(qsa_scores_parity PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT "STRATA_SCORES_THREAD=1")endif()+ # QSAROWS (docs/QSAROWS.md): what the T rows of a verification window cost in a QSA layer's decode kernels at a long+ # context (block scores, top-k, attention over each row's selection, indexer append), row by row; GPU, synthetic, no+ # model. Not in the default build: `cmake --build <dir> --target qsa_rows_bench`.+ if(STRATA_ENABLE_CUDA)+ add_executable(qsa_rows_bench EXCLUDE_FROM_ALL src/kernels/qsa_rows_bench.cpp)+ target_include_directories(qsa_rows_bench PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_rows_bench PRIVATE strata_kernels CUDA::cudart)+ endif()# the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a# scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a# 262144-cell capacity (the top-k dispatch follows the capacity) and exits non-zero on a failed gatediff --cc docs/DETAILS.mdindex 83c956e1c,9c1ad3a23..5c5724f12--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@@ -2217,26 -2046,21 +2217,45 @@@ signal, early cleanup and `--errors` ermode is the negative control under memcheck: an eight-second watchdog releasesthe original launch/service deadlock and the test fails.+## CUDA opt-in switches measured for 0.1.45 (RTX 3060 and RTX A4000, sm_86; all off unless you set them)++Ported from open pull requests and A/B'd as whole engine runs against the default (a fresh engine per side, IQ3_XXS, 5 pairs, medians;+bit-exact means the 10-prompt + 4K/20K identity gate with the switch on equals the gate with it off). Decode after a 30K-token prompt+unless noted; the gain is small on these two cards and larger on the Turing cards the authors measured.++| Switch | What | Answers | RTX 3060 / A4000 |+|---|---|---|---|+| `STRATA_ATTN_DAILY=1` (#2075) | decode attention: two cells per step, a merge that skips empty chunks | bit-exact | +3.2% (5/5) / +0.7% |+| `STRATA_QSA_ROWS=1` (#2059) | block scores of a verify window, queries in registers ([QSAROWS.md](QSAROWS.md)) | bit-exact | -0.3% / +0.3% |+| `STRATA_EXPDEC=1` (#2058) | gate/up of the window's resident experts with the q8_1 rows in shared memory ([EXPERTDEC.md](EXPERTDEC.md)) | bit-exact | +0.5% / +1 to +5% (short context) |+| `STRATA_SIDE_MIXER=1/2` (#1889) | the mixer's input-only work on graph branches | bit-exact | +0.5% / -0.6% (short context) |+| `STRATA_B6_MMVQ_ROWS=1` (#1418) | exact multi-column MMVQ reusing the activation fragment across rows (K-quants, IQ4_XS) | bit-exact | +0.2..1% / +0.5..2% |+| `STRATA_QSA_SPLIT_ATTN=1` (#1888) | decode attention split across blocks on the int8 mma path (sm_80+, int8 KV) | rounding-level (teacher-forced KL 0.0016 against a floor of 0.0012) | +2.8% / +3.5% |+| `STRATA_HC_UPMIX=1` (#1370) | the hyper-connection up projection and `gr_mix_r` in one kernel (sm_80+) | bit-exact here (gate identical) | prompt +0.6% / +0.4% |+| `STRATA_PROMPT_ATTN_IMMA=1` (#1367) | int8-KV prompt attention on INT8 tensor cores | rounding-level (KL at the floor) | prompt read +2.5% |+| `STRATA_HC_REQ8=1` (#1262) | the hyper-connection projections as int8 + a scale per 32 (more expert-cache slots) | rounding-level (KL 0.0018) | decode -0.2% / +2.2% |+| `STRATA_QUANT_GATHER=1` (#1368) | turns the once-per-token MoE input quantization back to the per-row gather | n/a | no difference measured |+| `STRATA_PREFILL_GROUP_GATHER=0` (#1107) | turns the short-prompt grouped gather off | n/a | on is 0.6-1.3% faster |++`STRATA_DEQUANT_WIDE` (the wide dequantization of Q4_K / Q5_K / Q6_K / IQ4_XS for the prompt's dense weights) is now on by default on+sm_86 as well as sm_61: bit-exact, and the time to the first token of a 150 / 300 / 450-token prompt fell 11-15% on the RTX 3060 and 6-10% on the RTX A4000 (5 of 5 pairs each).+`STRATA_DEQUANT_WIDE=0` turns it off.+++ ### RTX 20 (Turing, sm_75) kernel series (opt-in, 0.1.45)++ Kernels by rafatxf (#2069-#2072, #2093, #2220-#2222), imanu86 (#2081) and xrip (#1821) for compute capability 7.5. They were measured+ on one RTX 2080 Ti (decode +5..6 % at 4K-32K, prompts at depth up to +30 %, per the PRs) and are bitwise equal to the kernels they+ replace, but nobody on the team has an sm_75 card to confirm them, so they are off until asked for:++ `STRATA_SM75_PERF=1` turns on every bitwise Turing kernel below on a 7.5 card (no other card is ever affected). Each has its own+ variable, which wins in both directions (`=0` off, `=1` on): `STRATA_SM75_IL` (interleaved small-batch matvec table),+ `STRATA_SM75_EXP` (expert layout mode 4), `STRATA_GR_UP75` / `STRATA_GR_DOWN_TG` (hyper-connection read), `STRATA_SCORES_HALVE`+ and `STRATA_TOPK_SPLIT` (decode QSA selection), `STRATA_TOPK_SAMPLE` (prompt top-k, one read of the scores), `STRATA_ATTN_V3`+ (decode attention), `STRATA_Q2_WARP` (Q2_0 matvec), `STRATA_FUSE_WAIT=2` and `STRATA_QIL` (fewer launches per decode layer),+ `STRATA_TC_SM75` (BF16 dequant, 16-byte stores), `STRATA_PA_V1_PREFETCH` (prompt attention v1 prefetch),+ `STRATA_SCORES_THREAD` (#1825: the prompt path's thread-per-head block scorer, bitwise the warp kernel's scores; `=1` takes it on any+ CUDA card, where it measured +1..3 % prompt speed on a Tesla P100 at 30-60K tokens, so it is not a default there).++ Options that change answers (other summation order) are separate and explicit: `STRATA_SELECT_SIMT=1|2` (FP32 tiled block scorer,+ KL 0.01 against the warp scorer per #2071), `STRATA_PA_V2T=1` (v2 prompt attention, #2092/#2146), `STRATA_PF_FUSED=1+ STRATA_PF_FUSED_SM75=1` (fused int8 prompt experts for the Q2_0 pack, #2145; only on a 7.5 card, sm_80+ is unchanged), `STRATA_BF16_TC=1` with `STRATA_XN_F16` (#2091).diff --cc src/core/verify.cppindex 0847405e9,08d368e83..d760698ee--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@@ -1256,30 -1198,7 +1306,30 @@@ bool Verifier::record_window(int T, cudfloat* hb = h_L_ + (size_t) gi * MT * C;float* gate = gate_L_ + (size_t) gi * MT * HV;float* beta = beta_L_ + (size_t) gi * MT * HV;+ if (sidem) { // the gates (a few blocks, latency-bound) beside the projections, joined below+ if (!branch(mfork_, cs, side_, "fork")) return false;+ gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,+ (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N,+ (int) HV, n, side_);+ if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the GDN gates' branch could not join"; return false; }+ }- if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs);+ if (!q8_attn) quant_in(xm, (int) N);+ if (side_proj) { // the gate projection (compute-bound for i-quants) beside qkv and the conv+ // the branch reads F4's interleaved copy too (written on cs before the fork; bitwise native_mmvq's+ // output), and qkv is recorded before the branch's launch, so its grid is dispatched first and the+ // conv / recurrence that wait for it start sooner (z is read only after the recurrence)+ const bool g_il = g_mmvq_il() && strata::kernels::native_mmvq_il_supported(wg->native_type, n, (int) ZV);+ if (g_il && !il_ready) {+ strata::kernels::native_q8_1_interleave(xq_, xil_, (int) N, n, cs);+ il_ready = true;+ }+ if (!branch(pfork_, cs, side2_, "fork")) return false;+ mm(wqkv, qkv + (size_t) tb * C, (int) N, (int) C);+ if (g_il) strata::kernels::native_mmvq_il(wg->native_type, wg->native_data, xq_, xil_, z_ + (size_t) tb * ZV,+ (int) N, (int) ZV, n, side2_);+ else native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, side2_);+ if (cudaEventRecord(pjoin_, side2_) != cudaSuccess) { err = "verify: the gate projection's branch could not join"; return false; }+ }if (br) { // STRATA_DF_BRANCH: a/b and z beside q/k/v and the conv; all of them read only this layer's inputfork(0);gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,@@@ -1407,18 -1299,14 +1457,18 @@@float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;// the per-token GEMVs / norms / RoPEs / copies of this layer as one launch over the// window's rows each - row-wise identical arithmetic (STRATA_DEC_BATCH=0: token by token)- // STRATA_DF_QB_Q4 (opt-in: STRATA_DF_QB_Q4=1): a Q4_0 KV cache too - its one per-token extra, the query's rotation, is- // row-wise and the batched path rotates (270650e); 0: Q4_0 token by token, as before+ // STRATA_DF_QB_Q4 (default on since 0.1.45; STRATA_DF_QB_Q4=0 gives the old token-by-token path): a Q4_0 KV cache+ // takes the batched window too - its one per-token extra, the query's rotation, is row-wise and the batched+ // path rotates (270650e). The generated ids are identical (p620 gfx1201, A4000 sm_86, P100 sm_60, short and+ // long prompts), and the 6-token verify graph is 1950 nodes instead of 2610 (#2403: an RX 6900 XT faulted in+ // the first verify graph launch with --kv q4_0 + KV streaming + MTP; whether the graph size is the cause is+ // not confirmed)static const bool qb_q4 = [] {const char* e = std::getenv("STRATA_DF_QB_Q4");- return e != nullptr && std::atoi(e) != 0;+ return e == nullptr || std::atoi(e) != 0;}();const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && (!st.kv_q4 || qb_q4);- if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs);+ if (!q8_attn) quant_in(xm, (int) N);// STRATA_DF_BRANCH: the query side (q and its norm/RoPE/rotation on side 0, the indexer query on side 1)// beside the K/V side; the indexer query joins before the block scores, the query before attentionconst bool qbr = br && qb;@@@ -1965,13 -1708,8 +2014,13 @@@copy_or_zero_from_mapped(parts_out, m_ymiss_ + (size_t) tb * K * N, (long long) n * K * N,skip_ + grp, ring, cs);} else {- const bool fuse_rows = fuse_wait_on() && !prof_on_ && !remote_opt_ && dec_batch && m_bal_ == nullptr; // 143-amdk- const bool fuse_rows = fuse_rows_on() && !prof_on_ && !remote_opt_ && dec_batch; // 143-amdk- if (!fuse_rows) wait_flag_ge(m_flag_, ring, cs); // the CPU's share is in the mapped rows++ const bool fuse_rows = fuse_rows_on() && !prof_on_ && !remote_opt_ && dec_batch && m_bal_ == nullptr; // 143-amdk+ if (!fuse_rows) {+ if (m_bal_ != nullptr) // set_pcie_balance: ... and ends as it waits for the CPU's rows+ wait_flag_ge_stamped(m_flag_, ring, m_bal_ + ((l - lb_) * 2 + grp) * 4 + 2, nullptr, cs);+ else+ wait_flag_ge(m_flag_, ring, cs); // the CPU's share is in the mapped rows+ }stamp(l, 23, grp);if (remote_opt_) // #578: the helper's rows come back reduced; skip them as wellremote_opt_->copy_rows(parts_out, m_ymiss_ + (size_t) tb * K * N, tb, n, p_dst, p_counts + 1, cs);diff --cc src/kernels/cuda/fused_gr.cuindex 38b7535b6,ce2a395e2..7b7274866--- a/src/kernels/cuda/fused_gr.cu+++ b/src/kernels/cuda/fused_gr.cu@@@ -1085,9 -1129,52 +1185,53 @@@ bool qfuse_hc_local()static const bool on = [] { const char* e = std::getenv("STRATA_QFUSE_HC_LOCAL"); return e == nullptr || std::atoi(e) != 0; }();return on;}+ // Turing: gr_up75_kernel for 3+ tokens (bitwise gr_up_multi_kernel; fused_gr_bench cold on an RTX 2080 Ti, up incl. a+ // stamp: T 3 27.3 -> 24.6 us, T 4 33.2 -> 28.7, T 6 46.6 -> 36.6); STRATA_GR_UP75=0 / 1 forces it off / on+ bool up75_on() {+ #if defined(__HIPCC__)+ return false;+ #else+ static const int env = [] { const char* v = std::getenv("STRATA_GR_UP75"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();+ if (env >= 0) return env != 0;+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) return false;+ if (!per_dev[dev]) {+ int major = 0, minor = 0;+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);+ per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1+ }+ return per_dev[dev] == 2;+ #endif+ }+ // sm_75: the staged down kernel's tokens per block (gridDim.y = ct / tg), 0 = one block row for all of them.+ // STRATA_GR_DOWN_TG=n forces n where it divides the launch (0 = off).+ int down_tg(int ct) {+ #if defined(__HIPCC__)+ (void) ct;+ return 0;+ #else+ static const int env = [] { const char* v = std::getenv("STRATA_GR_DOWN_TG"); return v ? std::atoi(v) : -1; }();+ if (ct < 2) return 0;+ if (env >= 0) return env > 0 && env <= 3 && ct % env == 0 ? env : 0;+ // fused_gr_bench cold on an RTX 2080 Ti (down incl. a stamp): T 4 29.1 -> 27.3 us, T 6 38.5 -> 32.2 with pairs;+ // one token per block is slower at every T (the weights are read from L2 once per group)+ static int per_dev[64]; // 0 unknown, 1 no, 2 sm_75+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) return 0;+ if (!per_dev[dev]) {+ int major = 0, minor = 0;+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);+ per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1+ }+ return per_dev[dev] == 2 && (ct == 4 || ct == 6) ? 2 : 0;+ #endif+ }void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long long* stamp_buf, int stamp_i0) {const int n_tok = m.T;+ const bool q8w = m.a[0].s_down != nullptr; // STRATA_HC_REQ8: w_down / w_up hold int8 codes#if STRATA_GR_FAST_BUILD// gfx906: the latency-hidden norm/up (STRATA_GR_FAST=0: off) - the same sums in the same order as the kernels// they replace (gr_parity checks the multi-token read against single-token calls bitwise)@@@ -1173,16 -1261,16 +1325,26 @@@return;}#endif- if (n_tok >= 3 && up75_on()) { // T 1-2: gr_up_multi_kernel is as fast or faster (fused_gr_bench, cold)+ if (q8w) { // STRATA_HC_REQ8: the up kernel on int8 rows+ switch (no_multi_gr ? kFusedGrMaxT : n_tok) {+ case 1: gr_up_multi_kernel<1, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 2: gr_up_multi_kernel<2, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 3: gr_up_multi_kernel<3, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 4: gr_up_multi_kernel<4, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 5: gr_up_multi_kernel<5, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 6: gr_up_multi_kernel<6, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ default: gr_up_multi_kernel<kFusedGrMaxT, false, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ }++ } else if (n_tok >= 3 && up75_on()) { // T 1-2: gr_up_multi_kernel is as fast or faster (fused_gr_bench, cold)+ switch (no_multi_gr ? kFusedGrMaxT : n_tok) {+ case 1: gr_up75_kernel<1, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 2: gr_up75_kernel<2, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 3: gr_up75_kernel<3, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 4: gr_up75_kernel<4, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 5: gr_up75_kernel<5, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ case 6: gr_up75_kernel<6, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ default: gr_up75_kernel<kFusedGrMaxT, false><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;+ }} elseswitch (no_multi_gr ? kFusedGrMaxT : n_tok) {case 1: gr_up_multi_kernel<1, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;diff --cc src/kernels/cuda/qsa_decode_attn.cuindex 062ef664e,71b5bc358..854b479ee--- a/src/kernels/cuda/qsa_decode_attn.cu+++ b/src/kernels/cuda/qsa_decode_attn.cu@@@ -284,47 -282,131 +285,172 @@@ __device__ __forceinline__ void attn_chfor (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];}+template <int KV_MODE, bool LANE_CELL = false, bool QUERY_SWIZZLE = false, bool REDUCE12 = false>+__global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __restrict__ q, QsaAttnPools p,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale, float* __restrict__ part_acc,+ float* __restrict__ part_m, float* __restrict__ part_l,+ int n_chunks, int cap = 0, long long scratch_stride = 0) {+ attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap,+ scratch_stride);+}++// The same blocks for queries of different slots: query blockIdx.z reads its own slot's pools, everything else is+// attn_chunk_kernel's batched form (qsa_decode_attn_slots).+template <int KV_MODE, bool LANE_CELL>+__global__ void __launch_bounds__(THREADS) attn_chunk_slots_kernel(const float* __restrict__ q, QsaAttnSlotPools ps,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale,+ float* __restrict__ part_acc,+ float* __restrict__ part_m,+ float* __restrict__ part_l, int n_chunks, int cap,+ long long scratch_stride) {+ attn_chunk_body<KV_MODE, LANE_CELL, false, false>(q, ps.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);+}++// Batch windows: query blockIdx.z attends over its own slot's pools (rows.p[z]); the default kernel otherwise.+template <int KV_MODE>+__global__ void __launch_bounds__(THREADS) attn_chunk_rows_kernel(const float* __restrict__ q, QsaAttnPoolsRows rows,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale,+ float* __restrict__ part_acc,+ float* __restrict__ part_m,+ float* __restrict__ part_l, int n_chunks, int cap,+ long long scratch_stride) {+ attn_chunk_body<KV_MODE, false, false, false>(q, rows.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);+}++ #if !defined(__HIPCC__)+ // Turing (INT8 pools): attn_chunk_kernel<1> bit for bit, rearranged for the SM's shared-memory pipe, which the+ // default kernel saturates (its time grew with the queries even with the SMs half empty). Three changes, none to an+ // operand or the order of an operation:+ // - the lane's slice of the 12 query heads (8 dimensions each) is read from shared memory once, into registers,+ // instead of two 16-byte reads per head per cell (with a two-way bank conflict at lane * 8);+ // - the 12 heads' lane sums are reduced with reduce12_gfx906 (16 shuffles, each sum warp_sum's pairs) instead of+ // 12 warp_sums (60 shuffles);+ // - the probabilities are stored cell-major, so the value loop reads a cell's 12 as three 16-byte broadcasts.+ // RTX 2080 Ti, 2051 selected cells: 1 / 2 / 3 / 4 queries 51.6 / 78.5 / 110 / 146 -> 34.5 / 41.0 / 73.0 / 84.9 us+ // (with the merge); 600 cells 45.9 / 74.6 / 105 / 137 -> 28.6 / 32.8 / 33.5 / 33.3 us. STRATA_ATTN_V3=0 keeps the+ // default kernel; =1 takes this one on any CUDA card.+ __global__ void __launch_bounds__(THREADS) attn_chunk_q8_v3_kernel(const float* __restrict__ q, QsaAttnPools p,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale, float* __restrict__ part_acc,+ float* __restrict__ part_m, float* __restrict__ part_l,+ int n_chunks, int cap = 0, long long scratch_stride = 0) {+ q += (size_t) blockIdx.z * (size_t) (n_kv_heads * G) * HD;+ ids += (size_t) blockIdx.z * (size_t) cap;+ step += (size_t) blockIdx.z * kStepCount;+ part_acc += (size_t) blockIdx.z * (size_t) scratch_stride;+ part_m += (size_t) blockIdx.z * (size_t) scratch_stride;+ part_l += (size_t) blockIdx.z * (size_t) scratch_stride;+ __shared__ __align__(16) float sq[G][HD];+ __shared__ __align__(16) float spt[CHUNK][G]; // cell-major: one cell's 12 heads are 3 float4+ __shared__ long long srow[CHUNK];+ const int n_ids = __ldg(step + kStepWidth);+ const int chunk = blockIdx.x, kvh = blockIdx.y;+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ const int c0 = chunk * CHUNK;+ const int n_here = min(CHUNK, n_ids - c0);+ const int slot = kvh * n_chunks + chunk;+ if (n_here <= 0) {+ if (t < G) { part_m[slot * G + t] = -FLT_MAX; part_l[slot * G + t] = 0.0f; }+ return;+ }+ // stored with the gfx906 swizzle (the two float4 halves of lanes with bit 2 swapped): fewer conflicts on the one read+ for (int i = t; i < G * HD; i += THREADS) {+ const int d = i % HD;+ sq[i / HD][d ^ ((d >> 3) & 4)] = q[(size_t) (kvh * G) * HD + i];+ }+ if (t < CHUNK) {+ long long r = -1;+ if (t < n_here) {+ const int cell = ids[c0 + t];+ const long long page = (long long) p.page_table[cell / page_size];+ if (page >= 0) r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);+ }+ srow[t] = r;+ }+ __syncthreads();+ float qr[G][8];+ #pragma unroll+ for (int h = 0; h < G; ++h) {+ const int qflip = lane & 4;+ const float4 qa = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + qflip]);+ const float4 qb = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + (qflip ^ 4)]);+ qr[h][0] = qa.x; qr[h][1] = qa.y; qr[h][2] = qa.z; qr[h][3] = qa.w;+ qr[h][4] = qb.x; qr[h][5] = qb.y; qr[h][6] = qb.z; qr[h][7] = qb.w;+ }+ for (int c = warp; c < CHUNK; c += WARPS) {+ if (c >= n_here || srow[c] < 0) {+ if (lane < G) spt[c][lane] = -FLT_MAX;+ continue;+ }+ float k8[8];+ load8<1>(p, false, srow[c], lane * 8, k8);+ float part[16];+ #pragma unroll+ for (int h = 0; h < G; ++h)+ part[h] = k8[0] * qr[h][0] + k8[1] * qr[h][1] + k8[2] * qr[h][2] + k8[3] * qr[h][3] ++ k8[4] * qr[h][4] + k8[5] * qr[h][5] + k8[6] * qr[h][6] + k8[7] * qr[h][7];+ #pragma unroll+ for (int h = G; h < 16; ++h) part[h] = 0.0f;+ const float score = reduce12_gfx906(part, lane);+ const int head = reduce12_head_of_lane(lane);+ if ((lane & 1) == 0 && head < G) spt[c][head] = score * scale;+ }+ __syncthreads();+ for (int h = warp; h < G; h += WARPS) {+ const float a = spt[lane][h], b = spt[lane + 32][h];+ const float m = warp_max(fmaxf(a, b));+ const float ea = (lane < n_here && srow[lane] >= 0) ? __expf(a - m) : 0.0f;+ const float eb = (lane + 32 < n_here && srow[lane + 32] >= 0) ? __expf(b - m) : 0.0f;+ spt[lane][h] = ea;+ spt[lane + 32][h] = eb;+ const float l = warp_sum(ea + eb);+ if (lane == 0) { part_m[slot * G + h] = m; part_l[slot * G + h] = l; }+ }+ __syncthreads();+ float acc[G];+ #pragma unroll+ for (int h = 0; h < G; ++h) acc[h] = 0.0f;+ for (int c = 0; c < n_here; ++c) {+ if (srow[c] < 0) continue; // masked above, weight 0+ const float sc = __half2float(__ushort_as_half(p.v_scale[srow[c] * (HD / KV_Q8_GROUP) + t / KV_Q8_GROUP]));+ const float v = (float) p.v_q[srow[c] * HD + t] * sc;+ const float4 p0 = *reinterpret_cast<const float4*>(&spt[c][0]);+ const float4 p1 = *reinterpret_cast<const float4*>(&spt[c][4]);+ const float4 p2 = *reinterpret_cast<const float4*>(&spt[c][8]);+ const float pr[G] = {p0.x, p0.y, p0.z, p0.w, p1.x, p1.y, p1.z, p1.w, p2.x, p2.y, p2.z, p2.w};+ #pragma unroll+ for (int h = 0; h < G; ++h) acc[h] = fmaf(pr[h], v, acc[h]);+ }+ #pragma unroll+ for (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];+ }+ bool attn_v3_on() {+ static int on[16] = {}; // by device ordinal: 0 unknown, 1 yes, -1 no+ static const int env = [] { const char* v = std::getenv("STRATA_ATTN_V3"); return v ? (v[0] == '0' ? 0 : 1) : -1; }();+ if (env >= 0) return env == 1;+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 16) return false;+ if (on[dev] == 0) {+ int maj = 0, mnr = 0;+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;+ if (!ok) cudaGetLastError();+ on[dev] = ok && strata::sm75_default(maj, mnr) ? 1 : -1; // Turing only, with STRATA_SM75_PERF=1+ }+ return on[dev] > 0;+ }+ #endif+// PR #540 (sskver): the same attention with fewer shuffles and less shared-memory traffic, bit for bit the kernel// above (same operands, same order); +7% prompt speed on a V100, where this kernel reads every prompt chunk. It// uses more registers (80 vs 38 on sm_70), so on CUDA it runs on cards below sm_75 only, and exists only in the@@@ -874,49 -814,27 +1000,55 @@@ void launch_chunks_batch(const float* q(int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);}} else+ #endif+ #if !defined(__HIPCC__)- if (kv_mode == 1 && attn_v3_on())++ if (kernel < 0 && kv_mode == 1 && attn_v3_on()) // (the parity harness names its kernel: never the environment)+ attn_chunk_q8_v3_kernel<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,+ scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);+ else#endifif (kv_mode == 3)- STRATA_ATTN_CHUNK(3)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);+ STRATA_ATTN_CHUNK_SEL(3, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,+ (int) s.page_size, scale, part_acc, part_m, part_l,+ n_chunks, (int) cap, stride);else if (kv_mode == 2)- STRATA_ATTN_CHUNK(2)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);+ STRATA_ATTN_CHUNK_SEL(2, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,+ (int) s.page_size, scale, part_acc, part_m, part_l,+ n_chunks, (int) cap, stride);else if (kv_mode == 1)- STRATA_ATTN_CHUNK(1)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);+ STRATA_ATTN_CHUNK_SEL(1, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,+ (int) s.page_size, scale, part_acc, part_m, part_l,+ n_chunks, (int) cap, stride);else- STRATA_ATTN_CHUNK(0)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,- attn, stride);+ STRATA_ATTN_CHUNK_SEL(0, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,+ (int) s.page_size, scale, part_acc, part_m, part_l,+ n_chunks, (int) cap, stride);+}++// qsa_decode_attn_batch's merge pass over the partials launch_chunks_batch wrote; `v2` names the merge kernel+void launch_merge_batch(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch, float* attn,+ int64_t n_q, int v2, cudaStream_t st) {+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);+ const float* part_acc = scratch;+ const float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;+ const float* part_l = part_m + (size_t) n_chunks * s.n_head;+ launch_merge(v2, part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head, (unsigned) n_q, st);+}++} // namespace++void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {+ if (n_q <= 0) return;+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps ||+ !pools.page_table || n_q > 65535) {+ std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");+ std::exit(1);+ }+ cudaStream_t st = (cudaStream_t) stream;+ launch_chunks_batch(q, pools, ids, steps, cap, s, scratch, n_q, st, -1);+ launch_merge_batch(steps, cap, s, scratch, attn, n_q, attn_merge_v2(), st);const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) {std::fprintf(stderr, "qsa_decode_attn_batch: %s\n", cudaGetErrorString(e));diff --cc src/kernels/cuda/qsa_prompt_attn.cuindex 06ed48c90,54f40e9aa..b2b5480ce--- a/src/kernels/cuda/qsa_prompt_attn.cu+++ b/src/kernels/cuda/qsa_prompt_attn.cu@@@ -679,250 -793,7 +794,251 @@@ __global__ void __launch_bounds__(THREA}}+ #undef PA2_PART+__global__ void __launch_bounds__(THREADS) decode_attn_i8_split_kernel(const float* __restrict__ q, QsaAttnPools p,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ steps, int n_kv_heads,+ int page_size, float scale_log2,+ float* __restrict__ part, int cap) {+ extern __shared__ __align__(16) unsigned char smem_raw[];+ Smem2& S = *reinterpret_cast<Smem2*>(smem_raw);+ const int qi = blockIdx.x, kvh = blockIdx.y, sp = blockIdx.z, nsp = gridDim.z;+ const int n_head = n_kv_heads * G;+ q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;+ // partials of (query, split): acc [n_head][HD] unnormalized, then m / l [n_head][2]+ float* pacc = part + (size_t) (qi * nsp + sp) * n_head * (HD + 2) + (size_t) kvh * G * HD;+ float* pml = part + (size_t) (qi * nsp + sp) * n_head * (HD + 2) + (size_t) n_head * HD + (size_t) kvh * G * 2;+ ids += (size_t) qi * cap;+ const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);+ // this split's chunks: [ci0, ci1) of the query's ceil(n / CH2)+ const int n_all = (n + CH2 - 1) / CH2, per = (n_all + nsp - 1) / nsp;+ const int ci0 = sp * per, ci1 = min(n_all, ci0 + per);+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ const int gid = lane >> 2, tig = lane & 3;+ const int dim0 = warp * 64;++ // q: the power-of-two prescale over all 12 heads (as v1), then this warp's 64 dims as hi/lo A fragments+ float qm = 0.0f;+ for (int i = t; i < G * HD; i += THREADS) qm = fmaxf(qm, fabsf(q[i]));+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));+ if (lane == 0) S.qmax[warp] = qm;+ if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }+ __syncthreads();+ qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));+ int qe = 0;+ if (qm > 0.0f) frexpf(qm, &qe);+ const float qup = ldexpf(1.0f, 14 - qe), qdown = ldexpf(scale_log2, qe - 14);+ uint32_t qh[4][4], ql[4][4];+#pragma unroll+ for (int kk = 0; kk < 4; ++kk) {+#pragma unroll+ for (int r = 0; r < 4; ++r) {+ const int row = gid + (r & 1) * 8, col = dim0 + kk * 16 + 2 * tig + (r >> 1) * 8;+ float2 x = make_float2(0.f, 0.f);+ if (row < G) x = *reinterpret_cast<const float2*>(q + (size_t) row * HD + col);+ x.x *= qup;+ x.y *= qup;+ const __half2 hi = __floats2half2_rn(x.x, x.y);+ const float2 hf = __half22float2(hi);+ const __half2 lo = __floats2half2_rn(x.x - hf.x, x.y - hf.y);+ qh[kk][r] = *reinterpret_cast<const uint32_t*>(&hi);+ ql[kk][r] = *reinterpret_cast<const uint32_t*>(&lo);+ }+ }++ if (ci0 >= ci1) { // an empty split: m = -inf, l = 0, acc = 0+ for (int i = t; i < G * HD; i += THREADS) pacc[(size_t) (i / HD) * HD + i % HD] = 0.0f;+ if (t < G) { pml[2 * t] = -INFINITY; pml[2 * t + 1] = 0.0f; }+ return;+ }+ // the chunk pipeline: cells two chunks ahead, their pool rows one chunk ahead, the data (cp.async) one ahead+ auto cell_of = [&](int c) -> int { return c < n ? __ldg(ids + c) : -1; };+ auto row_of = [&](int cell) -> long long {+ if (cell < 0) return -1;+ const long long page = (long long) __ldg(p.page_table + cell / page_size);+ return (page * n_kv_heads + kvh) * page_size + (cell % page_size);+ };+ auto issue = [&](long long r, int st, float& ksr, float& vsr) {+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int idx = lane + 32 * j, cell = idx >> 2, pc = idx & 3;+ const long long rr = __shfl_sync(0xffffffffu, r, cell);+ const bool ok = rr >= 0;+ const size_t off = ok ? (size_t) rr * HD + dim0 + pc * 16 : 0;+ cp_async16(&S.kv[st][warp][0][0][0] + swz(cell, pc * 16), p.k_q + off, ok);+ cp_async16(&S.kv[st][warp][1][0][0] + swz(cell, pc * 16), p.v_q + off, ok);+ }+ ksr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.k_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;+ vsr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.v_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;+ };+ float ksn, vsn;+ issue(row_of(cell_of(ci0 * CH2 + lane)), 0, ksn, vsn);+ cp_async_commit();+ S.sc[0][warp][0][lane] = ksn;+ S.sc[0][warp][1][lane] = vsn;+ long long r_next = row_of(cell_of((ci0 + 1) * CH2 + lane));+ int cell_next2 = cell_of((ci0 + 2) * CH2 + lane);++ float acc[8][4];+#pragma unroll+ for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;++ for (int ci = ci0; ci < ci1; ++ci) {+ const int st = (ci - ci0) & 1, c0 = ci * CH2;+ const bool more = ci + 1 < ci1;+ if (more) issue(r_next, st ^ 1, ksn, vsn);+ cp_async_commit();+ r_next = row_of(cell_next2);+ cell_next2 = cell_of((ci + 3) * CH2 + lane);+ cp_async_wait1();+ __syncwarp();+ const int8_t* K = &S.kv[st][warp][0][0][0];+ const int8_t* V = &S.kv[st][warp][1][0][0];+ // q.k over this warp's 64 dims, times the cell's K scale for this group+#pragma unroll+ for (int nt = 0; nt < CH2 / 8; ++nt) {+ float tg[4] = {0.f, 0.f, 0.f, 0.f};+ const int cell = nt * 8 + gid;+#pragma unroll+ for (int kk = 0; kk < 4; ++kk) {+ uint32_t b[2];+ b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig)));+ b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig + 8)));+ mma16816(tg, qh[kk], b);+ mma16816(tg, ql[kk], b);+ }+ const int c = nt * 8 + 2 * tig;+ const float s0 = S.sc[st][warp][0][c], s1 = S.sc[st][warp][0][c + 1];+ S.part[warp][gid][c] = tg[0] * s0;+ S.part[warp][gid][c + 1] = tg[1] * s1;+ S.part[warp][gid + 8][c] = tg[2] * s0;+ S.part[warp][gid + 8][c + 1] = tg[3] * s1;+ }+ __syncthreads();+ // online softmax over the four groups' sum (fixed order): row t/8, 4 cells per thread+ {+ const int r = t >> 3, sub = t & 7;+ float x[4], mx = -INFINITY;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int c = sub * 4 + j;+ x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * qdown+ : -INFINITY;+ mx = fmaxf(mx, x[j]);+ }+#pragma unroll+ for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));+ const float m_old = S.mrow[r];+ const float m_new = fmaxf(m_old, mx);+ float sum = 0.0f;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);+ S.p[r][sub * 4 + j] = e;+ sum += e;+ }+#pragma unroll+ for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);+ __syncwarp();+ if (sub == 0) {+ const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);+ S.alpha[r] = a;+ S.lsum[r] = fmaf(S.lsum[r], a, sum);+ S.mrow[r] = m_new;+ }+ }+ __syncthreads();+ // p.v over this warp's 64 dims (as v1)+ {+ float vmax = S.sc[st][warp][1][lane];+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));+ const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);+ float tmp[8][4];+#pragma unroll+ for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;+#pragma unroll+ for (int ks = 0; ks < CH2 / 16; ++ks) {+ const int cA = ks * 16 + 2 * tig, cB = cA + 8;+ const float w0 = S.sc[st][warp][1][cA] * vup, w1 = S.sc[st][warp][1][cA + 1] * vup,+ w2 = S.sc[st][warp][1][cB] * vup, w3 = S.sc[st][warp][1][cB + 1] * vup;+ const float p00 = S.p[gid][cA] * w0, p01 = S.p[gid][cA + 1] * w1;+ const float p10 = S.p[gid + 8][cA] * w0, p11 = S.p[gid + 8][cA + 1] * w1;+ const float p02 = S.p[gid][cB] * w2, p03 = S.p[gid][cB + 1] * w3;+ const float p12 = S.p[gid + 8][cB] * w2, p13 = S.p[gid + 8][cB + 1] * w3;+ uint32_t ah[4], al[4];+ ah[0] = pack_h2(p00, p01);+ ah[1] = pack_h2(p10, p11);+ ah[2] = pack_h2(p02, p03);+ ah[3] = pack_h2(p12, p13);+ {+ const __half2* h = reinterpret_cast<const __half2*>(ah);+ float2 f;+ f = __half22float2(h[0]); al[0] = pack_h2(p00 - f.x, p01 - f.y);+ f = __half22float2(h[1]); al[1] = pack_h2(p10 - f.x, p11 - f.y);+ f = __half22float2(h[2]); al[2] = pack_h2(p02 - f.x, p03 - f.y);+ f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);+ }+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int d = j * 8 + gid;+ const uint32_t x0 = (uint8_t) V[swz(cA, d)] | ((uint32_t) (uint8_t) V[swz(cA + 1, d)] << 8);+ const uint32_t x1 = (uint8_t) V[swz(cB, d)] | ((uint32_t) (uint8_t) V[swz(cB + 1, d)] << 8);+ uint32_t b[2];+ b[0] = i8x2_to_h2(x0);+ b[1] = i8x2_to_h2(x1);+ mma16816(tmp[j], ah, b);+ mma16816(tmp[j], al, b);+ }+ }+ const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);+ acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);+ acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);+ acc[j][3] = fmaf(acc[j][3], a1, tmp[j][3] * vdown);+ }+ }+ if (more) {+ S.sc[st ^ 1][warp][0][lane] = ksn;+ S.sc[st ^ 1][warp][1][lane] = vsn;+ }+ __syncwarp();+ }+ __syncthreads();+ // the split's unnormalized sums and each head row's maximum and sum (the merge rescales and divides)+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int d = dim0 + j * 8 + 2 * tig;+ *reinterpret_cast<float2*>(pacc + (size_t) gid * HD + d) = make_float2(acc[j][0], acc[j][1]);+ if (gid + 8 < G)+ *reinterpret_cast<float2*>(pacc + (size_t) (gid + 8) * HD + d) = make_float2(acc[j][2], acc[j][3]);+ }+ if (t < G) { pml[2 * t] = S.mrow[t]; pml[2 * t + 1] = S.lsum[t]; }+}++// the splits of each (query, head) merged with the log-sum-exp rescale (base 2, as the kernel's online softmax), in+// split order; a block per (query, head), a thread per dimension+__global__ void __launch_bounds__(HD) decode_attn_merge_kernel(const float* __restrict__ part, int n_head, int nsp,+ float* __restrict__ attn) {+ const int qi = blockIdx.x, h = blockIdx.y, d = threadIdx.x;+ const size_t qs = (size_t) qi * nsp * n_head * (HD + 2);+ float M = -INFINITY;+ for (int s = 0; s < nsp; ++s) M = fmaxf(M, part[qs + (size_t) s * n_head * (HD + 2) + (size_t) n_head * HD + 2 * h]);+ float L = 0.0f, o = 0.0f;+ for (int s = 0; s < nsp; ++s) {+ const float* ps = part + qs + (size_t) s * n_head * (HD + 2);+ const float m = ps[(size_t) n_head * HD + 2 * h], l = ps[(size_t) n_head * HD + 2 * h + 1];+ const float w = m == -INFINITY ? 0.0f : exp2f(m - M);+ L = fmaf(l, w, L);+ o = fmaf(ps[(size_t) h * HD + d], w, o);+ }+ attn[((size_t) qi * n_head + h) * HD + d] = L > 0.0f ? o * (1.0f / L) : 0.0f;+}++bool launch_i8(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {@@@ -953,356 -830,7 +1075,356 @@@}return true;}++// ---- v3 (int8 KV, opt-in STRATA_PROMPT_ATTN_IMMA=1): v2's block, pipeline and online softmax, with both products on+// INT8 tensor cores (mma m16n8k32 s8, int32 accumulation). K and V enter as the int8 codes they are - no int8 -> fp16+// conversion - and the other operand as THREE 8-bit parts: q per (head, 64-dim group) and p * v_scale per (head, chunk)+// scaled to 127, then X = rint(x * 65536) split exactly into h (signed), l (signed) and m (unsigned) bytes - 24 bits,+// as fine as FP32 itself and finer than the FP16 hi + lo split v2 uses (~22 bits). Each part's int32 sum is exact; the three meet in+// FP32 as h + (l * 256 + m) / 65536 (m unsigned: a u8 x s8 MMA). INT8 MMA runs ~3.1x FP16's rate on sm_120 (732 vs 233 measured), so three INT8+// MMAs cost about one FP16 MMA where v2 spends two. p.v takes each thread's 8 consecutive dims across the 8 n-tiles+// (n-tile j, column n = dim n * 8 + j), so a thread reads 8 bytes of a V row per cell and byte-transposes 4 cells.+__device__ __forceinline__ void imma16832(int* c, const uint32_t* a, const uint32_t* b) {+#if !STRATA_PA_SM80+ __trap();+#else+ asm volatile("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "+ "{%0,%1,%2,%3};\n"+ : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3])+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));+#endif+}+// the low part's MMA: unsigned A (m in 0..255), signed B+__device__ __forceinline__ void imma16832_us(int* c, const uint32_t* a, const uint32_t* b) {+#if !STRATA_PA_SM80+ __trap();+#else+ asm volatile("mma.sync.aligned.m16n8k32.row.col.s32.u8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "+ "{%0,%1,%2,%3};\n"+ : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3])+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));+#endif+}+// x (|x| <= 127) as X = rint(x * 65536) = h * 65536 + l * 256 + m exactly: h, l signed bytes, m an unsigned one+__device__ __forceinline__ void split3(float x, int& h, int& l, int& m) {+ const int X = __float2int_rn(x * 65536.0f);+ h = (X + 32768) >> 16;+ const int r = X - (h << 16); // -32768 .. 32767+ l = r >> 8; // -128 .. 127+ m = r & 255; // 0 .. 255+}+__device__ __forceinline__ uint32_t pack4(int a, int b, int c, int d) {+ return (uint32_t) (a & 0xff) | ((uint32_t) (b & 0xff) << 8) | ((uint32_t) (c & 0xff) << 16) | ((uint32_t) (d & 0xff) << 24);+}+__device__ __forceinline__ float join3(int h, int l, int m) { // the parts' exact int32 sums -> h + (l*256 + m)/65536+ return (float) h + (float) ((l << 8) + m) * (1.0f / 65536.0f);+}+// four words of four bytes each (word i = cell i) -> four words of the same byte position across the cells+__device__ __forceinline__ void tr4x4(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3, uint32_t* o) {+ const uint32_t t0 = __byte_perm(w0, w1, 0x5140), t1 = __byte_perm(w0, w1, 0x7362);+ const uint32_t t2 = __byte_perm(w2, w3, 0x5140), t3 = __byte_perm(w2, w3, 0x7362);+ o[0] = __byte_perm(t0, t2, 0x5410);+ o[1] = __byte_perm(t0, t2, 0x7632);+ o[2] = __byte_perm(t1, t3, 0x5410);+ o[3] = __byte_perm(t1, t3, 0x7632);+}++// v3's shared memory: ONE stage of K and one of V per warp (a warp gathers and reads only its own 64-dim slices, so+// reusing a buffer needs only that warp's own order): K(c + 1) is gathered while the block runs softmax(c) and+// p.v(c), V(c + 1) while it runs q.k(c + 1) and softmax(c + 1); and only the 12 real head rows of the partial sums+// and p. 25.5 KB instead of v2's 45.6 KB: three blocks an SM instead of two (one block an SM ran 1.54x slower than+// two - this kernel waits on latency, not on its MMAs).+struct Smem3 {+ int8_t kv[4][2][CH2][64]; // warp, K/V, cell, 64 dims in 16-byte pieces XOR-swizzled by the cell+ float sc[4][2][CH2]; // warp, K/V scale of the cell for the warp's group+ float part[4][G][CH2 + 1]; // q.k per dim group (the 12 head rows)+ float p[G][CH2 + 1];+ float alpha[16];+ float lsum[16];+ float mrow[16];+};++__global__ void __launch_bounds__(THREADS, 3) prompt_attn_i8v3_kernel(const float* __restrict__ q, QsaAttnPools p,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ steps, int n_kv_heads,+ int page_size, float scale_log2,+ float* __restrict__ attn, int cap) {+ extern __shared__ __align__(16) unsigned char smem_raw[];+ Smem3& S = *reinterpret_cast<Smem3*>(smem_raw);+ const int qi = blockIdx.x, kvh = blockIdx.y;+ const int n_head = n_kv_heads * G;+ q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;+ attn += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;+ ids += (size_t) qi * cap;+ const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ const int gid = lane >> 2, tig = lane & 3;+ const int dim0 = warp * 64;+ if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; S.alpha[t] = 0.0f; }++ // q: rows gid / gid + 8 (heads; 12..15 are pad), this warp's 64 dims as the A fragments of two k-steps of 32,+ // three 8-bit parts with a scale per (row, 64-dim group)+ uint32_t qa[3][2][4];+ float qinv[2];+ {+ float x[2][2][2][4]; // [row half][k-step][k half][4 dims]+ float mx[2] = {0.0f, 0.0f};+#pragma unroll+ for (int rh = 0; rh < 2; ++rh) {+ const int row = gid + rh * 8;+#pragma unroll+ for (int ks = 0; ks < 2; ++ks)+#pragma unroll+ for (int kh = 0; kh < 2; ++kh) {+ float4 v = make_float4(0.f, 0.f, 0.f, 0.f);+ if (row < G) v = *reinterpret_cast<const float4*>(q + (size_t) row * HD + dim0 + ks * 32 + kh * 16 + tig * 4);+ x[rh][ks][kh][0] = v.x; x[rh][ks][kh][1] = v.y; x[rh][ks][kh][2] = v.z; x[rh][ks][kh][3] = v.w;+ mx[rh] = fmaxf(mx[rh], fmaxf(fmaxf(fabsf(v.x), fabsf(v.y)), fmaxf(fabsf(v.z), fabsf(v.w))));+ }+#pragma unroll+ for (int o = 1; o < 4; o <<= 1) mx[rh] = fmaxf(mx[rh], __shfl_xor_sync(0xffffffffu, mx[rh], o));+ }+ float up[2];+#pragma unroll+ for (int rh = 0; rh < 2; ++rh) {+ up[rh] = mx[rh] > 0.0f ? 127.0f / mx[rh] : 0.0f;+ qinv[rh] = mx[rh] * (1.0f / 127.0f);+ }+#pragma unroll+ for (int ks = 0; ks < 2; ++ks)+#pragma unroll+ for (int r = 0; r < 4; ++r) { // a0: row gid, k 0..15 half; a1: row gid + 8; a2: row gid, k 16..31; a3+ const int rh = r & 1, kh = r >> 1;+ int hh[4], ll[4], mm[4];+#pragma unroll+ for (int e = 0; e < 4; ++e) split3(x[rh][ks][kh][e] * up[rh], hh[e], ll[e], mm[e]);+ qa[0][ks][r] = pack4(hh[0], hh[1], hh[2], hh[3]);+ qa[1][ks][r] = pack4(ll[0], ll[1], ll[2], ll[3]);+ qa[2][ks][r] = pack4(mm[0], mm[1], mm[2], mm[3]);+ }+ }++ const int n_chunks = (n + CH2 - 1) / CH2;+ auto cell_of = [&](int c) -> int { return c < n ? __ldg(ids + c) : -1; };+ auto row_of = [&](int cell) -> long long {+ if (cell < 0) return -1;+ const long long page = (long long) __ldg(p.page_table + cell / page_size);+ return (page * n_kv_heads + kvh) * page_size + (cell % page_size);+ };+ // this warp's 64-dim slice of K (kv = 0) or V (1) of the chunk whose rows the lanes hold, and the cells' scales+ auto issue = [&](long long r, int kv) {+ const int8_t* pool = kv == 0 ? p.k_q : p.v_q;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int idx = lane + 32 * j, cell = idx >> 2, pc = idx & 3;+ const long long rr = __shfl_sync(0xffffffffu, r, cell);+ const bool ok = rr >= 0;+ const size_t off = ok ? (size_t) rr * HD + dim0 + pc * 16 : 0;+ cp_async16(&S.kv[warp][kv][0][0] + swz(cell, pc * 16), pool + off, ok);+ }+ const uint16_t* sc = kv == 0 ? p.k_scale : p.v_scale;+ S.sc[warp][kv][lane] = r >= 0 ? __half2float(__ushort_as_half(__ldg(sc + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;+ };+ long long r_nxt = row_of(cell_of(lane)); // chunk 0+ issue(r_nxt, 0);+ cp_async_commit();+ issue(r_nxt, 1);+ cp_async_commit();+ r_nxt = row_of(cell_of(CH2 + lane)); // chunk 1+ int cell_nn = cell_of(2 * CH2 + lane); // chunk 2+ __syncthreads();++ float acc[8][4];+#pragma unroll+ for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;+ const int8_t* K = &S.kv[warp][0][0][0];+ const int8_t* V = &S.kv[warp][1][0][0];++ for (int ci = 0; ci < n_chunks; ++ci) {+ const int c0 = ci * CH2;+ const bool more = ci + 1 < n_chunks;+ cp_async_wait1(); // K(ci) is in (pending: V(ci))+ __syncwarp();+ // q.k over this warp's 64 dims: n-tile nt holds cells nt * 8 .. + 7 (column n = cell nt * 8 + n)+#pragma unroll+ for (int nt = 0; nt < CH2 / 8; ++nt) {+ int ch[4] = {0, 0, 0, 0}, cl[4] = {0, 0, 0, 0}, cm[4] = {0, 0, 0, 0};+ const int cell = nt * 8 + gid;+#pragma unroll+ for (int ks = 0; ks < 2; ++ks) {+ uint32_t b[2];+ b[0] = *reinterpret_cast<const uint32_t*>(K + swz(cell, ks * 32 + tig * 4));+ b[1] = *reinterpret_cast<const uint32_t*>(K + swz(cell, ks * 32 + 16 + tig * 4));+ imma16832(ch, qa[0][ks], b);+ imma16832(cl, qa[1][ks], b);+ imma16832_us(cm, qa[2][ks], b);+ }+ const int c = nt * 8 + 2 * tig;+ const float s0 = S.sc[warp][0][c], s1 = S.sc[warp][0][c + 1];+ S.part[warp][gid][c] = join3(ch[0], cl[0], cm[0]) * qinv[0] * s0;+ S.part[warp][gid][c + 1] = join3(ch[1], cl[1], cm[1]) * qinv[0] * s1;+ if (gid + 8 < G) {+ S.part[warp][gid + 8][c] = join3(ch[2], cl[2], cm[2]) * qinv[1] * s0;+ S.part[warp][gid + 8][c + 1] = join3(ch[3], cl[3], cm[3]) * qinv[1] * s1;+ }+ }+ __syncwarp(); // every lane has read K(ci) and its scales+ if (more) issue(r_nxt, 0);+ cp_async_commit();+ __syncthreads();+ // online softmax over the four groups' sum (fixed order): rows 0..11, 8 threads a row (warp 3 idles)+ if (t < G * 8) {+ const int r = t >> 3, sub = t & 7;+ float x[4], mx = -INFINITY;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int c = sub * 4 + j;+ x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * scale_log2+ : -INFINITY;+ mx = fmaxf(mx, x[j]);+ }+#pragma unroll+ for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));+ const float m_old = S.mrow[r];+ const float m_new = fmaxf(m_old, mx);+ float sum = 0.0f;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);+ S.p[r][sub * 4 + j] = e;+ sum += e;+ }+#pragma unroll+ for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);+ __syncwarp();+ if (sub == 0) {+ const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);+ S.alpha[r] = a;+ S.lsum[r] = fmaf(S.lsum[r], a, sum);+ S.mrow[r] = m_new;+ }+ }+ __syncthreads();+ cp_async_wait1(); // V(ci) is in (pending: K(ci + 1))+ __syncwarp();+ // p.v over this warp's 64 dims: A = p * v_scale (rows gid / gid + 8, k = the chunk's 32 cells) in three parts+ // scaled per row; B = V (k = cells, n-tile j column n = dim n * 8 + j)+ {+ uint32_t pa[3][4];+ float pinv[2];+ { // a scale per row: a chunk whose p are all small still gets its 24 bits (one per warp and chunk lost+ // ~3x of the accuracy against FP64)+ float a[4][4], mx[2] = {0.0f, 0.0f};+#pragma unroll+ for (int r = 0; r < 4; ++r) { // a0: row gid, cells tig * 4 ..; a1: row gid + 8; a2 / a3: cells 16 + ...+ const int row = gid + (r & 1) * 8, cb = (r >> 1) * 16 + tig * 4;+#pragma unroll+ for (int e = 0; e < 4; ++e) {+ a[r][e] = row < G ? S.p[row][cb + e] * S.sc[warp][1][cb + e] : 0.0f;+ mx[r & 1] = fmaxf(mx[r & 1], a[r][e]);+ }+ }+ float up[2];+#pragma unroll+ for (int rh = 0; rh < 2; ++rh) {+#pragma unroll+ for (int o = 1; o < 4; o <<= 1) mx[rh] = fmaxf(mx[rh], __shfl_xor_sync(0xffffffffu, mx[rh], o));+ up[rh] = mx[rh] > 0.0f ? 127.0f / mx[rh] : 0.0f;+ pinv[rh] = mx[rh] * (1.0f / 127.0f);+ }+#pragma unroll+ for (int r = 0; r < 4; ++r) {+ int hh[4], ll[4], mm[4];+#pragma unroll+ for (int e = 0; e < 4; ++e) split3(a[r][e] * up[r & 1], hh[e], ll[e], mm[e]);+ pa[0][r] = pack4(hh[0], hh[1], hh[2], hh[3]);+ pa[1][r] = pack4(ll[0], ll[1], ll[2], ll[3]);+ pa[2][r] = pack4(mm[0], mm[1], mm[2], mm[3]);+ }+ }+ // B: this thread's 8 dims (gid * 8 .. + 7) of cells tig * 4 .. + 3 (b0) and 16 + tig * 4 .. + 3 (b1)+ uint32_t bv[2][8]; // [k half][dim j]+#pragma unroll+ for (int kh = 0; kh < 2; ++kh) {+ uint32_t lo[4], hi[4];+#pragma unroll+ for (int e = 0; e < 4; ++e) {+ const uint2 w = *reinterpret_cast<const uint2*>(V + swz(kh * 16 + tig * 4 + e, gid * 8));+ lo[e] = w.x;+ hi[e] = w.y;+ }+ tr4x4(lo[0], lo[1], lo[2], lo[3], &bv[kh][0]);+ tr4x4(hi[0], hi[1], hi[2], hi[3], &bv[kh][4]);+ }+ __syncwarp(); // every lane has read V(ci) and its scales+ if (more) issue(r_nxt, 1);+ cp_async_commit();+ const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ int ch[4] = {0, 0, 0, 0}, cl[4] = {0, 0, 0, 0}, cm[4] = {0, 0, 0, 0};+ const uint32_t b[2] = {bv[0][j], bv[1][j]};+ imma16832(ch, pa[0], b);+ imma16832(cl, pa[1], b);+ imma16832_us(cm, pa[2], b);+ acc[j][0] = fmaf(acc[j][0], a0, join3(ch[0], cl[0], cm[0]) * pinv[0]);+ acc[j][1] = fmaf(acc[j][1], a0, join3(ch[1], cl[1], cm[1]) * pinv[0]);+ acc[j][2] = fmaf(acc[j][2], a1, join3(ch[2], cl[2], cm[2]) * pinv[1]);+ acc[j][3] = fmaf(acc[j][3], a1, join3(ch[3], cl[3], cm[3]) * pinv[1]);+ }+ }+ r_nxt = row_of(cell_nn);+ cell_nn = cell_of((ci + 3) * CH2 + lane);+ }+ __syncthreads();+ const float l0 = S.lsum[gid], l1 = gid + 8 < G ? S.lsum[gid + 8] : 0.0f;+ const float i0 = l0 > 0.0f ? 1.0f / l0 : 0.0f, i1 = l1 > 0.0f ? 1.0f / l1 : 0.0f;+#pragma unroll+ for (int j = 0; j < 8; ++j) { // column 2 * tig is dim 16 * tig + j, column 2 * tig + 1 dim 16 * tig + 8 + j+ const int d0 = dim0 + 16 * tig + j, d1 = d0 + 8;+ attn[(size_t) gid * HD + d0] = acc[j][0] * i0;+ attn[(size_t) gid * HD + d1] = acc[j][1] * i0;+ if (gid + 8 < G) {+ attn[(size_t) (gid + 8) * HD + d0] = acc[j][2] * i1;+ attn[(size_t) (gid + 8) * HD + d1] = acc[j][3] * i1;+ }+ }+}++bool launch_i8v3(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,+ const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {+ static bool attr[64] = {};+ int dev = 0;+ cudaGetDevice(&dev);+ const int bytes = (int) sizeof(Smem3);+ if (dev < 0 || dev >= 64) return false;+ if (!attr[dev]) {+ if (cudaFuncSetAttribute(prompt_attn_i8v3_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=+ cudaSuccess) {+ cudaGetLastError();+ return false;+ }+ // the shared-memory side of the L1 split as large as it goes, so three blocks fit an SM+ cudaFuncSetAttribute(prompt_attn_i8v3_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100);+ cudaGetLastError();+ attr[dev] = true;+ }+ const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);+ for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {+ const int64_t nb = n_q - q0 < 65535 ? n_q - q0 : 65535;+ prompt_attn_i8v3_kernel<<<dim3((unsigned) nb, (unsigned) s.n_head_kv), THREADS, bytes, st>>>(+ q + q0 * s.n_head * HD, pools, ids + q0 * cap, steps + q0 * kStepCount, (int) s.n_head_kv,+ (int) s.page_size, scale_log2, attn + q0 * s.n_head * HD, (int) cap);+ }+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "qsa_prompt_attn_batch (v3): %s\n", cudaGetErrorString(e));+ std::exit(1);+ }+ return true;+}+- template <int KV_MODE>+ template <int KV_MODE, bool PF = false>bool launch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {static bool attr[64] = {}; // per device, as above@@@ -2193,13 -1644,16 +2321,22 @@@ bool qsa_prompt_attn_batch(const float// exist before sm_80.static const bool v1 = std::getenv("STRATA_PROMPT_ATTN_V1") != nullptr;if (volta) return launch70<1>(q, pools, ids, steps, cap, s, attn, n_q, st);- if (v1 || turing) return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);+ // Turing: v2 with register staging instead of cp.async, opt-in (STRATA_PA_V2T=1) until measured+ static const bool v2t = [] { const char* e = std::getenv("STRATA_PA_V2T"); return e != nullptr && e[0] == '1'; }();+ if (v1 || (turing && !v2t)) {+ // #1821 (xrip): v1 with the next chunk's rows prefetched in registers and K/V sharing one buffer (two blocks an+ // SM), the same bits. STRATA_PA_V1_PREFETCH=0 / 1 forces it off / on; unset: a Turing card with+ // STRATA_SM75_PERF=1.+ static const int pf_env = [] { const char* e = std::getenv("STRATA_PA_V1_PREFETCH"); return e ? (std::atoi(e) != 0 ? 1 : 0) : -1; }();+ if (pf_env == 1 || (pf_env < 0 && t75 && strata::sm75_series())) return launch<1, true>(q, pools, ids, steps, cap, s, attn, n_q, st);+ return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);+ }+ // STRATA_PROMPT_ATTN_IMMA=1 (opt-in, sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel)+ if (g_pa_imma < 0) {+ const char* v = std::getenv("STRATA_PROMPT_ATTN_IMMA");+ g_pa_imma = v != nullptr && v[0] == '1' ? 1 : 0;+ }- if (g_pa_imma == 1) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);++ if (g_pa_imma == 1 && !turing) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);return launch_i8(q, pools, ids, steps, cap, s, attn, n_q, st);}if (!pools.k_pool || !pools.v_pool) return false;diff --cc src/kernels/cuda/qsa_select.cuindex 203c15809,2651982d4..e956dc8b9--- a/src/kernels/cuda/qsa_select.cu+++ b/src/kernels/cuda/qsa_select.cu@@@ -579,23 -556,102 +580,119 @@@ __global__ void __launch_bounds__(256)}}+// The FP32 tiled scores and the tail block; false when the grid does not fit. `rdna2_tiles`: gfx103x's tiling.+static bool block_scores_simt(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps,+ int64_t nq, int64_t max_blocks, int64_t reach, float* scores, cudaStream_t stream,+ bool rdna2_tiles) {+ const int64_t qt = 32, nb = rdna2_tiles ? 128 : 64;+ const dim3 grid((unsigned) ((reach + nb - 1) / nb), (unsigned) ((nq + qt - 1) / qt));+ if (grid.y > 65535) return false;+ if (rdna2_tiles)+ block_scores_simt_kernel<32, 128, 16, 8><<<grid, 256, 0, stream>>>(pooled, q_idx, steps, nq, max_blocks, reach, scores);+ else+ block_scores_simt_kernel<32, 64, 32, 4><<<grid, 256, 0, stream>>>(pooled, q_idx, steps, nq, max_blocks, reach, scores);+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, stream>>>(dead, q_idx, steps, max_blocks, scores);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }+ return true;+}++ // block_scores_simt_kernel with larger tiles (QT queries x NB blocks, RI x CJ products per thread) and the next K+ // chunk's loads in registers while the current one is used. Every (row, block) dot is the same fmaf chain over+ // k = 0..127 from 0, and the relu sum over the heads in the same order: the same bits as block_scores_simt_kernel.+ // STRATA_SELECT_SIMT=2.+ constexpr int S2_KC = 32;+ template <int QT, int NB>+ __global__ void __launch_bounds__(256, 1) block_scores_simt2_kernel(const float* __restrict__ pooled,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int64_t nq,+ int64_t max_blocks, int64_t reach,+ float* __restrict__ out) {+ constexpr int ROWS = QT * IDX_HEADS, RI = ROWS / 16, CJ = NB / 16, QS = ROWS + 4, KS = NB + 4;+ constexpr int QL = ROWS * S2_KC / 4 / 256, KL = NB * S2_KC / 4 / 256; // float4 loads per thread+ static_assert(RI % 4 == 0 && CJ % 4 == 0 && QL >= 1 && KL >= 1, "tile");+ __shared__ __align__(16) float Qs[S2_KC][QS];+ __shared__ __align__(16) float Ks[S2_KC][KS];+ const int64_t q0 = (int64_t) blockIdx.y * QT, b0 = (int64_t) blockIdx.x * NB;+ const int64_t qlast = (q0 + QT < nq ? q0 + QT : nq) - 1;+ if (b0 >= steps[qlast * kStepCount + kStepNBid]) return; // n_bid rises with the position+ const int tid = threadIdx.x, tx = tid & 15, ty = tid >> 4;+ float acc[RI][CJ];+ #pragma unroll+ for (int i = 0; i < RI; ++i)+ #pragma unroll+ for (int j = 0; j < CJ; ++j) acc[i][j] = 0.0f;+ float4 qv[QL], kv[KL];+ auto load = [&](int kc) {+ #pragma unroll+ for (int i = 0; i < QL; ++i) {+ const int idx = tid + i * 256, r = idx >> 3, k4 = idx & 7;+ qv[i] = make_float4(0.f, 0.f, 0.f, 0.f);+ if (q0 + (r >> 2) < nq)+ qv[i] = *reinterpret_cast<const float4*>(q_idx + (q0 * IDX_HEADS + r) * IDX_DIM + kc + k4 * 4);+ }+ #pragma unroll+ for (int i = 0; i < KL; ++i) {+ const int idx = tid + i * 256, c = idx >> 3, k4 = idx & 7;+ kv[i] = make_float4(0.f, 0.f, 0.f, 0.f);+ if (b0 + c < reach) kv[i] = *reinterpret_cast<const float4*>(pooled + (b0 + c) * IDX_DIM + kc + k4 * 4);+ }+ };+ load(0);+ #pragma unroll 1+ for (int kc = 0; kc < IDX_DIM; kc += S2_KC) {+ #pragma unroll+ for (int i = 0; i < QL; ++i) {+ const int idx = tid + i * 256, r = idx >> 3, k4 = idx & 7;+ Qs[k4 * 4 + 0][r] = qv[i].x; Qs[k4 * 4 + 1][r] = qv[i].y; Qs[k4 * 4 + 2][r] = qv[i].z; Qs[k4 * 4 + 3][r] = qv[i].w;+ }+ #pragma unroll+ for (int i = 0; i < KL; ++i) {+ const int idx = tid + i * 256, c = idx >> 3, k4 = idx & 7;+ Ks[k4 * 4 + 0][c] = kv[i].x; Ks[k4 * 4 + 1][c] = kv[i].y; Ks[k4 * 4 + 2][c] = kv[i].z; Ks[k4 * 4 + 3][c] = kv[i].w;+ }+ __syncthreads();+ if (kc + S2_KC < IDX_DIM) load(kc + S2_KC); // in flight during this chunk's products+ #pragma unroll 4+ for (int k = 0; k < S2_KC; ++k) {+ float qr[RI], kr[CJ];+ #pragma unroll+ for (int m = 0; m < RI / 4; ++m) {+ const float4 a = *reinterpret_cast<const float4*>(&Qs[k][ty * RI + 4 * m]);+ qr[4 * m] = a.x; qr[4 * m + 1] = a.y; qr[4 * m + 2] = a.z; qr[4 * m + 3] = a.w;+ }+ #pragma unroll+ for (int g = 0; g < CJ / 4; ++g) {+ const float4 b = *reinterpret_cast<const float4*>(&Ks[k][g * 64 + tx * 4]);+ kr[4 * g] = b.x; kr[4 * g + 1] = b.y; kr[4 * g + 2] = b.z; kr[4 * g + 3] = b.w;+ }+ #pragma unroll+ for (int i = 0; i < RI; ++i)+ #pragma unroll+ for (int j = 0; j < CJ; ++j) acc[i][j] = fmaf(qr[i], kr[j], acc[i][j]);+ }+ __syncthreads();+ }+ #pragma unroll+ for (int qi = 0; qi < RI / 4; ++qi) {+ const int64_t q = q0 + ty * (RI / 4) + qi;+ if (q >= nq) continue;+ const int64_t n_bid = steps[q * kStepCount + kStepNBid];+ #pragma unroll+ for (int j = 0; j < CJ; ++j) {+ const int64_t b = b0 + (j >> 2) * 64 + tx * 4 + (j & 3);+ if (b >= n_bid || b >= max_blocks) continue;+ float sc = 0.0f;+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float d = acc[qi * 4 + h][j];+ sc += d > 0.0f ? d : 0.0f;+ }+ out[q * max_blocks + b] = sc;+ }+ }+ }+// ---- the same top-k with each query's keys read once: 1,024 threads hold up to TK_PER consecutive blocks' keys in// registers (contexts up to 4 * 1024 * TK_PER cells), per-warp histograms, block-wide scans. The selection rule is// block_topk_kernel's (radix threshold, ties to the lowest index, cells ascending): identical ids.@@@ -752,8 -794,9 +849,11 @@@ __global__ void __launch_bounds__(TK_T// neighbours at a time. The cells are emitted ascending in the order (warp, row, lane): warp w holds the `per` rows// of 32 consecutive blocks from block w * 32 * per. block_topk_kernel's selection rule (radix threshold, ties to the// lowest index): identical ids.+// campaign3: also compiled on HIP (gfx11: plain shared/atomic code, no sm_90 features; the dispatch still needs+// reach > fit, i.e. the deep contexts where the ref radix was the only option on AMD).+ constexpr int TS_MAXQ = 4096;+ __device__ int g_tk_fail[TS_MAXQ]; // block_topk_sample_kernel: 1 = the query is block_topk_wide_kernel<true>'s+ template <bool ONLY_FAILED = false>__global__ void __launch_bounds__(TK_T) block_topk_wide_kernel(const float* __restrict__ scores,const int32_t* __restrict__ steps, int64_t max_blocks,int64_t cap, int32_t* __restrict__ ids) {@@@ -928,214 -969,91 +1029,299 @@@ __device__ __forceinline__ void block_s}}+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit, QsaScoreRows srow = {}) {+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);+}++// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).+struct ScoreSlotKeys {+ const float* pooled[MQ];+ const float* dead[MQ];+};+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit) {+ const unsigned z = blockIdx.y;+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory+ const float* pooled = keys.pooled[0];+ const float* dead = keys.dead[0];+#pragma unroll+ for (unsigned i = 1; i < MQ; ++i)+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,+ QsaScoreRows{});+}++// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.+template <int NQ>+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out) {+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];+ if (threadIdx.x < nq) {+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];+ }+ __syncthreads();+ int64_t top = 0;+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)+ const int lane = threadIdx.x & 31;+ float4 qr[NQ][IDX_HEADS];+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi)+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)+ : make_float4(0.f, 0.f, 0.f, 0.f);+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi) {+ if (qi >= nq) break;+ const int64_t n_bid = s_nbid[qi];+ if (b > n_bid) continue;+ const float4 k4 = (b == n_bid) ? kd : kp;+ float d[IDX_HEADS];+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float4 q4 = qr[qi][h];+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ }+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head+ float r2[2];+#pragma unroll+ for (int i = 0; i < 2; ++i)+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);+ r += __shfl_xor_sync(0xffffffffu, r, 4);+ r += __shfl_xor_sync(0xffffffffu, r, 2);+ r += __shfl_xor_sync(0xffffffffu, r, 1);+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel+ const float s1 = __shfl_sync(0xffffffffu, r, 8);+ const float s2 = __shfl_sync(0xffffffffu, r, 16);+ const float s3 = __shfl_sync(0xffffffffu, r, 24);+ if (lane == 0) {+ float score = 0.0f;+ score += r > 0.0f ? r : 0.0f;+ score += s1 > 0.0f ? s1 : 0.0f;+ score += s2 > 0.0f ? s2 : 0.0f;+ score += s3 > 0.0f ? s3 : 0.0f;+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;+ out[qi * max_blocks + b] = score;+ }+ }+ }+}++#if defined(__HIPCC__)+// Optional gfx1100 port of PR1825 (14bf3215): same FP32 expression and reduction tree.+// Decode and the original top-k stay unchanged. No additional persistent weight/state allocation.+constexpr int TS_Q = 64; // queries per CTA (8 per wave, four lanes per query)+constexpr int TS_KB = 32; // key blocks per shared-memory tile+constexpr int TS_BR = 512; // key blocks per CTA++__device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression+}++__global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int64_t nq,+ int64_t max_blocks, int64_t reach,+ float* __restrict__ out) {+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];+ __shared__ float so[TS_Q][TS_KB + 1];+ __shared__ int64_t s_nbid[TS_Q];+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer+ // shared-memory loads per FMA. The MIO-pipe motivation was measured by PR1825 on NVIDIA.+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};+ const int t = threadIdx.x, lane = t & 31;+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;+ float4 qv[IDX_HEADS][8];+ {+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+#pragma unroll+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];+ }+ __syncthreads();+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;+ int64_t b_hi = b_lo + TS_BR;+ if (b_hi > reach) b_hi = reach;+ if (b_hi > top) b_hi = top;+ if (b_hi > max_blocks) b_hi = max_blocks;+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);+ __syncthreads(); // the previous tile's reads of ks and so are done+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];+ __syncthreads();+ for (int j = 0; j < nb; ++j) {+ float4 kk[8];+#pragma unroll+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];+ float e[IDX_HEADS];+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) ++ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) ++ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) ++ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)+ }+ if (j4 == 0) {+ float score = 0.0f;+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;+ so[ql][j] = score;+ }+ }+ __syncthreads();+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {+ const int q = i / nb, j = i % nb;+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];+ }+ }+}++// The exact scorer runs on gfx1100 (the PR's own card) when STRATA_SCORES_THREAD=1, and on gfx12 (RDNA4) unless STRATA_SCORES_THREAD=0:+// there it is bitwise equal to the warp scorer on an R9700 (qsa_scores_gfx1100_parity --full, 35 cases; exact_pp residual dumps at 70K and+// 131K tokens) and 3.6-4.4x faster in the scorer stage, +8% (131K) to +15% (230K) on a prompt read. Other wave32 parts have no measurement.+// Returns 0 (not offered), 1 (gfx1100: only on request), 2 (gfx12: the default).+int scores_thread_arch() {+ int device = -1;+ if (cudaGetDevice(&device) != cudaSuccess) { cudaGetLastError(); return 0; }+ static thread_local int cached_device = -1;+ static thread_local int arch = 0;+ if (device != cached_device) {+ cudaDeviceProp properties{};+ arch = 0;+ if (cudaGetDeviceProperties(&properties, device) == cudaSuccess)+ arch = gfx_arch_is_gfx12(properties.gcnArchName) ? 2 : gfx_arch_is(properties.gcnArchName, "gfx1100") ? 1 : 0;+ cached_device = device;+ cudaGetLastError();+ }+ return arch;+}+#endif++ #if !defined(__HIPCC__)+ // ---- prompt calls on cards without the tensor-core scorer (Turing): block_scores_kernel's scores, BITWISE, with a+ // thread per (query, head) instead of a warp per (query, block). There a warp spends 20 shuffles and five 512-byte+ // loads per score; here a lane holds its query head (128 floats) in registers, every lane of the warp reads the same+ // key from shared memory (a broadcast), and the 4-value partials are added in the tree lane 0 of the xor-shuffle sum+ // computes: partial g covers values 4g..4g+3, level o adds g and g + o. Float addition is commutative, so the same+ // pairs in the same tree give the same bits. The heads are then summed 0..3 from 0.0f as there. Blocks < n_bid only:+ // the tail block n_bid (the `dead` key, +1e9) is block_scores_tail_kernel's.+ constexpr int TS_Q = 64; // queries per CTA (8 per warp, a lane per query head)+ constexpr int TS_KB = 32; // key blocks per shared-memory tile+ constexpr int TS_BR = 512; // key blocks per CTA++ __device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression+ }++ __global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int64_t nq,+ int64_t max_blocks, int64_t reach,+ float* __restrict__ out) {+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];+ __shared__ float so[TS_Q][TS_KB + 1];+ __shared__ int64_t s_nbid[TS_Q];+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer+ // shared-memory loads per FMA (the kernel was bound by the MIO pipe, ncu: mio_throttle first).+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};+ const int t = threadIdx.x, lane = t & 31;+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;+ float4 qv[IDX_HEADS][8];+ {+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ #pragma unroll+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];+ }+ __syncthreads();+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;+ int64_t b_hi = b_lo + TS_BR;+ if (b_hi > reach) b_hi = reach;+ if (b_hi > top) b_hi = top;+ if (b_hi > max_blocks) b_hi = max_blocks;+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);+ __syncthreads(); // the previous tile's reads of ks and so are done+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];+ __syncthreads();+ for (int j = 0; j < nb; ++j) {+ float4 kk[8];+ #pragma unroll+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];+ float e[IDX_HEADS];+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) ++ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) ++ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) ++ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)+ }+ if (j4 == 0) {+ float score = 0.0f;+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;+ so[ql][j] = score;+ }+ }+ __syncthreads();+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {+ const int q = i / nb, j = i % nb;+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];+ }+ }+ }+ #endif+#if !defined(__HIPCC__)// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to@@@ -1358,38 -1276,138 +1544,168 @@@ __global__ void __launch_bounds__(CL_T#endif}#endif // !__HIPCC__+bool scores_multi_on() {+ static const bool on = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();+ return on;+}+// STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits+bool scores_early_exit() {+ static const bool on = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();+ return on;+}} // namespace+bool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,+ const int32_t* steps, int64_t nq, int64_t max_blocks, const QsaShapes& s, float* scores,+ void* stream) {+ if (nq <= 0) return true;+ if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R) {+ std::fprintf(stderr, "qsa_block_scores_slots: unsupported indexer geometry\n");+ std::exit(1);+ }+ if (nq > MQ || !scores_multi_on()) return false; // the single-query call would not run the multi kernel+ ScoreSlotKeys keys{};+ for (int64_t z = 0; z < nq; ++z) {+ keys.pooled[z] = pooled[z];+ keys.dead[z] = dead[z];+ }+ block_scores_slots_kernel<<<dim3(256, (unsigned) nq), SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(+ keys, q_idx, steps, max_blocks, scores, scores_early_exit());+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_slots: %s\n", cudaGetErrorString(e)); std::exit(1); }+ return true;+}++ // block_scores_multi_kernel with the warp sums as one recursive-halving reduction. A (query, head) dot is a lane's+ // 4-term product summed over the 32 lanes by an xor butterfly (offsets 16, 8, 4, 2, 1): 5 shuffles per value, 20 per+ // (block, query) - on Turing the decode scorer's time grew with the window's queries (420K cells: 117 / 265 / 509 us+ // for 1 / 3 / 6 queries), the shuffles, not the keys. Here the V = 4 x queries values (padded to a power of 2) are+ // reduced together: at offset o each lane keeps half of its values and adds its partner's copy of them - the pair+ // (lane l, lane l ^ o) the butterfly adds for that value at that offset, so every value's additions are the+ // butterfly's, operand for operand (float addition commutes): the same bits, with V - 1 shuffles plus the plain+ // butterfly's remaining stages instead of 5 V. Lane l ends with value (l >> (5 - log2 V)); the query's 4 heads are+ // then gathered by its head-0 lane and summed in block_scores_multi_kernel's order. STRATA_SCORES_HALVE=0: the old kernel.+ template <int V>+ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_halve_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out) {+ constexpr int NQ = V / IDX_HEADS; // query slots (nq <= NQ)+ constexpr int LV = V == 4 ? 2 : V == 8 ? 3 : V == 16 ? 4 : 5, SH = 5 - LV;+ static_assert(V == 4 || V == 8 || V == 16 || V == 32, "V");+ __shared__ __align__(16) float qs[NQ * IDX_HEADS * IDX_DIM];+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];+ if (threadIdx.x < NQ) {+ s_nkv[threadIdx.x] = threadIdx.x < nq ? steps[threadIdx.x * kStepCount + kStepNKv] : 0;+ s_nbid[threadIdx.x] = threadIdx.x < nq ? steps[threadIdx.x * kStepCount + kStepNBid] : -1;+ }+ __syncthreads();+ int64_t top = 0;+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // CTA-uniform (as block_scores_multi_kernel's early exit)+ for (int i = threadIdx.x; i < NQ * IDX_HEADS * IDX_DIM; i += blockDim.x)+ qs[i] = i < nq * IDX_HEADS * IDX_DIM ? q_idx[i] : 0.0f;+ __syncthreads();+ const int lane = threadIdx.x & 31;+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);+ // up to 4 queries: the lane's query values in registers (read once, not once per block from shared memory: the+ // shared-memory reads were the kernel's limit, ncu mio_throttle)+ constexpr bool QREG = V <= 16;+ float4 qr[QREG ? V : 1];+ if constexpr (QREG) {+ #pragma unroll+ for (int i = 0; i < V; ++i) qr[i] = *reinterpret_cast<const float4*>(qs + (i >> 2) * IDX_HEADS * IDX_DIM + (i & 3) * IDX_DIM + lane * 4);+ }+ int s_nbid_r[NQ]; // n_bid < 2^31+ #pragma unroll+ for (int qi = 0; qi < NQ; ++qi) s_nbid_r[qi] = (int) s_nbid[qi];+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);+ float v[V];+ #pragma unroll+ for (int qi = 0; qi < NQ; ++qi) {+ const float4 k4 = (b == s_nbid_r[qi]) ? kd : kp;+ const float* q = qs + qi * IDX_HEADS * IDX_DIM + lane * 4;+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ float4 q4;+ if constexpr (QREG) q4 = qr[qi * IDX_HEADS + h];+ else q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);+ v[qi * IDX_HEADS + h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ }+ }+ // recursive halving over offsets 16, 8, ... while more than one value is left, then the plain butterfly+ int cnt = V;+ #pragma unroll+ for (int o = 16; o > 0; o >>= 1) {+ if (cnt > 1) {+ const int half = cnt >> 1;+ const bool hi = (lane & o) != 0;+ #pragma unroll+ for (int i = 0; i < V / 2; ++i) {+ if (i < half) {+ const float send = hi ? v[i] : v[i + half];+ const float keep = hi ? v[i + half] : v[i];+ const float recv = __shfl_xor_sync(0xffffffffu, send, o);+ v[i] = keep + recv;+ }+ }+ cnt = half;+ } else {+ v[0] += __shfl_xor_sync(0xffffffffu, v[0], o);+ }+ }+ // lane l holds value j = l >> SH = qi * 4 + h; the head-0 lane of each query gathers heads 1-3+ const int j = lane >> SH, qi = j >> 2;+ const float d1 = __shfl_sync(0xffffffffu, v[0], (lane + (1 << SH)) & 31);+ const float d2 = __shfl_sync(0xffffffffu, v[0], (lane + (2 << SH)) & 31);+ const float d3 = __shfl_sync(0xffffffffu, v[0], (lane + (3 << SH)) & 31);+ if ((j & 3) == 0 && (lane & ((1 << SH) - 1)) == 0 && qi < nq && b <= s_nbid_r[qi & (NQ - 1)]) {+ const float d[4] = {v[0], d1, d2, d3};+ float score = 0.0f;+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) score += d[h] > 0.0f ? d[h] : 0.0f;+ if (b == s_nbid_r[qi & (NQ - 1)] && s_nkv[qi] % R != 0) score += 1e9f;+ out[qi * max_blocks + b] = score;+ }+ }+ }++ // #1825: the prompt path's thread-per-head scorer (bitwise the warp kernel's scores). Default: Turing with+ // STRATA_SM75_PERF=1 (the P100 gain measured here is +1..3% at 30-60K tokens, under the 3% bar for a new default);+ // STRATA_SCORES_THREAD=1 turns it on, =0 off, on any CUDA card+ bool scores_thread_on() {+ static const int env = [] { const char* v = std::getenv("STRATA_SCORES_THREAD"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();+ if (env >= 0) return env != 0;+ static int ok[64] = {}; // 0 unknown, 1 yes, 2 no+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }+ if (ok[dev] == 0) {+ int major = 0, minor = 0;+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);+ ok[dev] = strata::sm75_default(major, minor) ? 1 : 2; // Turing, STRATA_SM75_PERF=1+ }+ return ok[dev] == 1;+ }++ bool scores_halve_on() {+ static const int env = [] { const char* v = std::getenv("STRATA_SCORES_HALVE"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();+ if (env >= 0) return env != 0;+ static int ok[64] = {}; // 0 unknown, 1 yes, 2 no+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }+ if (ok[dev] == 0) {+ int major = 0, minor = 0;+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);+ ok[dev] = strata::sm75_default(major, minor) ? 1 : 2; // Turing, STRATA_SM75_PERF=1+ }+ return ok[dev] == 1;+ }+void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {if (nq <= 0) return;@@@ -1398,45 -1416,48 +1714,76 @@@std::exit(1);}// a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)- static const bool multi = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();- if (multi && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16+ if (scores_multi_on() && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16+ // QSAROWS: STRATA_QSA_ROWS=1 (exactly) takes block_scores_rows_kernel for calls of up to 4 queries - the same bits+ static const bool rows_env = [] { const char* v = std::getenv("STRATA_QSA_ROWS"); return v != nullptr && v[0] == '1' && v[1] == '\0'; }();+ if (rows_env && qsa_block_scores_rows(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream)) return;+ if (scores_halve_on()) { // Turing: block_scores_halve_kernel, the same bits+ const cudaStream_t st = (cudaStream_t) stream;+ if (nq <= 1) block_scores_halve_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);+ else if (nq <= 2) block_scores_halve_kernel<8><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);+ else if (nq <= 4) block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);+ else { // 5-8 queries: two calls of up to 4 (the register form; each query's scores are its own)+ block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, 4, max_blocks, scores);+ const int rest = (int) nq - 4;+ const float* q2 = q_idx + 4 * IDX_HEADS * IDX_DIM;+ const int32_t* st2 = steps + 4 * kStepCount;+ float* sc2 = scores + 4 * max_blocks;+ if (rest <= 1) block_scores_halve_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);+ else if (rest <= 2) block_scores_halve_kernel<8><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);+ else block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);+ }+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores halve: %s\n", cudaGetErrorString(e)); std::exit(1); }+ return;+ }- // STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits- static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, (int) nq,- max_blocks, scores, early);+ max_blocks, scores, scores_early_exit());const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores multi: %s\n", cudaGetErrorString(e)); std::exit(1); }return;}const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;-#if !defined(__HIPCC__)+#if defined(__HIPCC__)+ // STRATA_SCORES_THREAD: 1 = on (gfx1100, gfx12), 0 = off, unset = on for gfx12 only (gfx1100: off, nothing measured here)+ static const int thread_env = [] {+ const char* value = std::getenv("STRATA_SCORES_THREAD");+ return value == nullptr ? -1 : (std::atoi(value) == 1 ? 1 : 0);+ }();+ // The bound is the ACTUAL live extent. A reserved 256K context never enables this by itself.+ if (thread_env != 0 && active_blocks >= 16385 && active_blocks <= max_blocks && nq >= 64 &&+ (scores_thread_arch() == 2 || (thread_env == 1 && scores_thread_arch() == 1))) {+ static thread_local bool logged = false;+ if (!logged && (thread_env == -1 || std::getenv("STRATA_SCORES_THREAD_TRACE"))) {+ std::fprintf(stderr, "qsa exact thread: the prompt's block scorer runs as the four-lanes-per-query kernel (nq=%lld active_blocks=%lld "+ "capacity=%lld; the same scores bit for bit; STRATA_SCORES_THREAD=0 turns it off)\n",+ (long long)nq, (long long)active_blocks, (long long)max_blocks);+ logged = true;+ }+ const dim3 tgrid((unsigned)((reach + TS_BR - 1) / TS_BR), (unsigned)((nq + TS_Q - 1) / TS_Q));+ block_scores_thread_kernel<<<tgrid, 256, 0, (cudaStream_t)stream>>>(pooled, q_idx, steps, nq,+ max_blocks, reach, scores);+ block_scores_tail_kernel<<<(unsigned)nq, 32, 0, (cudaStream_t)stream>>>(dead, q_idx, steps,+ max_blocks, scores);+ const cudaError_t error = cudaGetLastError();+ if (error != cudaSuccess) {+ std::fprintf(stderr, "qsa exact thread: %s\n", cudaGetErrorString(error));+ std::exit(1);+ }+ return;+ }++#else+ // prompt calls (an active-block count): the thread-per-head kernel, the same bits (opt-in: scores_thread_on); else the+ // warp kernel below+ if (active_blocks > 0 && scores_thread_on()) {+ const dim3 tgrid((unsigned) ((reach + TS_BR - 1) / TS_BR), (unsigned) ((nq + TS_Q - 1) / TS_Q));+ block_scores_thread_kernel<<<tgrid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks, reach,+ scores);+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores thread: %s\n", cudaGetErrorString(e)); std::exit(1); }+ return;+ }#endifconst dim3 grid((unsigned) ((reach + SCORE_WARPS - 1) / SCORE_WARPS), (unsigned) nq);block_scores_kernel<<<grid, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, max_blocks,@@@ -1535,8 -1513,31 +1883,28 @@@ bool qsa_block_scores_tc(const float* p}();if (!simt) return false;const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;+ if (simt == 2) {+ // STRATA_SIMT2_TILE=2: 64 x 64 tiles (measured slower: 400K 3.00 vs 2.82 ms)+ static const int tile = [] { const char* v = std::getenv("STRATA_SIMT2_TILE"); return v ? std::atoi(v) : 1; }();+ if (tile != 2) {+ const dim3 grid((unsigned) ((reach + 127) / 128), (unsigned) ((nq + 31) / 32));+ if (grid.y > 65535) return false;+ block_scores_simt2_kernel<32, 128><<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq,+ max_blocks, reach, scores);+ } else {+ const dim3 grid((unsigned) ((reach + 63) / 64), (unsigned) ((nq + 63) / 64));+ if (grid.y > 65535) return false;+ block_scores_simt2_kernel<64, 64><<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq,+ max_blocks, reach, scores);+ }- } else {- const dim3 grid((unsigned) ((reach + SM_NB - 1) / SM_NB), (unsigned) ((nq + SM_QT - 1) / SM_QT));- if (grid.y > 65535) return false;- block_scores_simt_kernel<<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks,- reach, scores);++ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks,++ scores);++ const cudaError_t e = cudaGetLastError();++ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }++ return true;+ }- block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks,- scores);- const cudaError_t e = cudaGetLastError();- if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }- return true;+ return block_scores_simt(pooled, dead, q_idx, steps, nq, max_blocks, reach, scores, (cudaStream_t) stream,+ false);}}static bool attr[64] = {}; // the shared-memory opt-in is per device (a layer split runs it on several)@@@ -1724,9 -2193,15 +2567,15 @@@ void qsa_block_topk(const float* scoresconst char* v = std::getenv("STRATA_TOPK_STREAM");return v == nullptr || v[0] != '0';}();- if (reach > fit || (turing_wide && counted && reach > 22528)) // past the register kernel's reach, or Turing's band- block_topk_wide_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);- else+ if (reach > fit || (turing_wide && counted && reach > 22528)) { // past the register kernel's reach, or Turing's band++#if !defined(__HIPCC__)+ if (counted && nq <= TS_MAXQ && reach > 28672 && topk_sample_on()) { // one read of the scores (below ~115K cells the wide kernel is as fast)+ block_topk_sample_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);+ block_topk_wide_kernel<true><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);- } else {- block_topk_wide_kernel<false><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);- }- } else++ } else+ #endif++ block_topk_wide_kernel<false><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);++ } elseif (reach <= (int64_t) TK_T * TK_PER)block_topk_reg_kernel<TK_PER><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);elsediff --cc src/kernels/qsa_prompt_attn_parity.cppindex 706fa13b1,5de9f38fa..0a7b8b097--- a/src/kernels/qsa_prompt_attn_parity.cpp+++ b/src/kernels/qsa_prompt_attn_parity.cpp@@@ -298,36 -285,18 +298,42 @@@ int run(int fmt, int64_t ctx, int64_t ncudaEventRecord(e1);ck(cudaEventSynchronize(e1), "time");cudaEventElapsedTime(&ms_new, e0, e1);- const bool ok1 = err_new <= std::max(4.0 * err_old, 1e-6 * ref_scale);+ bool ok_im = true;+ if (fmt == 1) { // the INT8-MMA kernel: the same bounds as v2's, against FP64 and against v2+ double d_iv = 0;+ for (size_t i = 0; i < o.size(); ++i) d_iv = std::max(d_iv, (double) std::fabs(im[i] - nw[i]));+ float ms_im = 0, ms_v2 = 0;+ for (int round = 0; round < 2; ++round) { // alternating, v2 then imma, twice; the second round counts+ cudaEventRecord(e0);+ for (int r = 0; r < reps; ++r) new_run();+ cudaEventRecord(e1);+ ck(cudaEventSynchronize(e1), "time");+ cudaEventElapsedTime(&ms_v2, e0, e1);+ cudaEventRecord(e0);+ for (int r = 0; r < reps; ++r) imma_run();+ cudaEventRecord(e1);+ ck(cudaEventSynchronize(e1), "time");+ cudaEventElapsedTime(&ms_im, e0, e1);+ }+ ok_im = err_im <= std::max(4.0 * err_old, 1e-6 * ref_scale) && d_iv <= 1e-4 * scale;+ std::printf("%s int8 IMMA ctx %lld, %lld queries: vs FP64 %.3g (v2 %.3g, FP32 %.3g); IMMA vs v2 %.3g (%.2g of scale); "+ "v2 %.3f -> IMMA %.3f ms per chunk (%.2fx)\n", ok_im ? "PASS" : "FAIL", (long long) ctx, (long long) nq,+ err_im, err_new, err_old, d_iv, d_iv / scale, ms_v2 / reps, ms_im / reps, ms_v2 / ms_im);+ }+ k::qsa_prompt_attn_set_imma(-1);+ cudaFree(d_im);+ const bool ok1 = err_new <= std::max(4.0 * err_old, 1e-6 * ref_scale) && ok_im;const bool ok2 = diff <= 1e-4 * scale;+ uint64_t sum = 1469598103934665603ull; // the new kernel's output bits: two builds of it compare by this+ for (float f : nw) {+ uint32_t b;+ std::memcpy(&b, &f, 4);+ sum = (sum ^ b) * 1099511628211ull;+ }std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "- "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",+ "scale); %.3f -> %.3f ms per chunk (%.2fx); new output checksum %016llx\n",ok1 && ok2 ? "PASS" : "FAIL", fmt == 2 ? "q4_0" : fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,- err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);+ err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new, (unsigned long long) sum);cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);cudaFree(scratch);cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);diff --cc src/prefill/gemm.cuindex 50ca2b142,9ad9eb105..538e17eb3--- a/src/prefill/gemm.cu+++ b/src/prefill/gemm.cu@@@ -816,99 -621,19 +816,110 @@@ bool Gemm::bf16_hcd_exact(const uint16_#endif}+#if defined(__HIPCC__)+// ---- RDNA2 (gfx103x): the prompt path's GEMMs as FP32 SGEMMs ------------------------------------------------------+// The GEMMs here take 16-bit inputs (FP16 or BF16) and write FP32 (rocBLAS's "HS" / "BS" types). The rocBLAS the engine+// ships has tuned gfx1030 kernels for FP16 -> FP16 ("HH"), int8 and FP32 -> FP32 ("SS") only: every 16-bit-input,+// FP32-output GEMM runs a "fallback" kernel at ~5 TFLOPS. Measured on an RX 6800 (ROCm 10.2.0a20260930) with the+// engine's exact calls (opA = T, opB = N, fp32 compute): at the Coder's prompt shapes the FP16 GEMMs of one fresh+// 12K-token prompt take 14.1 s, the same GEMMs as SGEMM on widened copies 4.7 s including the widening (e.g.+// N 10240 x T 8192 x K 2560: 86.7 -> 27.7 ms); the BF16 hyper-connection GEMMs 10.9 -> 7.5 ms (N 320, K 10240).+// Widening FP16 / BF16 to FP32 is exact and SGEMM accumulates in FP32 like the native call, so the products are the+// same up to summation order. The weight is widened once per call, the activations in slices of up to 128 MiB of+// FP32 (an 8,192-token chunk at K = 2560 is one slice). Small N (< 64: routers, gates) keeps the native call:+// there the widening costs more than the GEMM gains. If the FP32 buffers cannot be allocated, the native call runs.+// OPT-IN here: STRATA_RDNA2_SGEMM=1 turns it on for gfx103x devices (other cards keep the native call, also in a+// mixed layer split); =all turns it on for every AMD card (A/B arms, tests); unset or 0 is the native path. The+// author ran it as the gfx103x default (RX 6800 343 -> 534 tok/s); it is not bit-identical to the native call.+namespace {+__global__ void rdna2_bf16_to_f32_kernel(const uint16_t* __restrict__ in, float* __restrict__ out, int64_t n) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i < n) out[i] = __uint_as_float((uint32_t) in[i] << 16);+}+__global__ void rdna2_f16_to_f32_kernel(const __half* __restrict__ in, float* __restrict__ out, int64_t n) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i < n) out[i] = __half2float(in[i]);+}+void rdna2_widen(const uint16_t* in, float* out, int64_t n, bool bf16, cudaStream_t st) {+ if (n <= 0) return;+ const unsigned blocks = (unsigned) ((n + 255) / 256);+ if (bf16) rdna2_bf16_to_f32_kernel<<<blocks, 256, 0, st>>>(in, out, n);+ else rdna2_f16_to_f32_kernel<<<blocks, 256, 0, st>>>(reinterpret_cast<const __half*>(in), out, n);+}+// whether this GEMM runs as an SGEMM (see above)+bool rdna2_sgemm_on(int64_t N) {+ static const int mode = [] { // 0 off (unset), 1 gfx103x devices, 2 every AMD device+ const char* v = std::getenv("STRATA_RDNA2_SGEMM");+ if (v == nullptr || v[0] == '\0') return 0;+ if (std::strcmp(v, "all") == 0) return 2;+ return std::atoi(v) > 0 ? 1 : 0;+ }();+ if (mode == 0 || N < 64) return false;+ if (mode == 2) return true;+ static std::atomic<int> arch[64] = {}; // per device: 0 not known, 1 gfx103x, 2 another architecture+ int dev = 0;+ if (hipGetDevice(&dev) != hipSuccess || dev < 0 || dev >= 64) { (void) hipGetLastError(); return false; }+ int v = arch[dev].load(std::memory_order_relaxed);+ if (v == 0) {+ hipDeviceProp_t prop{};+ if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return false; }+ v = std::strncmp(prop.gcnArchName, "gfx103", 6) == 0 ? 1 : 2;+ arch[dev].store(v, std::memory_order_relaxed);+ }+ return v == 1;+}+// `have`, `want`: FP32 elements (the buffers are the instance's 2-byte tc_ buffers, sized in 2-byte elements)+bool rdna2_grow(uint16_t*& p, int64_t& have_u16, int64_t want_f32) {+ const int64_t want = 2 * want_f32;+ if (have_u16 >= want) return true;+ if (p) cudaFree(p);+ p = nullptr;+ have_u16 = 0;+ if (cudaMalloc((void**) &p, (size_t) want * 2) != cudaSuccess) { (void) cudaGetLastError(); p = nullptr; return false; }+ have_u16 = want;+ return true;+}+constexpr int64_t kRdna2SliceF32 = 32ll << 20; // 128 MiB of FP32 activations per slice+} // namespace++// Y^T[N, T] (+)= W[N, K] . X^T[K, T] as SGEMMs on FP32 copies; false: not this card / shape, or no buffers (the caller+// runs its native GEMM, nothing was written)+bool Gemm::rdna2_sgemm(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy,+ float beta, bool bf16) {+ if (K <= 0 || !rdna2_sgemm_on(N)) return false;+ const int64_t rows = std::max<int64_t>(1, std::min<int64_t>(T, kRdna2SliceF32 / K));+ // growing frees a buffer an earlier GEMM on this stream may still read (as the sm_60 FP32 route does)+ if (tc_w_elems_ < 2 * N * K || tc_x_elems_ < 2 * rows * K) (void) hipStreamSynchronize((hipStream_t) stream_);+ if (!rdna2_grow(tc_w_, tc_w_elems_, N * K) || !rdna2_grow(tc_x_, tc_x_elems_, rows * K)) return false;+ float* const wf = reinterpret_cast<float*>(tc_w_);+ float* const xf = reinterpret_cast<float*>(tc_x_);+ const float alpha = 1.0f;+ rdna2_widen(W, wf, N * K, bf16, (cudaStream_t) stream_);+ for (int64_t t0 = 0; t0 < T; t0 += rows) {+ const int64_t n = std::min<int64_t>(rows, T - t0);+ rdna2_widen(X + t0 * K, xf, n * K, bf16, (cudaStream_t) stream_);+ // every slice is a disjoint block of Y, so each gets the caller's beta+ ck(hipblasSgemm((hipblasHandle_t) handle_, HIPBLAS_OP_T, HIPBLAS_OP_N, (int) N, (int) n, (int) K, &alpha, wf,+ (int) K, xf, (int) K, &beta, Y + t0 * ldy, (int) ldy),+ "hipblasSgemm (RDNA2)");+ }+ return true;+}+#endif++ bool Gemm::bf16_converts_x() const {+ #if defined(__HIPCC__) || defined(STRATA_EXPERIMENTAL_SM60)+ return false;+ #else+ return bf16_path() == 1;+ #endif+ }+ __global__ void f16_to_bf16_kernel(const __half* __restrict__ in, uint16_t* __restrict__ out, int64_t n) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i < n) out[i] = (uint16_t) (__float_as_uint(__half2float(in[i])) >> 16); // exact: the value was a BF16+ }void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy,- float beta, int64_t ldx) {+ float beta, int64_t ldx, bool x_f16) {if (T <= 0 || N <= 0) return;if (ldy <= 0) ldy = N;if (ldx <= K) ldx = 0;diff --cc src/prefill/kernels.cuindex 9b79e4010,50178462a..29c9176cb--- a/src/prefill/kernels.cu+++ b/src/prefill/kernels.cu@@@ -42,30 -42,18 +42,40 @@@ __device__ __forceinline__ uint16_t bf(__device__ __forceinline__ uint16_t bf_lo(float f, uint16_t hi) { return bf(f - __uint_as_float((uint32_t) hi << 16)); }__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }++}// A SwiGLU product for an FP16 GEMM: saturated, so a token with a massive activation cannot turn into inf and then// NaN in the down projection (decode's q8_1 has room to ~8e6; FP16 ends at 65504). A NaN stays NaN (fminf/fmaxf// would make it -65504 and hide where it came from); finite values below 65504 round exactly as before.+// STRATA_F16_RANGE=1 (set_f16_range): what reaches FP16's range on this device - the largest |value| the activation+// images converted (hf_sat is their one funnel) and how many were beyond 65504 or NaN; W and Y are counted in gemm.cu.+// Off, the cost is one flag read per value. The max is kept as float bits (positive floats order as unsigned ints),+// and a thread takes the atomic only when it beats the value it read, so the atomics stop once the max has settled.+#if defined(__HIPCC__) // HIP only: the CUDA kernels stay exactly as they were (no flag read in hf_sat)+__device__ int g_f16_range_on = 0;+__device__ unsigned g_f16_range_max = 0;+__device__ unsigned long long g_f16_range_over = 0;+__device__ __forceinline__ void f16_range_note(float f) {+ const float a = fabsf(f);+ const unsigned bits = __float_as_uint(a);+ if (bits > g_f16_range_max) atomicMax(&g_f16_range_max, bits);+ if (a > 65504.0f || isnan(f)) atomicAdd(&g_f16_range_over, 1ull);+}+#endif+__device__ __forceinline__ uint16_t hf_sat(float f) {+#if defined(__HIPCC__)+ if (g_f16_range_on) f16_range_note(f);+#endif+ return hf(isnan(f) ? f : fminf(fmaxf(f, -65504.0f), 65504.0f));+}+++ // Turing's FP16 image of the hyper-connection's xn16 (gemm.cu's bf16_to_f16_kernel applied to a BF16 value): the+ // writers store what the BF16 tensor-core path would convert, so the GEMM reads it without a conversion pass+ __device__ __forceinline__ uint16_t bf_to_hf_conv(uint16_t b) {+ float f = __uint_as_float((uint32_t) b << 16);+ if (f > 65504.0f && !isinf(f)) f = 65504.0f;+ else if (f < -65504.0f && !isinf(f)) f = -65504.0f;+ return __half_as_ushort(__float2half_rn(f));+ }-__device__ __forceinline__ uint16_t hf_sat(float f) { return hf(isnan(f) ? f : fminf(fmaxf(f, -65504.0f), 65504.0f)); }// The prompt path's 16-bit activation image for the BF16-weight GEMMs: BF16, or FP16 where the GEMM library is fast// only in FP16 (prompt_f16() in gemm.cu: rocBLAS on gfx103x). Set once per device before the first prompt.#if defined(__HIPCC__) // HIP only (#835): the CUDA kernels stay exactly as they were, they never read the flagdiff --cc src/prefill/moe_fused_iq.cuindex 637f0ec3d,ce51c838e..bc870c892--- a/src/prefill/moe_fused_iq.cu+++ b/src/prefill/moe_fused_iq.cu@@@ -21,12 -21,8 +21,13 @@@// (SwiGLU, H to int8 per 32 features; down into the per-slot rows) are moe_fused.cu's.#include "strata/prefill/moe_fused_iq.hpp"#include "strata/kernels/gfx_arch.hpp"+ #include "strata/core/emulate.hpp"+#if defined(__HIPCC__)+#include "moe_f16_common.cuh"+#include "moe_f16_kq.hpp"+#endif+#include <cuda_fp16.h>#include <cuda_runtime.h>@@@ -1224,13 -1169,11 +1320,16 @@@ bool kq_on()}();return on;}+// STRATA_PF_F16=1 (gfx11): the formats native_f16_kernel takes (the K-quants are moe_f16_kq.cu's)+bool f16_gu(int t) {+ return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS;+}+bool f16_d(int t) { return t == T_Q2_0 || t == T_IQ4_NL || t == T_Q8_0; }bool gu_covered(int t) {return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS+ #if !defined(__HIPCC__)+ || (t == T_Q2_0 && dev_info().t75) // the canonical Q2_0 pack's gate/up: Turing opt-in only (sm_80+ keeps MMQ for it)+ #endif#if defined(__HIPCC__)|| ((t == T_Q4_K || t == T_Q5_K) && kq_on()) // the gfx11 kernels only#endif@@@ -1243,17 -1186,10 +1342,20 @@@ bool d_covered(int t)#endif;}+// d_covered on this device: STRATA_PF_F16=1 also takes Q8_0 down on gfx11 without STRATA_PF_FUSED_KQ - the F16 kernel+// has no minimum term, and its output already differs from the int8 kernels' (UD-IQ4_XS: the five Q8_0-down layers+// would otherwise leave the fused path for MMQ).+bool d_covered_on(int t, const DevInfo& d) {+#if defined(__HIPCC__)+ if (t == T_Q8_0 && !d.w12 && f16::requested()) return true;+#endif+ (void) d;+ return d_covered(t);+}+ #if defined(__HIPCC__)+ int host_astages() { return ASTAGES; } // (#2145 defines it for CUDA above; HIP keeps the one depth)+ #endiftemplate <int T, bool GU>void launch(int ww, unsigned grid, const Batch& b, const NativeGeom& g, const Tables& tb, const void* act,const int32_t* src, void* out, float* dm, cudaStream_t s) {@@@ -1300,9 -1236,9 +1402,10 @@@ bool native_supported(int gu_type, int#endifif (env < 0 && gu_type != T_IQ3_S && gu_type != T_IQ3_XXS) return false; // default: only these twoconst DevInfo& d = dev_info();+ if (!d.ok || !d_covered_on(d_type, d)) return false;+ if (d.t75 && (gu_type != T_Q2_0 || d_type != T_Q2_0)) return false;- if (env < 0) return d.ok && d.w12;- return d.ok && (requested() || d.w12);+ if (env < 0) return d.w12;+ return requested() || d.w12;}void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {diff --cc src/prefill/prefill.cppindex 2357ea5e4,fe7a37dbe..17428925e--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@@ -1384,8 -1224,10 +1385,12 @@@ bool Prefill::init(const core::WeightTam.f16_io = prompt_f16();m.gemm.set_f16_io(m.f16_io);set_act_f16(m.f16_io);+ static const bool f16_range = std::getenv("STRATA_F16_RANGE") != nullptr; // what reaches FP16's range+ if (f16_range) { set_f16_range(true); gemm_set_f16_range(true); }+ // the hyper-connection's xn16 straight as the FP16 the tensor-core GEMM reads (its two readers then skip the+ // conversion pass; the same values): STRATA_XN_F16=0 keeps the BF16 image+ static const bool xn_env = [] { const char* e = std::getenv("STRATA_XN_F16"); return e == nullptr || e[0] != '0'; }();+ m.xn_f16 = xn_env && !m.f16_io && m.gemm.bf16_converts_x();}if (!carve(T, &o)) {size_t fb = 0, tb = 0;@@@ -2251,26 -1992,10 +2256,26 @@@ bool native_proj(Gemm& gm, const core::return true;}bool bf16_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y, int64_t T, const std::string& name,- std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, int64_t ldx = 0, bool x_f16 = false) {+ std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, int64_t ldx = 0,- void* stream = nullptr) {++ void* stream = nullptr, bool x_f16 = false) {if (w->kind != core::WeightKind::Bf16InF32 || !w->data) { err = "prefill: " + name + " is not a resident BF16 tensor"; return false; }- gm.bf16(X, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx, x_f16);- if (X_lo) gm.bf16(X_lo, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f);+ const uint16_t* wb = (const uint16_t*) w->data;+ if (w->q8_scale != nullptr) {+ // STRATA_HC_REQ8: the GEMM takes bf16, so the int8 codes are expanded into one reused scratch on this stream+ static uint16_t* scratch = nullptr;+ static int64_t cap = 0;+ if (w->elements > cap) {+ if (scratch) cudaFree(scratch);+ if (cudaMalloc((void**) &scratch, (size_t) w->elements * 2) != cudaSuccess) {+ scratch = nullptr; cap = 0; err = "prefill: the STRATA_HC_REQ8 scratch"; return false;+ }+ cap = w->elements;+ }+ strata::kernels::hc_q8_to_bf16(w->data, w->q8_scale, scratch, (long long) w->elements, stream);+ wb = scratch;+ }- gm.bf16(X, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx);++ gm.bf16(X, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx, x_f16);+ if (X_lo) gm.bf16(X_lo, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f);return true;}@@@ -2964,10 -2641,10 +2970,10 @@@ bool Prefill::run_impl(const int64_t* t}hcd = true;}- if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs)) return false;- if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, xf16)) return false;++ if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs, xf16)) return false;gr_silu(m.lo, m.lo16, T, m.cs, m.lo16_lo);bool upmixed = false;- if (hc_upmix() && T >= pf_switch_min_t() && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo &&+ if (wu->q8_scale == nullptr && hc_upmix() && T >= std::max(pf_switch_min_t(), hc_upmix_min_t()) && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo &&wu->kind == core::WeightKind::Bf16InF32 && wu->data && wu->ne0 == LR && wu->ne1 == D) {static int checks = [] { const char* e = std::getenv("STRATA_HC_UPMIX_CHECK"); return e ? std::atoi(e) : 0; }();if (checks > 0) { // the default pair first, kept for the comparison@@@ -2993,8 -2670,8 +2999,8 @@@m.mixed, m.mixed_bf, m.mixed_h, T, m.cs);}}- if (!upmixed && !bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo)) return false;- if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, xf16)) return false;+ if (!upmixed && !bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo, 0, m.cs)) return false;- if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs)) return false;++ if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs, xf16)) return false;if (upmixed) {} else if (gr_unfused()) {gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h, m.mixed_bf_lo);