Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group cuda86 (f-145-pp-cuda86)
Niko12215a6f4ed5fbe1
diff --cc CMakeLists.txtindex fc480ac1e,04ebbc5ca..012d46c85--- a/CMakeLists.txt+++ b/CMakeLists.txt@@@ -417,15 -408,25 +417,34 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)+ # STRATA_QSA_SLOTS: a batch window's scores, top-k and attention as one launch over rows of different slots+ # against one call per row, bitwise (synthetic, no model)+ add_executable(qsa_slots_parity src/kernels/qsa_slots_parity.cpp)+ target_include_directories(qsa_slots_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_slots_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_slots_parity COMMAND qsa_slots_parity)+ add_test(NAME qsa_slots_parity_lanecell COMMAND qsa_slots_parity)+ set_tests_properties(qsa_slots_parity qsa_slots_parity_lanecell PROPERTIES SKIP_RETURN_CODE 77)+ set_tests_properties(qsa_slots_parity_lanecell PROPERTIES ENVIRONMENT "STRATA_ATTN_LANECELL=1")+ # fork: the Daily's decode attention (STRATA_ATTN_DAILY=1: PR #540's chunk kernel on every card and the bounded+ # merge) against upstream's kernels, bitwise; the production entries checked with the variable off and on+ add_executable(qsa_merge_daily_parity src/kernels/qsa_merge_daily_parity.cpp)+ target_include_directories(qsa_merge_daily_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_merge_daily_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_merge_daily_parity_default COMMAND qsa_merge_daily_parity)+ add_test(NAME qsa_merge_daily_parity_daily COMMAND qsa_merge_daily_parity)+ set_tests_properties(qsa_merge_daily_parity_default PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=0"+ SKIP_RETURN_CODE 77)+ set_tests_properties(qsa_merge_daily_parity_daily PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=1"+ SKIP_RETURN_CODE 77)+ endif()+ # QSAROWS (docs/QSAROWS.md): what the T rows of a verification window cost in a QSA layer's decode kernels at a long+ # context (block scores, top-k, attention over each row's selection, indexer append), row by row; GPU, synthetic, no+ # model. Not in the default build: `cmake --build <dir> --target qsa_rows_bench`.+ if(STRATA_ENABLE_CUDA)+ add_executable(qsa_rows_bench EXCLUDE_FROM_ALL src/kernels/qsa_rows_bench.cpp)+ target_include_directories(qsa_rows_bench PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_rows_bench PRIVATE strata_kernels CUDA::cudart)endif()# the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a# scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a@@@ -1586,14 -1549,13 +1621,20 @@@ if(STRATA_NATIVE_EXPERTStarget_link_libraries(prefill_fused_iq_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})add_test(NAME prefill_fused_iq_test COMMAND prefill_fused_iq_test)set_tests_properties(prefill_fused_iq_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)+ # the MoE input quantized per token and scattered to its rows (mmq::quantize_scatter) against the per-row+ # gather: the same q8_1 bytes for every MMQ layout (--bench: both at 2K, 8K and 32K tokens)+ add_executable(prefill_mmq_scatter_test tests/cuda/prefill_mmq_scatter_test.cpp)+ target_link_libraries(prefill_mmq_scatter_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})+ add_test(NAME prefill_mmq_scatter_test COMMAND prefill_mmq_scatter_test)+ set_tests_properties(prefill_mmq_scatter_test PROPERTIES TIMEOUT 120)endif()+ if(STRATA_BUILD_TESTS AND _strata_mmq_tab_on)+ # STRATA_MMQ_DIRECT: the pointer-table MMQ bit for bit against the gathered MMQ (exit 77 below sm_80)+ add_executable(prefill_mmq_tab_test tests/cuda/prefill_mmq_tab_test.cu)+ target_link_libraries(prefill_mmq_tab_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})+ add_test(NAME prefill_mmq_tab_test COMMAND prefill_mmq_tab_test)+ set_tests_properties(prefill_mmq_tab_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)+ endif()if(STRATA_BUILD_TESTS AND STRATA_MMQ_KQUANTS)# the UD-Q4_K_XL expert formats through MMQ against ggml's dequantized weights (synthetic, a few seconds)add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp)diff --cc include/strata/kernels/qsa_decode_attn.hppindex d68bab43e,c752b774d..3270d8b6f--- a/include/strata/kernels/qsa_decode_attn.hpp+++ b/include/strata/kernels/qsa_decode_attn.hpp@@@ -43,19 -43,16 +43,30 @@@ void qsa_decode_attn_step(const float*void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);+/// Up to this many queries of different slots in one qsa_decode_attn_slots call.+constexpr int kQsaAttnMaxSlots = 8;++/// One KV pool set per query of a qsa_decode_attn_slots call (passed by value as a kernel argument).+struct QsaAttnSlotPools {+ QsaAttnPools p[kQsaAttnMaxSlots];+};++/// qsa_decode_attn_batch for `n_q` queries that each attend over their OWN pools (`pools.p[i]`, one KV cache per batch+/// slot): one launch instead of one call per query, every block computing exactly what the per-query call computes.+/// The pools must all have one KV format. False (nothing launched) where the per-query call would run another kernel+/// (n_q > kQsaAttnMaxSlots, mixed formats, the pre-sm_75 / gfx103x or gfx906 variants): the caller loops instead.+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);+ /// For src/kernels/qsa_merge_daily_parity.cpp only: qsa_decode_attn_batch's two passes apart, whatever the environment+ /// says. `chunks_only` runs the chunk pass into `scratch` with `daily_chunk` 0: attn_chunk_kernel (upstream's default),+ /// 1: the Daily's (attn_chunk_kernel_pre75, what STRATA_ATTN_DAILY=1 picks on CUDA). `merge_only` merges those+ /// partials with `v2` 0: attn_merge_kernel, 1: attn_merge_v2_kernel (STRATA_ATTN_MERGE_V2, on with STRATA_ATTN_DAILY).+ /// Width must not exceed an unaligned cap (the chunk pass keeps its upstream contract); for aligned caps the grid+ /// bounds widths above cap. One input can go through every pair and the outputs be compared bit for bit.+ void qsa_decode_attn_chunks_only(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, int daily_chunk,+ void* stream);+ void qsa_decode_attn_merge_only(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch,+ float* attn, int64_t n_q, int v2, void* stream);} // namespace strata::kernelsdiff --cc include/strata/kernels/qsa_select.hppindex 3ff769a9a,0df308302..5d1331feb--- a/include/strata/kernels/qsa_select.hpp+++ b/include/strata/kernels/qsa_select.hpp@@@ -29,12 -29,12 +29,18 @@@ void qsa_block_scores(const float* poolint64_t active_blocks = -1); ///< perf-review C-1: > 0 launches only this many blocks (the///< batch's largest n_bid + 1; not for a captured graph)+/// qsa_block_scores for `nq` queries of different slots, query i scoring its own slot's keys (pooled[i], dead[i]; host+/// arrays of device pointers): one launch whose blocks each compute exactly what the single-query call computes.+/// False (nothing launched) where that call would run another kernel (nq > 8, STRATA_SCORES_MULTI=0): loop instead.+bool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,+ const int32_t* steps, int64_t nq, int64_t max_blocks, const QsaShapes& s, float* scores,+ void* stream);+ /// QSAROWS (docs/QSAROWS.md): block_scores_multi_kernel's scores for a call of up to 4 queries, bit for bit, with the+ /// queries in registers and the four heads reduce-scattered (six shuffles instead of twenty a block and query).+ /// qsa_block_scores takes it with STRATA_QSA_ROWS=1; false (nothing launched) above 4 queries or on another geometry.+ /// For qsa_rows_bench's parity and timing it is callable on its own.+ bool qsa_block_scores_rows(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream);/// The same scores on tensor cores (3xTF32, FP32-level accuracy but another summation order: not bitwise; the tail/// block n_bid is the warp kernel's arithmetic). For the prompt path; false (nothing launched) on another geometry.diff --cc include/strata/prefill/moe_mmq.hppindex 3d7b4ccdb,7bb46728a..fee662fcc--- a/include/strata/prefill/moe_mmq.hpp+++ b/include/strata/prefill/moe_mmq.hpp@@@ -46,15 -46,13 +46,21 @@@ void scale_gu_rows(float* gu, int64_t l/// The down product's rows [0, nrows) (`ld` floats each, relative `bounds`) times their expert's s_down - on the FP32/// output, not folded into up, where the hidden would sit near 1e-5 (FP16-subnormal block scales in q8 formats).void scale_down_rows(float* d, int64_t ld, const int32_t* bounds, int n, const float* tails, int64_t nrows, void* stream);+ /// The rows quantize(x, src, ...) writes for tokens x k_used rows in expert order, written token by token: token t's+ /// row of x is quantized once and stored at rows slot[t * k_used + k] (`slot` the inverse of `src`) - the same bytes,+ /// a k_used-th of the quantizations (llama.cpp's quantize_scatter_mmq_q8_1_cuda). STRATA_QUANT_GATHER=1: quantize()+ /// over `src`, as before.+ void quantize_scatter(const float* x, const int32_t* slot, const int32_t* src, void* xq, int ggml_type, int64_t cols,+ int64_t ld, int64_t tokens, int k_used, void* stream);+constexpr int kGatherGroupMax = 16;+/// STRATA_MMQ_DIRECT: each expert's own weight address (gate|up or down), in place of `w + z * expert_bytes` -+/// expert z < n <= kGatherGroupMax starts at p[z]. The launch (tiles, stream-k split, fixup) is the same as+/// with `w`: only the base address of each expert's matrix changes.+struct WeightTab {+ const void* p[kGatherGroupMax] = {};+};+/// One launch over n experts whose weights lie `expert_bytes` apart from `w`: for expert e, the activation rows/// [bounds[e], bounds[e+1]) of `xq` (bounds on the device, n+1 entries) times its [w_rows, w_cols] matrix into/// dst rows of the same indices (`ld_dst` floats apart, via `ids`: dst row = ids[row], an identity table works).diff --cc src/core/verify.cppindex 9a9830fe7,9237b0d07..8c945c4e1--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@@ -1300,9 -1334,126 +1361,126 @@@ bool Verifier::record_window(int T, cudIQ * ID, N, IQ * ID, n, df_side_[1]);norm_rope_on(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ, df_side_[1]);}- if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs);- else for (int t = tb; t < te; ++t)- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs);+ // STRATA_SIDE_MIXER=1 / =2 (fork Eddoursul 34a35db + b063c13): the indexer's key projection,+ // the tail's snapshot, its keys appended, its query projection, norm and RoPE, the block scores and the+ // top-k read only the mixer's input and the indexer's own state - =1 runs them on a branch beside the+ // attention's projections; =2 also the queries' chain on a second branch. Each step below is+ // the in-line code with its stream as a parameter; with the switch off they run on cs in the same order+ // as before (STRATA_DF_BRANCH, the same idea for fewer of these steps, is off whenever this is on).+ auto norm_rope_s = [&](float* data, const WeightRef* norm, int rows, int cols, const int32_t* pos, cudaStream_t ns) {+ if (fuse_nr && native_norm_rope_usable(cols, (int) s.n_rot)) {+ native_qsa_rms_norm_rope(data, cols, (const float*) norm->data, data, rows, cols, (int) s.n_rot, EPS,+ rope_scaling(), pos, ns);+ return;+ }+ norm_rope_on(data, norm, rows, cols, pos, ns);+ };+ auto idx_key = [&](cudaStream_t ns) {+ if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, ns);+ else for (int t = tb; t < te; ++t)+ bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, ns);+ };+ auto idx_snap = [&](cudaStream_t ns) {+ if (batch_rec_) { // snapshot each slot's indexer tail before its first proposed row- for (int t = tb; t < te; ++t)++ for (int t = tb; t < te && !keep_rows; ++t)+ if (t == tb || brow_[t] != brow_[t - 1])+ copy_from_mapped(tail_snap_b_ + ((size_t) brow_[t] * nQall + qi) * TS,+ slot_ss(t).qsa_states[qi].idx_tail, TS, ns);+ } else+ if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, ns);+ };+ auto idx_append = [&](cudaStream_t ns) {+ if (dec_batch && !g_no_batch_kv && !batch_rec_) {+ const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};+ native_qsa_indexer_append_steps(idx_raw + tb * ID, step_ + tb * kStepCount + kStepPos, kStepCount,+ n, 0, (const float*) wikn->data, EPS, ib, s, st.max_cells,+ rope_scaling(), ns);+ } else {+ for (int t = tb; t < te; ++t) {+ const QsaState& sx = slot_ss(t).qsa_states[qi];+ const QsaIndexerBuffers ib{sx.idx_tail, sx.idx_dead, sx.idx_pooled, sx.idx_block_pos};+ native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,+ (const float*) wikn->data, EPS, ib, s, sx.max_cells,+ rope_scaling(), ns);+ }+ }+ };+ auto idx_query = [&](cudaStream_t ns) {+ if (qb) {+ bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,+ N, IQ * ID, n, ns);+ norm_rope_s(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ, ns);+ } else+ for (int t = tb; t < te; ++t) {+ float* qx = qidx_ + t * IQ * ID;+ bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), ns);+ norm_rope_s(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH, ns);+ }+ };+ auto idx_select = [&](cudaStream_t ns) {+ qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,+ s, scores_ + (size_t) tb * max_blocks_, ns);+ qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,+ sel_ + (size_t) tb * cap_, ns);+ };+ // the attention's queries: the projection (mm on cs, which may read F4's interleaved copy; on a branch+ // F4's kernel too, its copy written on cs before the fork - bitwise the same), the q/gate+ // split with norm and RoPE, the rotation+ const bool q_il = side_proj && g_mmvq_il() &&+ strata::kernels::native_mmvq_il_supported(wq->native_type, n, (int) (NH * 2 * HD));+ auto q_chain = [&](cudaStream_t ns) -> bool {+ if (ns == cs) mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));+ else if (q_il) strata::kernels::native_mmvq_il(wq->native_type, wq->native_data, xq_, xil_,+ qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD), n, ns);+ else native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N,+ (int) (NH * 2 * HD), n, ns);+ if (qb) {+ if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)+ native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,+ qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,+ rope_scaling(), pos_ + tb * NH, ns);+ } else {+ copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, ns);+ norm_rope_s(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH, ns);+ }+ if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, ns); // <Hq, Hk> = <q, k>+ } else {+ for (int t = tb; t < te; ++t) {+ float* qc = qcur_ + t * NH * HD;+ if (fuse_nr) {+ native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,+ (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, ns);+ } else {+ if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,+ (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, ns) != cudaSuccess) {+ err = "verify: the q/gate split failed";+ return false;+ }+ norm_rope_s(qc, wqn, (int) NH, (int) HD, pos_ + t * NH, ns);+ }+ if (st.kv_rot) fwht256_inplace_cuda(qc, NH, ns); // <Hq, Hk> = <q, k>+ }+ }+ return true;+ };+ if (sidem) {+ if (!branch(mfork_, cs, side_, "fork")) return false;+ idx_key(side_);+ idx_snap(side_);+ idx_append(side_);+ idx_query(side_);+ idx_select(side_);+ if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the indexer's branch could not join"; return false; }+ }+ if (side_proj) { // the queries beside the keys and values (fork b063c13)+ if (q_il && !il_ready) { // the copy before the fork (mm(wk) / mm(wv) on cs reuse it)+ strata::kernels::native_q8_1_interleave(xq_, xil_, (int) N, n, cs);+ il_ready = true;+ }+ if (!branch(pfork_, cs, side2_, "fork") || !q_chain(side2_)) return false;+ if (cudaEventRecord(pjoin_, side2_) != cudaSuccess) { err = "verify: the queries' branch could not join"; return false; }+ }+ if (!sidem) idx_key(cs);stamp(l, 7, grp);mm(wk, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD));mm(wv, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD));@@@ -1387,82 -1519,11 +1546,47 @@@if (qbr) {join(1); // the indexer query, for the block scores} else {- mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));- if (qb) {- if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)- native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,- qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,- rope_scaling(), pos_ + tb * NH, cs);- } else {- // the q/gate split as a copy kernel: the window's chain stays kernel to kernel (PDL)- copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, cs);- norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);- }- if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, cs); // <Hq, Hk> = <q, k>- bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,- N, IQ * ID, n, cs);- norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);- } else {- for (int t = tb; t < te; ++t) {- float* qc = qcur_ + t * NH * HD;- if (fuse_nr) {- native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,- (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, cs);- } else {- if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,- (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {- err = "verify: the q/gate split failed";- return false;- }- norm_rope(qc, wqn, (int) NH, (int) HD, pos_ + t * NH);- }- if (st.kv_rot) fwht256_inplace_cuda(qc, NH, cs); // <Hq, Hk> = <q, k>- }- for (int t = tb; t < te; ++t) {- float* qx = qidx_ + t * IQ * ID;- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);- norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);- }- }+ if (!side_proj && !q_chain(cs)) return false;+ if (!sidem) idx_query(cs);}stamp(l, 10, grp);- if (batch_rec_) { // each row selects and attends over its own slot's K/V+ bool streamed = false; // a streamed KV cache keeps the per-row resolve-then-attend order+ if (batch_rec_ && g_qsa_slots)+ for (int t = tb; t < te; ++t) streamed = streamed || slot_ss(t).qsa_states[qi].kv_mode == 1;+ if (batch_rec_ && g_qsa_slots && !streamed) {+ // the rows' block scores, top-k and attention, up to 8 rows per launch. The rows only read+ // their slots' keys and K/V here and write their own scores/sel/scratch/attn rows, so running+ // each step for all rows before the next is the per-row loop's result+ for (int t0 = tb; t0 < te; t0 += kQsaAttnMaxSlots) {+ const int m = std::min(te - t0, kQsaAttnMaxSlots);+ const float* pooled[kQsaAttnMaxSlots];+ const float* dead[kQsaAttnMaxSlots];+ QsaAttnSlotPools px;+ for (int i = 0; i < m; ++i) {+ const QsaState& sx = slot_ss(t0 + i).qsa_states[qi];+ pooled[i] = sx.idx_pooled;+ dead[i] = sx.idx_dead;+ px.p[i] = qsa_attn_pools(sx);+ }+ // a launcher that would not run the per-row call's kernel launches nothing: per-row calls+ if (!qsa_block_scores_slots(pooled, dead, qidx_ + t0 * IQ * ID, step_ + t0 * kStepCount, m,+ max_blocks_, s, scores_ + (size_t) t0 * max_blocks_, cs))+ for (int t = t0; t < t0 + m; ++t)+ qsa_block_scores(pooled[t - t0], dead[t - t0], qidx_ + t * IQ * ID, step_ + t * kStepCount,+ 1, max_blocks_, s, scores_ + (size_t) t * max_blocks_, cs);+ qsa_block_topk(scores_ + (size_t) t0 * max_blocks_, step_ + t0 * kStepCount, m, max_blocks_,+ cap_, s, sel_ + (size_t) t0 * cap_, cs);+ if (!qsa_decode_attn_slots(qcur_ + t0 * NH * HD, px, sel_ + (size_t) t0 * cap_,+ step_ + t0 * kStepCount, cap_, s,+ attn_scratch_ + (size_t) t0 * attn_scratch_floats_,+ attn_ + t0 * NH * HD, m, cs))+ for (int t = t0; t < t0 + m; ++t)+ qsa_decode_attn_batch(qcur_ + t * NH * HD, px.p[t - t0], sel_ + (size_t) t * cap_,+ step_ + t * kStepCount, cap_, s,+ attn_scratch_ + (size_t) t * attn_scratch_floats_,+ attn_ + t * NH * HD, 1, cs);+ }+ } else if (batch_rec_) { // each row selects and attends over its own slot's K/Vfor (int t = tb; t < te; ++t) {const QsaState& sx = slot_ss(t).qsa_states[qi];qsa_block_scores(sx.idx_pooled, sx.idx_dead, qidx_ + t * IQ * ID, step_ + t * kStepCount, 1,diff --cc src/kernels/cuda/qsa_decode_attn.cuindex 85257e6cf,311ad0fa7..fe1ee65e5--- a/src/kernels/cuda/qsa_decode_attn.cu+++ b/src/kernels/cuda/qsa_decode_attn.cu@@@ -643,27 -726,33 +754,44 @@@ bool pre75_attn()#define STRATA_ATTN_CHUNK(M) attn_chunk_kernel<M>#endif+// S25: STRATA_ATTN_LANECELL=1 - the score phase one cell per thread (bit-identical scores, no shuffle reductions)+bool lane_cell_attn() {+ static const bool on = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();+ return on;+}++// 0 fp16, 1 int8, 2 q4_0, 3 int8 K with q4_0 V+int attn_kv_mode(const QsaAttnPools& p) {+ return p.k_q4 != nullptr ? 2 : (p.k_q != nullptr && p.v_q4 != nullptr ? 3 : (p.k_q != nullptr ? 1 : 0));+}+- } // namespace+ // STRATA_ATTN_MERGE_V2: 1 merges with attn_merge_v2_kernel, 0 with attn_merge_kernel. Unset it follows+ // STRATA_ATTN_DAILY (the Daily had it on by default on NVIDIA), so upstream's merger stays the default.+ int attn_merge_v2() {+ static const int on = [] {+ int r = daily_attn() ? 1 : 0;+ const char* v = std::getenv("STRATA_ATTN_MERGE_V2");+ if (v != nullptr && *v != '\0') r = std::atoi(v) != 0 ? 1 : 0;+ return r;+ }();+ return on;+ }- void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,- int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {- if (n_q <= 0) return;- if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps ||- !pools.page_table || n_q > 65535) {- std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");- std::exit(1);- }+ // the merge of `n_q` queries' partials: v2 = 0 attn_merge_kernel, 1 attn_merge_v2_kernel (bitwise the same output)+ void launch_merge(int v2, const float* part_acc, const float* part_m, const float* part_l, int n_chunks,+ const int32_t* steps, float* attn, long long stride, unsigned n_head, unsigned n_q, cudaStream_t st) {+ if (v2)+ attn_merge_v2_kernel<kMergeU><<<dim3(n_head, n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks, steps,+ attn, stride);+ else+ attn_merge_kernel<<<dim3(n_head, n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks, attn, stride);+ }++ // qsa_decode_attn_batch's chunk pass: `n_q` queries' partials into `scratch` (one stride each). `kernel` < 0 is the+ // environment's choice (lane cell, gfx906 swizzle, STRATA_ATTN_DAILY / PR #540); 0 is attn_chunk_kernel and 1 PR #540's+ // kernel where it is built (the parity harness: never the environment).+ void launch_chunks_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, cudaStream_t st, int kernel) {const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3: (pools.k_q != nullptr ? 1 : 0));const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);@@@ -674,8 -763,12 +802,11 @@@float* part_l = part_m + (size_t) n_chunks * s.n_head;const float scale = 1.0f / sqrtf((float) HD);const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);- cudaStream_t st = (cudaStream_t) stream;- const bool lane_cell = lane_cell_attn();+ #if defined(STRATA_ATTN_PRE75_BUILT)+ const bool pre = kernel < 0 ? pre75_attn() : kernel == 1;+ #endif+ // S25: STRATA_ATTN_LANECELL=1 - the score phase one cell per thread (bit-identical scores, no shuffle reductions)- static const bool lane_cell_env = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();- const bool lane_cell = kernel < 0 && lane_cell_env;++ const bool lane_cell = kernel < 0 && lane_cell_attn();#if defined(STRATA_HIP_GFX906)static const bool query_swizzle = [] {const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';@@@ -748,64 -870,6 +908,64 @@@ void qsa_decode_attn_batch(const float}}+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {+ if (n_q <= 0) return true;+ if (n_q > kQsaAttnMaxSlots) return false;+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps) {+ std::fprintf(stderr, "qsa_decode_attn_slots: unsupported geometry or missing buffers\n");+ std::exit(1);+ }+ const int kv_mode = attn_kv_mode(pools.p[0]);+ for (int64_t z = 0; z < n_q; ++z) {+ if (!pools.p[z].page_table) {+ std::fprintf(stderr, "qsa_decode_attn_slots: missing page table\n");+ std::exit(1);+ }+ if (attn_kv_mode(pools.p[z]) != kv_mode) return false;+ }+ const bool lane_cell = lane_cell_attn();+ // only where qsa_decode_attn_batch runs attn_chunk_kernel; its other variants keep the per-query calls+#if defined(STRATA_ATTN_PRE75_BUILT)+ if (!lane_cell && pre75_attn()) return false;+#endif+#if defined(STRATA_HIP_GFX906)+ static const bool query_swizzle = [] {+ const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';+ }();+ if (query_swizzle && !lane_cell && kv_mode == 1 && query_swizzle_gfx906_device()) return false;+#endif+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);+ float* part_acc = scratch;+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;+ float* part_l = part_m + (size_t) n_chunks * s.n_head;+ const float scale = 1.0f / sqrtf((float) HD);+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);+ cudaStream_t st = (cudaStream_t) stream;+#define STRATA_ATTN_SLOTS(M, LC) attn_chunk_slots_kernel<M, LC><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)+ if (lane_cell) {+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, true);+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, true);+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, true);+ else STRATA_ATTN_SLOTS(0, true);+ } else {+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, false);+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, false);+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, false);+ else STRATA_ATTN_SLOTS(0, false);+ }+#undef STRATA_ATTN_SLOTS- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,- attn, stride);++ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,++ (unsigned) n_q, st);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "qsa_decode_attn_slots: %s\n", cudaGetErrorString(e));+ std::exit(1);+ }+ return true;+}+uint64_t qsa_decode_attn_scratch_floats(int64_t cap, const QsaShapes& s) {const int64_t chunks = (cap + CHUNK - 1) / CHUNK;return (uint64_t) chunks * (uint64_t) s.n_head * (HD + 2) + 64;diff --cc src/kernels/cuda/qsa_select.cuindex d55141722,8995937d9..39db4f65d--- a/src/kernels/cuda/qsa_select.cu+++ b/src/kernels/cuda/qsa_select.cu@@@ -867,37 -883,82 +882,113 @@@ __device__ __forceinline__ void block_s}}+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit) {+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit);+}++// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).+struct ScoreSlotKeys {+ const float* pooled[MQ];+ const float* dead[MQ];+};+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit) {+ const unsigned z = blockIdx.y;+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory+ const float* pooled = keys.pooled[0];+ const float* dead = keys.dead[0];+#pragma unroll+ for (unsigned i = 1; i < MQ; ++i)+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit);+}++ // QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue+ // pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a+ // call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory+ // (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two+ // pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices+ // of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products+ // are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered+ // to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the+ // same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the+ // heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.+ template <int NQ>+ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out) {+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];+ if (threadIdx.x < nq) {+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];+ }+ __syncthreads();+ int64_t top = 0;+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)+ const int lane = threadIdx.x & 31;+ float4 qr[NQ][IDX_HEADS];+ #pragma unroll+ for (int qi = 0; qi < NQ; ++qi)+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)+ : make_float4(0.f, 0.f, 0.f, 0.f);+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);+ #pragma unroll+ for (int qi = 0; qi < NQ; ++qi) {+ if (qi >= nq) break;+ const int64_t n_bid = s_nbid[qi];+ if (b > n_bid) continue;+ const float4 k4 = (b == n_bid) ? kd : kp;+ float d[IDX_HEADS];+ #pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float4 q4 = qr[qi][h];+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ }+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head+ float r2[2];+ #pragma unroll+ for (int i = 0; i < 2; ++i)+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);+ r += __shfl_xor_sync(0xffffffffu, r, 4);+ r += __shfl_xor_sync(0xffffffffu, r, 2);+ r += __shfl_xor_sync(0xffffffffu, r, 1);+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel+ const float s1 = __shfl_sync(0xffffffffu, r, 8);+ const float s2 = __shfl_sync(0xffffffffu, r, 16);+ const float s3 = __shfl_sync(0xffffffffu, r, 24);+ if (lane == 0) {+ float score = 0.0f;+ score += r > 0.0f ? r : 0.0f;+ score += s1 > 0.0f ? s1 : 0.0f;+ score += s2 > 0.0f ? s2 : 0.0f;+ score += s3 > 0.0f ? s3 : 0.0f;+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;+ out[qi * max_blocks + b] = score;+ }+ }+ }+ }+#if !defined(__HIPCC__)// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to@@@ -1160,9 -1191,15 +1251,12 @@@ void qsa_block_scores(const float* poolstd::exit(1);}// a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)- static const bool multi = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();- if (multi && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16+ if (scores_multi_on() && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16+ // QSAROWS: STRATA_QSA_ROWS=1 (exactly) takes block_scores_rows_kernel for calls of up to 4 queries - the same bits+ static const bool rows_env = [] { const char* v = std::getenv("STRATA_QSA_ROWS"); return v != nullptr && v[0] == '1' && v[1] == '\0'; }();+ if (rows_env && qsa_block_scores_rows(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream)) return;- // STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits- static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, (int) nq,- max_blocks, scores, early);+ max_blocks, scores, scores_early_exit());const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores multi: %s\n", cudaGetErrorString(e)); std::exit(1); }return;diff --cc src/prefill/prefill.cppindex 3d6d0ec14,01e870500..49ea499cc--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@@ -336,78 -336,9 +344,80 @@@ inline int bf16x2_mode(bool f16_io)return f16_io ? 0 : v;}inline bool bf16x2(bool f16_io) { return bf16x2_mode(f16_io) != 0; }+// STRATA_HANDOFF_PIPE=P (opt-in, 2..16; unset/0/1 = the one copy): a layer split hands a chunk's rows on in P row pieces.+// The stage records an event when its chunk's work is done and one after each piece's copy into the pinned hand-off+// buffer, waits on the host only for the first (what on_stage_chunk's checkpoint parts and the error check need), and+// starts the next stage at once: that stage's upload of piece i waits on piece i's event on the GPU, so its uploads+// overlap the remaining downloads (separate PCIe links; the 3090s here have no P2P) and the thread start is hidden+// behind them. The same bytes land in the same rows before the next stage's first kernel: exact. No device memory;+// the events and the existing two pinned buffers per stage only.+static constexpr int kHandPiecesMax = 16;+static int handoff_pipe_pieces() {+ static const int v = [] {+ const char* e = std::getenv("STRATA_HANDOFF_PIPE");+ const int p = e ? std::atoi(e) : 0;+ return p >= 2 ? std::min(p, kHandPiecesMax) : 0;+ }();+ return v;+}+// rows [r0, r0 + nr) of piece i of a T-row chunk in P pieces (the same split on both sides of a hand-off)+static void handoff_piece(int64_t T, int P, int i, int64_t& r0, int64_t& nr) {+ const int64_t per = (T + P - 1) / P;+ r0 = std::min<int64_t>(T, (int64_t) i * per);+ nr = std::min<int64_t>(T, r0 + per) - r0;+}+// STRATA_HANDOFF_PIPE: the next stage's chunks run on one thread of the stage's own, started once and pinned once+// (std::async started a thread per chunk: ~1 ms from the hand-off to the next stage's first call, measured with nsys).+// The futures are the same (run_impl's result, or what it threw), so the chain's waits do not change.+// (STRATA_HANDOFF_PIPE_WORKER=0: a thread per chunk again, for an A/B of the two halves.)+static bool handoff_pipe_worker() {+ static const bool v = [] { const char* e = std::getenv("STRATA_HANDOFF_PIPE_WORKER"); return !e || std::atoi(e) != 0; }();+ return handoff_pipe_pieces() > 0 && v;+}+struct HandWorker {+ std::mutex mu;+ std::condition_variable cv;+ std::deque<std::packaged_task<bool()>> jobs;+ bool stop = false;+ std::thread th;+ HandWorker() {+ th = std::thread([this] {+ strata::aux_cpus::pin_current_thread();+ for (;;) {+ std::packaged_task<bool()> job;+ {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [this] { return stop || !jobs.empty(); });+ if (jobs.empty()) return; // stopped, and every job taken has run (no broken promise)+ job = std::move(jobs.front());+ jobs.pop_front();+ }+ job();+ }+ });+ }+ std::future<bool> submit(std::packaged_task<bool()> job) {+ std::future<bool> f = job.get_future();+ {+ std::lock_guard<std::mutex> lk(mu);+ jobs.push_back(std::move(job));+ }+ cv.notify_one();+ return f;+ }+ ~HandWorker() {+ {+ std::lock_guard<std::mutex> lk(mu);+ stop = true;+ }+ cv.notify_one();+ if (th.joinable()) th.join();+ }+};+- // S23 (opt-in STRATA_HC_UPMIX=1): the hyper-connection read's up projection and gr_mix_r as one kernel (gr_upmix, gfx11);+ // S23 (opt-in STRATA_HC_UPMIX=1): the hyper-connection read's up projection and gr_mix_r as one kernel (gr_upmix: gfx11,+ // and CUDA sm_80+ on chunks of 33+ tokens - there, with CUDA 13.3 on an RTX 5090, cuBLAS's up GEMM sums K in the fused+ // kernel's order and the read is the default pair's bits: gr_upmix_parity, STRATA_HC_UPMIX_CHECK);// STRATA_HC_UPMIX_CHECK=N also runs the default pair on the first N reads and reports the difference of `mixed`static int64_t pf_switch_min_t() { // S23: STRATA_PF_SWITCH_MIN_T=N - the rounding-level prompt switches only on chunks of// N or more tokens (shorter prompts keep the default numerics, and their outputs; the switches pay on long ones)@@@ -4055,38 -3925,7 +4105,38 @@@ bool Prefill::run_impl(const int64_t* tconst size_t q = j % MMQ_GROUP;// the group's last expert also zeroes the MMQ tail after its slot (see MMQ_TAIL)const bool last_of_group = q + 1 == MMQ_GROUP || j + 1 == order.size();+ if (direct) {+ const auto& f = lay.fmt[(size_t) l];+ uint8_t* gp = cslot >= 0 && ((uintptr_t) blob_dev | (uintptr_t) f.down_off) % 16 == 0+ ? m.cache->slot_tail(cslot, f.bytes, mmq_plan().d_over[(size_t) l])+ : nullptr;+ if (gp) { // read in its slot: the guard after the blob is zeroed at the group end+ tgu.p[q] = blob_dev;+ td.p[q] = blob_dev + f.down_off;+ gd.p[gd.n++] = gp;+ ++stats_.experts_direct;+ } else {+ const int c = ns++;+ tgu.p[q] = m.grp_gu + (size_t) c * mmq_gub;+ td.p[q] = m.grp_d + (size_t) c * mmq_db;- if (group_gather) {++ if (gg_now) {+ gg.blob[c] = blob_dev;+ gg.n = c + 1;+ if (slot >= 0) gg_slots[gg_nslots++] = slot;+ } else {+ mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off,+ mmq_db, m.grp_gu + (size_t) c * mmq_gub,+ m.grp_d + (size_t) c * mmq_db, m.cs);+ }+ }- if (slot >= 0 && !group_gather) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }++ if (slot >= 0 && !gg_now) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }+ if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;- if (group_gather) { // also when the last entry was direct (a no-op without gathers)++ if (gg_now) { // also when the last entry was direct (a no-op without gathers)+ flush(0);+ gg = mmq::GatherGroup{};+ }+ } else {- if (group_gather) {+ if (gg_now) {gg.blob[q] = blob_dev;gg.n = (int) q + 1;if (slot >= 0) gg_slots[gg_nslots++] = slot;@@@ -4107,9 -3946,8 +4157,9 @@@cudaMemsetAsync(m.grp_d + (size_t) (q + 1) * mmq_db, 0, MMQ_TAIL, m.cs);}}- if (slot >= 0 && !group_gather) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }+ if (slot >= 0 && !gg_now) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;+ }// the group's products: gate/up, swiglu, the group's H to q8_1, downconst size_t j0 = j - q, g = j0 / MMQ_GROUP, n = order.size();const int ngx = (int) (q + 1);@@@ -4193,12 -4016,13 +4243,14 @@@const int32_t e = order[j];if (stage_of[j] < 0) {++stats_.experts_resident;- if (!compute(j, m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]), -1)) return false;+ if (!compute(j, m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]), -1,+ m.host_res[(size_t) l * m.g->n_expert + e])) return false;} else {- pt.mark(kPfWaitCopy, cs);- cudaStreamWaitEvent(m.cs, m.copied[stage_of[j]], 0);+ if (!group_resident) { // grouped: the flush waits once, on the group's last copy+ pt.mark(kPfWaitCopy, cs);+ cudaStreamWaitEvent(m.cs, m.copied[stage_of[j]], 0);+ }- if (!compute(j, m.stage_dev[stage_of[j]], stage_of[j])) return false;+ if (!compute(j, m.stage_dev[stage_of[j]], stage_of[j], -1)) return false;--pending; // compute recorded the slot's release event before any reuse}}