Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group cuda86 (f-145-pp-cuda86)

Niko12215a6f4ed5fbe1
diff --cc CMakeLists.txt
index fc480ac1e,04ebbc5ca..012d46c85
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -417,15 -408,25 +417,34 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_
target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)
add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)
set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)
+ # STRATA_QSA_SLOTS: a batch window's scores, top-k and attention as one launch over rows of different slots
+ # against one call per row, bitwise (synthetic, no model)
+ add_executable(qsa_slots_parity src/kernels/qsa_slots_parity.cpp)
+ target_include_directories(qsa_slots_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_slots_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_slots_parity COMMAND qsa_slots_parity)
+ add_test(NAME qsa_slots_parity_lanecell COMMAND qsa_slots_parity)
+ set_tests_properties(qsa_slots_parity qsa_slots_parity_lanecell PROPERTIES SKIP_RETURN_CODE 77)
+ set_tests_properties(qsa_slots_parity_lanecell PROPERTIES ENVIRONMENT "STRATA_ATTN_LANECELL=1")
+ # fork: the Daily's decode attention (STRATA_ATTN_DAILY=1: PR #540's chunk kernel on every card and the bounded
+ # merge) against upstream's kernels, bitwise; the production entries checked with the variable off and on
+ add_executable(qsa_merge_daily_parity src/kernels/qsa_merge_daily_parity.cpp)
+ target_include_directories(qsa_merge_daily_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_merge_daily_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_merge_daily_parity_default COMMAND qsa_merge_daily_parity)
+ add_test(NAME qsa_merge_daily_parity_daily COMMAND qsa_merge_daily_parity)
+ set_tests_properties(qsa_merge_daily_parity_default PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=0"
+ SKIP_RETURN_CODE 77)
+ set_tests_properties(qsa_merge_daily_parity_daily PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=1"
+ SKIP_RETURN_CODE 77)
+ endif()
+ # QSAROWS (docs/QSAROWS.md): what the T rows of a verification window cost in a QSA layer's decode kernels at a long
+ # context (block scores, top-k, attention over each row's selection, indexer append), row by row; GPU, synthetic, no
+ # model. Not in the default build: `cmake --build <dir> --target qsa_rows_bench`.
+ if(STRATA_ENABLE_CUDA)
+ add_executable(qsa_rows_bench EXCLUDE_FROM_ALL src/kernels/qsa_rows_bench.cpp)
+ target_include_directories(qsa_rows_bench PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_rows_bench PRIVATE strata_kernels CUDA::cudart)
endif()
# the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a
# scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a
@@@ -1586,14 -1549,13 +1621,20 @@@ if(STRATA_NATIVE_EXPERTS
target_link_libraries(prefill_fused_iq_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
add_test(NAME prefill_fused_iq_test COMMAND prefill_fused_iq_test)
set_tests_properties(prefill_fused_iq_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
+ # the MoE input quantized per token and scattered to its rows (mmq::quantize_scatter) against the per-row
+ # gather: the same q8_1 bytes for every MMQ layout (--bench: both at 2K, 8K and 32K tokens)
+ add_executable(prefill_mmq_scatter_test tests/cuda/prefill_mmq_scatter_test.cpp)
+ target_link_libraries(prefill_mmq_scatter_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+ add_test(NAME prefill_mmq_scatter_test COMMAND prefill_mmq_scatter_test)
+ set_tests_properties(prefill_mmq_scatter_test PROPERTIES TIMEOUT 120)
endif()
+ if(STRATA_BUILD_TESTS AND _strata_mmq_tab_on)
+ # STRATA_MMQ_DIRECT: the pointer-table MMQ bit for bit against the gathered MMQ (exit 77 below sm_80)
+ add_executable(prefill_mmq_tab_test tests/cuda/prefill_mmq_tab_test.cu)
+ target_link_libraries(prefill_mmq_tab_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
+ add_test(NAME prefill_mmq_tab_test COMMAND prefill_mmq_tab_test)
+ set_tests_properties(prefill_mmq_tab_test PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
+ endif()
if(STRATA_BUILD_TESTS AND STRATA_MMQ_KQUANTS)
# the UD-Q4_K_XL expert formats through MMQ against ggml's dequantized weights (synthetic, a few seconds)
add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp)
diff --cc include/strata/kernels/qsa_decode_attn.hpp
index d68bab43e,c752b774d..3270d8b6f
--- a/include/strata/kernels/qsa_decode_attn.hpp
+++ b/include/strata/kernels/qsa_decode_attn.hpp
@@@ -43,19 -43,16 +43,30 @@@ void qsa_decode_attn_step(const float*
void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);
+/// Up to this many queries of different slots in one qsa_decode_attn_slots call.
+constexpr int kQsaAttnMaxSlots = 8;
+
+/// One KV pool set per query of a qsa_decode_attn_slots call (passed by value as a kernel argument).
+struct QsaAttnSlotPools {
+ QsaAttnPools p[kQsaAttnMaxSlots];
+};
+
+/// qsa_decode_attn_batch for `n_q` queries that each attend over their OWN pools (`pools.p[i]`, one KV cache per batch
+/// slot): one launch instead of one call per query, every block computing exactly what the per-query call computes.
+/// The pools must all have one KV format. False (nothing launched) where the per-query call would run another kernel
+/// (n_q > kQsaAttnMaxSlots, mixed formats, the pre-sm_75 / gfx103x or gfx906 variants): the caller loops instead.
+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);
+ /// For src/kernels/qsa_merge_daily_parity.cpp only: qsa_decode_attn_batch's two passes apart, whatever the environment
+ /// says. `chunks_only` runs the chunk pass into `scratch` with `daily_chunk` 0: attn_chunk_kernel (upstream's default),
+ /// 1: the Daily's (attn_chunk_kernel_pre75, what STRATA_ATTN_DAILY=1 picks on CUDA). `merge_only` merges those
+ /// partials with `v2` 0: attn_merge_kernel, 1: attn_merge_v2_kernel (STRATA_ATTN_MERGE_V2, on with STRATA_ATTN_DAILY).
+ /// Width must not exceed an unaligned cap (the chunk pass keeps its upstream contract); for aligned caps the grid
+ /// bounds widths above cap. One input can go through every pair and the outputs be compared bit for bit.
+ void qsa_decode_attn_chunks_only(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, int daily_chunk,
+ void* stream);
+ void qsa_decode_attn_merge_only(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch,
+ float* attn, int64_t n_q, int v2, void* stream);
} // namespace strata::kernels
diff --cc include/strata/kernels/qsa_select.hpp
index 3ff769a9a,0df308302..5d1331feb
--- a/include/strata/kernels/qsa_select.hpp
+++ b/include/strata/kernels/qsa_select.hpp
@@@ -29,12 -29,12 +29,18 @@@ void qsa_block_scores(const float* pool
int64_t active_blocks = -1); ///< perf-review C-1: > 0 launches only this many blocks (the
///< batch's largest n_bid + 1; not for a captured graph)
+/// qsa_block_scores for `nq` queries of different slots, query i scoring its own slot's keys (pooled[i], dead[i]; host
+/// arrays of device pointers): one launch whose blocks each compute exactly what the single-query call computes.
+/// False (nothing launched) where that call would run another kernel (nq > 8, STRATA_SCORES_MULTI=0): loop instead.
+bool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,
+ const int32_t* steps, int64_t nq, int64_t max_blocks, const QsaShapes& s, float* scores,
+ void* stream);
+ /// QSAROWS (docs/QSAROWS.md): block_scores_multi_kernel's scores for a call of up to 4 queries, bit for bit, with the
+ /// queries in registers and the four heads reduce-scattered (six shuffles instead of twenty a block and query).
+ /// qsa_block_scores takes it with STRATA_QSA_ROWS=1; false (nothing launched) above 4 queries or on another geometry.
+ /// For qsa_rows_bench's parity and timing it is callable on its own.
+ bool qsa_block_scores_rows(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream);
/// The same scores on tensor cores (3xTF32, FP32-level accuracy but another summation order: not bitwise; the tail
/// block n_bid is the warp kernel's arithmetic). For the prompt path; false (nothing launched) on another geometry.
diff --cc include/strata/prefill/moe_mmq.hpp
index 3d7b4ccdb,7bb46728a..fee662fcc
--- a/include/strata/prefill/moe_mmq.hpp
+++ b/include/strata/prefill/moe_mmq.hpp
@@@ -46,15 -46,13 +46,21 @@@ void scale_gu_rows(float* gu, int64_t l
/// The down product's rows [0, nrows) (`ld` floats each, relative `bounds`) times their expert's s_down - on the FP32
/// output, not folded into up, where the hidden would sit near 1e-5 (FP16-subnormal block scales in q8 formats).
void scale_down_rows(float* d, int64_t ld, const int32_t* bounds, int n, const float* tails, int64_t nrows, void* stream);
+ /// The rows quantize(x, src, ...) writes for tokens x k_used rows in expert order, written token by token: token t's
+ /// row of x is quantized once and stored at rows slot[t * k_used + k] (`slot` the inverse of `src`) - the same bytes,
+ /// a k_used-th of the quantizations (llama.cpp's quantize_scatter_mmq_q8_1_cuda). STRATA_QUANT_GATHER=1: quantize()
+ /// over `src`, as before.
+ void quantize_scatter(const float* x, const int32_t* slot, const int32_t* src, void* xq, int ggml_type, int64_t cols,
+ int64_t ld, int64_t tokens, int k_used, void* stream);
+constexpr int kGatherGroupMax = 16;
+/// STRATA_MMQ_DIRECT: each expert's own weight address (gate|up or down), in place of `w + z * expert_bytes` -
+/// expert z < n <= kGatherGroupMax starts at p[z]. The launch (tiles, stream-k split, fixup) is the same as
+/// with `w`: only the base address of each expert's matrix changes.
+struct WeightTab {
+ const void* p[kGatherGroupMax] = {};
+};
+
/// One launch over n experts whose weights lie `expert_bytes` apart from `w`: for expert e, the activation rows
/// [bounds[e], bounds[e+1]) of `xq` (bounds on the device, n+1 entries) times its [w_rows, w_cols] matrix into
/// dst rows of the same indices (`ld_dst` floats apart, via `ids`: dst row = ids[row], an identity table works).
diff --cc src/core/verify.cpp
index 9a9830fe7,9237b0d07..8c945c4e1
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -1300,9 -1334,126 +1361,126 @@@ bool Verifier::record_window(int T, cud
IQ * ID, N, IQ * ID, n, df_side_[1]);
norm_rope_on(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ, df_side_[1]);
}
- if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs);
- else for (int t = tb; t < te; ++t)
- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs);
+ // STRATA_SIDE_MIXER=1 / =2 (fork Eddoursul 34a35db + b063c13): the indexer's key projection,
+ // the tail's snapshot, its keys appended, its query projection, norm and RoPE, the block scores and the
+ // top-k read only the mixer's input and the indexer's own state - =1 runs them on a branch beside the
+ // attention's projections; =2 also the queries' chain on a second branch. Each step below is
+ // the in-line code with its stream as a parameter; with the switch off they run on cs in the same order
+ // as before (STRATA_DF_BRANCH, the same idea for fewer of these steps, is off whenever this is on).
+ auto norm_rope_s = [&](float* data, const WeightRef* norm, int rows, int cols, const int32_t* pos, cudaStream_t ns) {
+ if (fuse_nr && native_norm_rope_usable(cols, (int) s.n_rot)) {
+ native_qsa_rms_norm_rope(data, cols, (const float*) norm->data, data, rows, cols, (int) s.n_rot, EPS,
+ rope_scaling(), pos, ns);
+ return;
+ }
+ norm_rope_on(data, norm, rows, cols, pos, ns);
+ };
+ auto idx_key = [&](cudaStream_t ns) {
+ if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, ns);
+ else for (int t = tb; t < te; ++t)
+ bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, ns);
+ };
+ auto idx_snap = [&](cudaStream_t ns) {
+ if (batch_rec_) { // snapshot each slot's indexer tail before its first proposed row
- for (int t = tb; t < te; ++t)
++ for (int t = tb; t < te && !keep_rows; ++t)
+ if (t == tb || brow_[t] != brow_[t - 1])
+ copy_from_mapped(tail_snap_b_ + ((size_t) brow_[t] * nQall + qi) * TS,
+ slot_ss(t).qsa_states[qi].idx_tail, TS, ns);
+ } else
+ if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, ns);
+ };
+ auto idx_append = [&](cudaStream_t ns) {
+ if (dec_batch && !g_no_batch_kv && !batch_rec_) {
+ const QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};
+ native_qsa_indexer_append_steps(idx_raw + tb * ID, step_ + tb * kStepCount + kStepPos, kStepCount,
+ n, 0, (const float*) wikn->data, EPS, ib, s, st.max_cells,
+ rope_scaling(), ns);
+ } else {
+ for (int t = tb; t < te; ++t) {
+ const QsaState& sx = slot_ss(t).qsa_states[qi];
+ const QsaIndexerBuffers ib{sx.idx_tail, sx.idx_dead, sx.idx_pooled, sx.idx_block_pos};
+ native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,
+ (const float*) wikn->data, EPS, ib, s, sx.max_cells,
+ rope_scaling(), ns);
+ }
+ }
+ };
+ auto idx_query = [&](cudaStream_t ns) {
+ if (qb) {
+ bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
+ N, IQ * ID, n, ns);
+ norm_rope_s(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ, ns);
+ } else
+ for (int t = tb; t < te; ++t) {
+ float* qx = qidx_ + t * IQ * ID;
+ bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), ns);
+ norm_rope_s(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH, ns);
+ }
+ };
+ auto idx_select = [&](cudaStream_t ns) {
+ qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,
+ s, scores_ + (size_t) tb * max_blocks_, ns);
+ qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,
+ sel_ + (size_t) tb * cap_, ns);
+ };
+ // the attention's queries: the projection (mm on cs, which may read F4's interleaved copy; on a branch
+ // F4's kernel too, its copy written on cs before the fork - bitwise the same), the q/gate
+ // split with norm and RoPE, the rotation
+ const bool q_il = side_proj && g_mmvq_il() &&
+ strata::kernels::native_mmvq_il_supported(wq->native_type, n, (int) (NH * 2 * HD));
+ auto q_chain = [&](cudaStream_t ns) -> bool {
+ if (ns == cs) mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));
+ else if (q_il) strata::kernels::native_mmvq_il(wq->native_type, wq->native_data, xq_, xil_,
+ qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD), n, ns);
+ else native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N,
+ (int) (NH * 2 * HD), n, ns);
+ if (qb) {
+ if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)
+ native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,
+ qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,
+ rope_scaling(), pos_ + tb * NH, ns);
+ } else {
+ copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, ns);
+ norm_rope_s(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH, ns);
+ }
+ if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, ns); // <Hq, Hk> = <q, k>
+ } else {
+ for (int t = tb; t < te; ++t) {
+ float* qc = qcur_ + t * NH * HD;
+ if (fuse_nr) {
+ native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,
+ (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, ns);
+ } else {
+ if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
+ (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, ns) != cudaSuccess) {
+ err = "verify: the q/gate split failed";
+ return false;
+ }
+ norm_rope_s(qc, wqn, (int) NH, (int) HD, pos_ + t * NH, ns);
+ }
+ if (st.kv_rot) fwht256_inplace_cuda(qc, NH, ns); // <Hq, Hk> = <q, k>
+ }
+ }
+ return true;
+ };
+ if (sidem) {
+ if (!branch(mfork_, cs, side_, "fork")) return false;
+ idx_key(side_);
+ idx_snap(side_);
+ idx_append(side_);
+ idx_query(side_);
+ idx_select(side_);
+ if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the indexer's branch could not join"; return false; }
+ }
+ if (side_proj) { // the queries beside the keys and values (fork b063c13)
+ if (q_il && !il_ready) { // the copy before the fork (mm(wk) / mm(wv) on cs reuse it)
+ strata::kernels::native_q8_1_interleave(xq_, xil_, (int) N, n, cs);
+ il_ready = true;
+ }
+ if (!branch(pfork_, cs, side2_, "fork") || !q_chain(side2_)) return false;
+ if (cudaEventRecord(pjoin_, side2_) != cudaSuccess) { err = "verify: the queries' branch could not join"; return false; }
+ }
+ if (!sidem) idx_key(cs);
stamp(l, 7, grp);
mm(wk, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD));
mm(wv, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD));
@@@ -1387,82 -1519,11 +1546,47 @@@
if (qbr) {
join(1); // the indexer query, for the block scores
} else {
- mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));
- if (qb) {
- if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)
- native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,
- qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,
- rope_scaling(), pos_ + tb * NH, cs);
- } else {
- // the q/gate split as a copy kernel: the window's chain stays kernel to kernel (PDL)
- copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, cs);
- norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);
- }
- if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, cs); // <Hq, Hk> = <q, k>
- bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
- N, IQ * ID, n, cs);
- norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);
- } else {
- for (int t = tb; t < te; ++t) {
- float* qc = qcur_ + t * NH * HD;
- if (fuse_nr) {
- native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,
- (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, cs);
- } else {
- if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
- (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
- err = "verify: the q/gate split failed";
- return false;
- }
- norm_rope(qc, wqn, (int) NH, (int) HD, pos_ + t * NH);
- }
- if (st.kv_rot) fwht256_inplace_cuda(qc, NH, cs); // <Hq, Hk> = <q, k>
- }
- for (int t = tb; t < te; ++t) {
- float* qx = qidx_ + t * IQ * ID;
- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);
- norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);
- }
- }
+ if (!side_proj && !q_chain(cs)) return false;
+ if (!sidem) idx_query(cs);
}
stamp(l, 10, grp);
- if (batch_rec_) { // each row selects and attends over its own slot's K/V
+ bool streamed = false; // a streamed KV cache keeps the per-row resolve-then-attend order
+ if (batch_rec_ && g_qsa_slots)
+ for (int t = tb; t < te; ++t) streamed = streamed || slot_ss(t).qsa_states[qi].kv_mode == 1;
+ if (batch_rec_ && g_qsa_slots && !streamed) {
+ // the rows' block scores, top-k and attention, up to 8 rows per launch. The rows only read
+ // their slots' keys and K/V here and write their own scores/sel/scratch/attn rows, so running
+ // each step for all rows before the next is the per-row loop's result
+ for (int t0 = tb; t0 < te; t0 += kQsaAttnMaxSlots) {
+ const int m = std::min(te - t0, kQsaAttnMaxSlots);
+ const float* pooled[kQsaAttnMaxSlots];
+ const float* dead[kQsaAttnMaxSlots];
+ QsaAttnSlotPools px;
+ for (int i = 0; i < m; ++i) {
+ const QsaState& sx = slot_ss(t0 + i).qsa_states[qi];
+ pooled[i] = sx.idx_pooled;
+ dead[i] = sx.idx_dead;
+ px.p[i] = qsa_attn_pools(sx);
+ }
+ // a launcher that would not run the per-row call's kernel launches nothing: per-row calls
+ if (!qsa_block_scores_slots(pooled, dead, qidx_ + t0 * IQ * ID, step_ + t0 * kStepCount, m,
+ max_blocks_, s, scores_ + (size_t) t0 * max_blocks_, cs))
+ for (int t = t0; t < t0 + m; ++t)
+ qsa_block_scores(pooled[t - t0], dead[t - t0], qidx_ + t * IQ * ID, step_ + t * kStepCount,
+ 1, max_blocks_, s, scores_ + (size_t) t * max_blocks_, cs);
+ qsa_block_topk(scores_ + (size_t) t0 * max_blocks_, step_ + t0 * kStepCount, m, max_blocks_,
+ cap_, s, sel_ + (size_t) t0 * cap_, cs);
+ if (!qsa_decode_attn_slots(qcur_ + t0 * NH * HD, px, sel_ + (size_t) t0 * cap_,
+ step_ + t0 * kStepCount, cap_, s,
+ attn_scratch_ + (size_t) t0 * attn_scratch_floats_,
+ attn_ + t0 * NH * HD, m, cs))
+ for (int t = t0; t < t0 + m; ++t)
+ qsa_decode_attn_batch(qcur_ + t * NH * HD, px.p[t - t0], sel_ + (size_t) t * cap_,
+ step_ + t * kStepCount, cap_, s,
+ attn_scratch_ + (size_t) t * attn_scratch_floats_,
+ attn_ + t * NH * HD, 1, cs);
+ }
+ } else if (batch_rec_) { // each row selects and attends over its own slot's K/V
for (int t = tb; t < te; ++t) {
const QsaState& sx = slot_ss(t).qsa_states[qi];
qsa_block_scores(sx.idx_pooled, sx.idx_dead, qidx_ + t * IQ * ID, step_ + t * kStepCount, 1,
diff --cc src/kernels/cuda/qsa_decode_attn.cu
index 85257e6cf,311ad0fa7..fe1ee65e5
--- a/src/kernels/cuda/qsa_decode_attn.cu
+++ b/src/kernels/cuda/qsa_decode_attn.cu
@@@ -643,27 -726,33 +754,44 @@@ bool pre75_attn()
#define STRATA_ATTN_CHUNK(M) attn_chunk_kernel<M>
#endif
+// S25: STRATA_ATTN_LANECELL=1 - the score phase one cell per thread (bit-identical scores, no shuffle reductions)
+bool lane_cell_attn() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();
+ return on;
+}
+
+// 0 fp16, 1 int8, 2 q4_0, 3 int8 K with q4_0 V
+int attn_kv_mode(const QsaAttnPools& p) {
+ return p.k_q4 != nullptr ? 2 : (p.k_q != nullptr && p.v_q4 != nullptr ? 3 : (p.k_q != nullptr ? 1 : 0));
+}
+
- } // namespace
+ // STRATA_ATTN_MERGE_V2: 1 merges with attn_merge_v2_kernel, 0 with attn_merge_kernel. Unset it follows
+ // STRATA_ATTN_DAILY (the Daily had it on by default on NVIDIA), so upstream's merger stays the default.
+ int attn_merge_v2() {
+ static const int on = [] {
+ int r = daily_attn() ? 1 : 0;
+ const char* v = std::getenv("STRATA_ATTN_MERGE_V2");
+ if (v != nullptr && *v != '\0') r = std::atoi(v) != 0 ? 1 : 0;
+ return r;
+ }();
+ return on;
+ }
- void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
- int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
- if (n_q <= 0) return;
- if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps ||
- !pools.page_table || n_q > 65535) {
- std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
- std::exit(1);
- }
+ // the merge of `n_q` queries' partials: v2 = 0 attn_merge_kernel, 1 attn_merge_v2_kernel (bitwise the same output)
+ void launch_merge(int v2, const float* part_acc, const float* part_m, const float* part_l, int n_chunks,
+ const int32_t* steps, float* attn, long long stride, unsigned n_head, unsigned n_q, cudaStream_t st) {
+ if (v2)
+ attn_merge_v2_kernel<kMergeU><<<dim3(n_head, n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks, steps,
+ attn, stride);
+ else
+ attn_merge_kernel<<<dim3(n_head, n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks, attn, stride);
+ }
+
+ // qsa_decode_attn_batch's chunk pass: `n_q` queries' partials into `scratch` (one stride each). `kernel` < 0 is the
+ // environment's choice (lane cell, gfx906 swizzle, STRATA_ATTN_DAILY / PR #540); 0 is attn_chunk_kernel and 1 PR #540's
+ // kernel where it is built (the parity harness: never the environment).
+ void launch_chunks_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, cudaStream_t st, int kernel) {
const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
: (pools.k_q != nullptr ? 1 : 0));
const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
@@@ -674,8 -763,12 +802,11 @@@
float* part_l = part_m + (size_t) n_chunks * s.n_head;
const float scale = 1.0f / sqrtf((float) HD);
const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
- cudaStream_t st = (cudaStream_t) stream;
- const bool lane_cell = lane_cell_attn();
+ #if defined(STRATA_ATTN_PRE75_BUILT)
+ const bool pre = kernel < 0 ? pre75_attn() : kernel == 1;
+ #endif
+ // S25: STRATA_ATTN_LANECELL=1 - the score phase one cell per thread (bit-identical scores, no shuffle reductions)
- static const bool lane_cell_env = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();
- const bool lane_cell = kernel < 0 && lane_cell_env;
++ const bool lane_cell = kernel < 0 && lane_cell_attn();
#if defined(STRATA_HIP_GFX906)
static const bool query_swizzle = [] {
const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';
@@@ -748,64 -870,6 +908,64 @@@ void qsa_decode_attn_batch(const float
}
}
+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
+ if (n_q <= 0) return true;
+ if (n_q > kQsaAttnMaxSlots) return false;
+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: unsupported geometry or missing buffers\n");
+ std::exit(1);
+ }
+ const int kv_mode = attn_kv_mode(pools.p[0]);
+ for (int64_t z = 0; z < n_q; ++z) {
+ if (!pools.p[z].page_table) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: missing page table\n");
+ std::exit(1);
+ }
+ if (attn_kv_mode(pools.p[z]) != kv_mode) return false;
+ }
+ const bool lane_cell = lane_cell_attn();
+ // only where qsa_decode_attn_batch runs attn_chunk_kernel; its other variants keep the per-query calls
+#if defined(STRATA_ATTN_PRE75_BUILT)
+ if (!lane_cell && pre75_attn()) return false;
+#endif
+#if defined(STRATA_HIP_GFX906)
+ static const bool query_swizzle = [] {
+ const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';
+ }();
+ if (query_swizzle && !lane_cell && kv_mode == 1 && query_swizzle_gfx906_device()) return false;
+#endif
+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
+ float* part_acc = scratch;
+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
+ float* part_l = part_m + (size_t) n_chunks * s.n_head;
+ const float scale = 1.0f / sqrtf((float) HD);
+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
+ cudaStream_t st = (cudaStream_t) stream;
+#define STRATA_ATTN_SLOTS(M, LC) attn_chunk_slots_kernel<M, LC><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)
+ if (lane_cell) {
+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, true);
+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, true);
+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, true);
+ else STRATA_ATTN_SLOTS(0, true);
+ } else {
+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, false);
+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, false);
+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, false);
+ else STRATA_ATTN_SLOTS(0, false);
+ }
+#undef STRATA_ATTN_SLOTS
- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,
- attn, stride);
++ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,
++ (unsigned) n_q, st);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: %s\n", cudaGetErrorString(e));
+ std::exit(1);
+ }
+ return true;
+}
+
uint64_t qsa_decode_attn_scratch_floats(int64_t cap, const QsaShapes& s) {
const int64_t chunks = (cap + CHUNK - 1) / CHUNK;
return (uint64_t) chunks * (uint64_t) s.n_head * (HD + 2) + 64;
diff --cc src/kernels/cuda/qsa_select.cu
index d55141722,8995937d9..39db4f65d
--- a/src/kernels/cuda/qsa_select.cu
+++ b/src/kernels/cuda/qsa_select.cu
@@@ -867,37 -883,82 +882,113 @@@ __device__ __forceinline__ void block_s
}
}
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit) {
+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit);
+}
+
+// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the
+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).
+struct ScoreSlotKeys {
+ const float* pooled[MQ];
+ const float* dead[MQ];
+};
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit) {
+ const unsigned z = blockIdx.y;
+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory
+ const float* pooled = keys.pooled[0];
+ const float* dead = keys.dead[0];
+#pragma unroll
+ for (unsigned i = 1; i < MQ; ++i)
+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }
+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,
+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit);
+}
+
+ // QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue
+ // pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a
+ // call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory
+ // (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two
+ // pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices
+ // of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products
+ // are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered
+ // to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the
+ // same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the
+ // heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.
+ template <int NQ>
+ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out) {
+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];
+ if (threadIdx.x < nq) {
+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];
+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];
+ }
+ __syncthreads();
+ int64_t top = 0;
+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)
+ const int lane = threadIdx.x & 31;
+ float4 qr[NQ][IDX_HEADS];
+ #pragma unroll
+ for (int qi = 0; qi < NQ; ++qi)
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)
+ : make_float4(0.f, 0.f, 0.f, 0.f);
+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;
+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+ #pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) {
+ if (qi >= nq) break;
+ const int64_t n_bid = s_nbid[qi];
+ if (b > n_bid) continue;
+ const float4 k4 = (b == n_bid) ? kd : kp;
+ float d[IDX_HEADS];
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float4 q4 = qr[qi][h];
+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ }
+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head
+ float r2[2];
+ #pragma unroll
+ for (int i = 0; i < 2; ++i)
+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);
+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);
+ r += __shfl_xor_sync(0xffffffffu, r, 4);
+ r += __shfl_xor_sync(0xffffffffu, r, 2);
+ r += __shfl_xor_sync(0xffffffffu, r, 1);
+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel
+ const float s1 = __shfl_sync(0xffffffffu, r, 8);
+ const float s2 = __shfl_sync(0xffffffffu, r, 16);
+ const float s3 = __shfl_sync(0xffffffffu, r, 24);
+ if (lane == 0) {
+ float score = 0.0f;
+ score += r > 0.0f ? r : 0.0f;
+ score += s1 > 0.0f ? s1 : 0.0f;
+ score += s2 > 0.0f ? s2 : 0.0f;
+ score += s3 > 0.0f ? s3 : 0.0f;
+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;
+ out[qi * max_blocks + b] = score;
+ }
+ }
+ }
+ }
+
#if !defined(__HIPCC__)
// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or
// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
@@@ -1160,9 -1191,15 +1251,12 @@@ void qsa_block_scores(const float* pool
std::exit(1);
}
// a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)
- static const bool multi = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();
- if (multi && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16
+ if (scores_multi_on() && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16
+ // QSAROWS: STRATA_QSA_ROWS=1 (exactly) takes block_scores_rows_kernel for calls of up to 4 queries - the same bits
+ static const bool rows_env = [] { const char* v = std::getenv("STRATA_QSA_ROWS"); return v != nullptr && v[0] == '1' && v[1] == '\0'; }();
+ if (rows_env && qsa_block_scores_rows(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream)) return;
- // STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits
- static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();
block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, (int) nq,
- max_blocks, scores, early);
+ max_blocks, scores, scores_early_exit());
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores multi: %s\n", cudaGetErrorString(e)); std::exit(1); }
return;
diff --cc src/prefill/prefill.cpp
index 3d6d0ec14,01e870500..49ea499cc
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -336,78 -336,9 +344,80 @@@ inline int bf16x2_mode(bool f16_io)
return f16_io ? 0 : v;
}
inline bool bf16x2(bool f16_io) { return bf16x2_mode(f16_io) != 0; }
+// STRATA_HANDOFF_PIPE=P (opt-in, 2..16; unset/0/1 = the one copy): a layer split hands a chunk's rows on in P row pieces.
+// The stage records an event when its chunk's work is done and one after each piece's copy into the pinned hand-off
+// buffer, waits on the host only for the first (what on_stage_chunk's checkpoint parts and the error check need), and
+// starts the next stage at once: that stage's upload of piece i waits on piece i's event on the GPU, so its uploads
+// overlap the remaining downloads (separate PCIe links; the 3090s here have no P2P) and the thread start is hidden
+// behind them. The same bytes land in the same rows before the next stage's first kernel: exact. No device memory;
+// the events and the existing two pinned buffers per stage only.
+static constexpr int kHandPiecesMax = 16;
+static int handoff_pipe_pieces() {
+ static const int v = [] {
+ const char* e = std::getenv("STRATA_HANDOFF_PIPE");
+ const int p = e ? std::atoi(e) : 0;
+ return p >= 2 ? std::min(p, kHandPiecesMax) : 0;
+ }();
+ return v;
+}
+// rows [r0, r0 + nr) of piece i of a T-row chunk in P pieces (the same split on both sides of a hand-off)
+static void handoff_piece(int64_t T, int P, int i, int64_t& r0, int64_t& nr) {
+ const int64_t per = (T + P - 1) / P;
+ r0 = std::min<int64_t>(T, (int64_t) i * per);
+ nr = std::min<int64_t>(T, r0 + per) - r0;
+}
+// STRATA_HANDOFF_PIPE: the next stage's chunks run on one thread of the stage's own, started once and pinned once
+// (std::async started a thread per chunk: ~1 ms from the hand-off to the next stage's first call, measured with nsys).
+// The futures are the same (run_impl's result, or what it threw), so the chain's waits do not change.
+// (STRATA_HANDOFF_PIPE_WORKER=0: a thread per chunk again, for an A/B of the two halves.)
+static bool handoff_pipe_worker() {
+ static const bool v = [] { const char* e = std::getenv("STRATA_HANDOFF_PIPE_WORKER"); return !e || std::atoi(e) != 0; }();
+ return handoff_pipe_pieces() > 0 && v;
+}
+struct HandWorker {
+ std::mutex mu;
+ std::condition_variable cv;
+ std::deque<std::packaged_task<bool()>> jobs;
+ bool stop = false;
+ std::thread th;
+ HandWorker() {
+ th = std::thread([this] {
+ strata::aux_cpus::pin_current_thread();
+ for (;;) {
+ std::packaged_task<bool()> job;
+ {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [this] { return stop || !jobs.empty(); });
+ if (jobs.empty()) return; // stopped, and every job taken has run (no broken promise)
+ job = std::move(jobs.front());
+ jobs.pop_front();
+ }
+ job();
+ }
+ });
+ }
+ std::future<bool> submit(std::packaged_task<bool()> job) {
+ std::future<bool> f = job.get_future();
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ jobs.push_back(std::move(job));
+ }
+ cv.notify_one();
+ return f;
+ }
+ ~HandWorker() {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ stop = true;
+ }
+ cv.notify_one();
+ if (th.joinable()) th.join();
+ }
+};
+
- // S23 (opt-in STRATA_HC_UPMIX=1): the hyper-connection read's up projection and gr_mix_r as one kernel (gr_upmix, gfx11);
+ // S23 (opt-in STRATA_HC_UPMIX=1): the hyper-connection read's up projection and gr_mix_r as one kernel (gr_upmix: gfx11,
+ // and CUDA sm_80+ on chunks of 33+ tokens - there, with CUDA 13.3 on an RTX 5090, cuBLAS's up GEMM sums K in the fused
+ // kernel's order and the read is the default pair's bits: gr_upmix_parity, STRATA_HC_UPMIX_CHECK);
// STRATA_HC_UPMIX_CHECK=N also runs the default pair on the first N reads and reports the difference of `mixed`
static int64_t pf_switch_min_t() { // S23: STRATA_PF_SWITCH_MIN_T=N - the rounding-level prompt switches only on chunks of
// N or more tokens (shorter prompts keep the default numerics, and their outputs; the switches pay on long ones)
@@@ -4055,38 -3925,7 +4105,38 @@@ bool Prefill::run_impl(const int64_t* t
const size_t q = j % MMQ_GROUP;
// the group's last expert also zeroes the MMQ tail after its slot (see MMQ_TAIL)
const bool last_of_group = q + 1 == MMQ_GROUP || j + 1 == order.size();
+ if (direct) {
+ const auto& f = lay.fmt[(size_t) l];
+ uint8_t* gp = cslot >= 0 && ((uintptr_t) blob_dev | (uintptr_t) f.down_off) % 16 == 0
+ ? m.cache->slot_tail(cslot, f.bytes, mmq_plan().d_over[(size_t) l])
+ : nullptr;
+ if (gp) { // read in its slot: the guard after the blob is zeroed at the group end
+ tgu.p[q] = blob_dev;
+ td.p[q] = blob_dev + f.down_off;
+ gd.p[gd.n++] = gp;
+ ++stats_.experts_direct;
+ } else {
+ const int c = ns++;
+ tgu.p[q] = m.grp_gu + (size_t) c * mmq_gub;
+ td.p[q] = m.grp_d + (size_t) c * mmq_db;
- if (group_gather) {
++ if (gg_now) {
+ gg.blob[c] = blob_dev;
+ gg.n = c + 1;
+ if (slot >= 0) gg_slots[gg_nslots++] = slot;
+ } else {
+ mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off,
+ mmq_db, m.grp_gu + (size_t) c * mmq_gub,
+ m.grp_d + (size_t) c * mmq_db, m.cs);
+ }
+ }
- if (slot >= 0 && !group_gather) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
++ if (slot >= 0 && !gg_now) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
+ if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;
- if (group_gather) { // also when the last entry was direct (a no-op without gathers)
++ if (gg_now) { // also when the last entry was direct (a no-op without gathers)
+ flush(0);
+ gg = mmq::GatherGroup{};
+ }
+ } else {
- if (group_gather) {
+ if (gg_now) {
gg.blob[q] = blob_dev;
gg.n = (int) q + 1;
if (slot >= 0) gg_slots[gg_nslots++] = slot;
@@@ -4107,9 -3946,8 +4157,9 @@@
cudaMemsetAsync(m.grp_d + (size_t) (q + 1) * mmq_db, 0, MMQ_TAIL, m.cs);
}
}
- if (slot >= 0 && !group_gather) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
+ if (slot >= 0 && !gg_now) { cudaEventRecord(m.used[slot], m.cs); m.used_of[slot] = slot; }
if (q + 1 < MMQ_GROUP && j + 1 < order.size()) return true;
+ }
// the group's products: gate/up, swiglu, the group's H to q8_1, down
const size_t j0 = j - q, g = j0 / MMQ_GROUP, n = order.size();
const int ngx = (int) (q + 1);
@@@ -4193,12 -4016,13 +4243,14 @@@
const int32_t e = order[j];
if (stage_of[j] < 0) {
++stats_.experts_resident;
- if (!compute(j, m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]), -1)) return false;
+ if (!compute(j, m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]), -1,
+ m.host_res[(size_t) l * m.g->n_expert + e])) return false;
} else {
- pt.mark(kPfWaitCopy, cs);
- cudaStreamWaitEvent(m.cs, m.copied[stage_of[j]], 0);
+ if (!group_resident) { // grouped: the flush waits once, on the group's last copy
+ pt.mark(kPfWaitCopy, cs);
+ cudaStreamWaitEvent(m.cs, m.copied[stage_of[j]], 0);
+ }
- if (!compute(j, m.stage_dev[stage_of[j]], stage_of[j])) return false;
+ if (!compute(j, m.stage_dev[stage_of[j]], stage_of[j], -1)) return false;
--pending; // compute recorded the slot's release event before any reuse
}
}