Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
qsa_select: multi-CTA decode top-k and a shuffle-free decode scorer, opt-in STRATA_SELECT_LONG=1 (#3820)
ruibeikaab434a8aa6e2f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex ff5494d28..3628d1443 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -470,6 +470,19 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_test(NAME qsa_pattn_parity COMMAND qsa_pattn_parity)set_tests_properties(qsa_pattn_parity PROPERTIES SKIP_RETURN_CODE 77)endif()+ # PR #3820: STRATA_SELECT_LONG's multi-CTA top-k and shuffle-free decode scorer against qsa_block_topk and+ # qsa_block_scores, bitwise, on random scores (GPU, synthetic, no model). The switches force the new kernels on any+ # card; the "n3" case fixes three CTAs a query. Only in the experimental sm_60 build (skips with 77 otherwise).+ if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)+ add_executable(qsa_select_long_parity src/kernels/qsa_select_long_parity.cpp)+ target_include_directories(qsa_select_long_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})+ target_link_libraries(qsa_select_long_parity PRIVATE strata_kernels CUDA::cudart)+ add_test(NAME qsa_select_long_parity COMMAND qsa_select_long_parity)+ add_test(NAME qsa_select_long_parity_n3 COMMAND qsa_select_long_parity)+ set_tests_properties(qsa_select_long_parity qsa_select_long_parity_n3 PROPERTIES SKIP_RETURN_CODE 77)+ set_tests_properties(qsa_select_long_parity PROPERTIES ENVIRONMENT "STRATA_SCORES_DX=1;STRATA_TOPK_MULTI=1")+ set_tests_properties(qsa_select_long_parity_n3 PROPERTIES ENVIRONMENT "STRATA_SCORES_DX=1;STRATA_TOPK_MULTI=1;STRATA_TOPK_MULTI_N=3")+ endif()# PR #600: the Volta (sm_70) prompt attention's latency pipeline against the FP32 kernel it replaces. Only in the# experimental sm_60/sm_70 build, where prompt_attn_v70_kernel is compiled in.if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)diff --git a/docs/OLDER_GPUS.md b/docs/OLDER_GPUS.mdindex 93a0aa7c3..149cb09f1 100644--- a/docs/OLDER_GPUS.md+++ b/docs/OLDER_GPUS.md@@ -129,7 +129,7 @@ in [DETAILS.md](DETAILS.md#rtx-30-series-sm_86-and-tesla-p100-sm_60-bitwise-deco## Pascal GP100: a Q8_0 decode kernel (opt-in)GP100 (P100, PH402) is the one Pascal chip without `__dp4a`. Its dense projections read their weights at 40-80 GB/s of about-730, decoding K- and i-quant blocks one int at a time. Four opt-in changes, all measured on a PH402 only:+730, decoding K- and i-quant blocks one int at a time. Five opt-in changes, all measured on a PH402 only:- **`STRATA_Q8_SM60=1`** packs every Q8_0 dense matrix and the head into the planes of `STRATA_Q8_PACKED` and runs them on`src/kernels/cuda/q8_sm60.cuh`: a warp per row, 16-byte weight loads, activations copied once per workgroup into shared memory,@@ -155,6 +155,14 @@ GP100 (P100, PH402) is the one Pascal chip without `__dp4a`. Its dense projectiochunk's attention at 43.3 -> 30.8 ms per QSA layer at 26K context on one GP100 die, and 2-3% less time on 8K-150K token promptson four dies; we have not measured it here, and nobody has measured compute capability 6.1. `qsa_pattn_parity` (built with thetests in this build) checks the bytes on random data.+- **`STRATA_SELECT_LONG=1`** (decode, CUDA build with `STRATA_EXPERIMENTAL_SM60`, compute capability 6.x by default): past+ `STRATA_SELECT_LONG_AT` cells (default 65,536) a decode window's QSA select runs a top-k over several CTAs a query (six kernels)+ and a decode block scorer without warp shuffles, in place of one CTA a query. They return the same ids and the same score bits+ (`qsa_select_long_parity` checks this on random data). Each costs a few more kernels a call, which only a long context pays+ back, so the verifier keeps a second set of window graphs for it: expect some more VRAM for graphs, and nothing changes below+ the threshold. `STRATA_TOPK_MULTI=0` and `STRATA_SCORES_DX=0` turn one half off. The author measured, on four GP100 dies with+ every expert resident and `--pipeline-windows 3`, 51.0 -> 52.5 tok/s (+2.9%) decoding 400 tokens at 115K context, and no change+ below 64K; we have not measured it. A build without the sm_6x kernels ignores the switch with a warning.None of these changes is measured on compute capability 6.1, which has `__dp4a`. Two practical notes for GP100 cards: the defaultapplication clock is 759 MHz and a layer split leaves each die idle most of a window, so lock the clocksdiff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hppindex dc6613da7..4ed588087 100644--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@ -544,9 +544,12 @@ private:cudaStream_t df_side_[2] = {};cudaEvent_t df_fork_ = nullptr;cudaEvent_t df_join_[2] = {};- cudaGraphExec_t exec_[9] = {};- cudaGraphExec_t exec_nr_[9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise- bool launched_[9] = {}, launched_nr_[9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)+ // [sel_long_][T]: 1 is the long-context QSA select (STRATA_SELECT_LONG=1: multi-CTA top-k, shuffle-free decode+ // scores; see run()); without it only [0] is ever captured+ cudaGraphExec_t exec_[2][9] = {};+ cudaGraphExec_t exec_nr_[2][9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise+ bool launched_[2][9] = {}, launched_nr_[2][9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)+ int sel_long_ = 0;// fork 34a35db (STRATA_SIDE_MIXER=1): the GDN gates and the QSA indexer on a branch of the window's graph beside the// mixer's projections, joined before the recurrence / the attentioncudaStream_t side_ = nullptr;@@ -630,6 +633,7 @@ private:float *qidx_ = nullptr, *scores_ = nullptr, *attn_ = nullptr, *attn32_ = nullptr, *attn_scratch_ = nullptr;float* tail_snap_ = nullptr; // per QSA layerint32_t* sel_ = nullptr;+ int32_t* topk_scratch_ = nullptr; // qsa_block_topk_long's multi-CTA exchange (STRATA_SELECT_LONG=1 only)float *logits_ = nullptr, *w_ = nullptr, *shared_ = nullptr, *parts_ = nullptr, *hit_out_ = nullptr;int32_t *ids_ = nullptr, *hit_slot_ = nullptr, *hit_dst_ = nullptr, *hit_count_ = nullptr;int32_t* plan_ = nullptr; // device copy of the plan blockdiff --git a/include/strata/kernels/qsa_select.hpp b/include/strata/kernels/qsa_select.hppindex e8beacce3..35ee7a586 100644--- a/include/strata/kernels/qsa_select.hpp+++ b/include/strata/kernels/qsa_select.hpp@@ -78,4 +78,20 @@ bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t nvoid qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,const QsaShapes& s, int32_t* ids, void* stream);+/// The long-context decode select (the verifier's STRATA_SELECT_LONG=1 windows; CUDA builds with+/// STRATA_EXPERIMENTAL_SM60, elsewhere the plain calls). qsa_block_topk_long: qsa_block_topk's ids; a call with+/// `multi_scratch` (qsa_topk_multi_scratch_ints(nq) device ints, one call at a time per scratch, no initial value), no+/// active count and <= 16 queries runs several CTAs per query (block_topk_multi_*: STRATA_TOPK_MULTI=0 off, =1 on any+/// card, default compute capability 6.x past the register kernel's reach); otherwise it is qsa_block_topk.+/// qsa_block_scores_long: qsa_block_scores's bits; a call of <= 8 queries with no active count runs+/// block_scores_dx_kernel (STRATA_SCORES_DX=0 off, =1 on any card, default compute capability 6.x); otherwise it is+/// qsa_block_scores.+void qsa_block_topk_long(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,+ const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks, int32_t* multi_scratch);+void qsa_block_scores_long(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks);+/// Device ints a qsa_block_topk_long call of `nq` queries may use as multi_scratch (0 on a build without the multi-CTA+/// path: pass null).+int64_t qsa_topk_multi_scratch_ints(int64_t nq);+} // namespace strata::kernelsdiff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cppindex 351665ecd..8aa89f3d5 100644--- a/metal/src/engine/stubs_0145.cpp+++ b/metal/src/engine/stubs_0145.cpp@@ -150,5 +150,9 @@ namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }// the MTP drafter's STRATA_MTP_Q8_SM60 packing (mtp.cpp): nothing packed on Metalnamespace strata::kernels { bool native_q8_0_sm60_pack_quiet(void const* p0, int p1, int p2, char const* p3) { return false; } }namespace strata::kernels { bool native_q8_0_sm60_pack_in_place(void* p0, int p1, int p2, char const* p3) { return false; } }+// STRATA_SELECT_LONG (verify.cpp's long-context decode select): CUDA only; never reached on Metal (no scratch asked for)+namespace strata::kernels { long long qsa_topk_multi_scratch_ints(long long p0) { return 0; } }+namespace strata::kernels { void qsa_block_topk_long(float const* p0, int const* p1, long long p2, long long p3, long long p4, strata::kernels::QsaShapes const& p5, int* p6, void* p7, long long p8, int* p9) { strata::metal_engine::missing("strata::kernels::qsa_block_topk_long"); } }+namespace strata::kernels { void qsa_block_scores_long(float const* p0, float const* p1, float const* p2, int const* p3, long long p4, long long p5, strata::kernels::QsaShapes const& p6, float* p7, void* p8, long long p9) { strata::metal_engine::missing("strata::kernels::qsa_block_scores_long"); } }// only reached with pf_wmma (off on Metal)namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }diff --git a/src/core/verify.cpp b/src/core/verify.cppindex b82181c25..06306a239 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -166,6 +166,32 @@ inline bool g_qdedup() { static const bool on = [] { const char* v = std::getenv// fork F4 (Eddoursul): 2-4 token dense projections read an interleaved copy of the q8_1 activations (bitwise the same// outputs); STRATA_MMVQ_IL=0 keeps native_mmvq's multi-column kernelsinline bool g_mmvq_il() { static const bool on = [] { const char* v = std::getenv("STRATA_MMVQ_IL"); return v == nullptr || v[0] != '0'; }(); return on; }+// STRATA_SELECT_LONG=1 (opt-in): a window graph past select_long_at() cells takes the long-context QSA select+// (qsa_block_topk_long's multi-CTA top-k through topk_scratch_, qsa_block_scores_long's shuffle-free decode scorer;+// both compute capability 6.x by default, the same ids and score bits). Each costs ~6 more kernels a call, which only+// a longer context pays back (4x GP100: 0.5 ms a window more at 1K, even at 30K, 4.0-4.7 ms less at 300K), so the+// verifier keeps a second set of window graphs for it. Off: no scratch, no second graphs.+inline bool select_long_on() {+ static const bool on = [] {+ const char* v = std::getenv("STRATA_SELECT_LONG");+ if (v == nullptr || v[0] != '1') return false;+ if (strata::kernels::qsa_topk_multi_scratch_ints(1) <= 0) { // a build without the multi-CTA kernels: no second graphs+ std::fprintf(stderr, "strata verify: STRATA_SELECT_LONG=1 needs the CUDA build with STRATA_EXPERIMENTAL_SM60; "+ "ignored\n");+ return false;+ }+ return true;+ }();+ return on;+}+// STRATA_SELECT_LONG_AT: the threshold in cells (default 65536; 0: always)+inline int64_t select_long_at() {+ static const int64_t at = [] {+ const char* v = std::getenv("STRATA_SELECT_LONG_AT");+ return v != nullptr && v[0] != 0 ? (int64_t) std::atoll(v) : (int64_t) 65536;+ }();+ return at;+}// 143-amdk: STRATA_QFUSE_HC - only the hyper-connection read writes its consumer's q8_1 image (not the GDN recurrence): gfx12 default (=0 / 1 forces off / on)inline bool g_qfuse_hc() {#if defined(STRATA_USE_HIP)@@ -617,10 +643,12 @@ Verifier::~Verifier() {}if (cs_) cudaStreamSynchronize(cs_);if (sh_cs_) cudaStreamSynchronize(sh_cs_);- for (auto& e : exec_)- if (e) cudaGraphExecDestroy(e);- for (auto& e : exec_nr_)- if (e) cudaGraphExecDestroy(e);+ for (auto& row : exec_)+ for (auto& e : row)+ if (e) cudaGraphExecDestroy(e);+ for (auto& row : exec_nr_)+ for (auto& e : row)+ if (e) cudaGraphExecDestroy(e);if (commit_exec_) cudaGraphExecDestroy(commit_exec_);for (auto& kv : exec_bm_)if (kv.second) cudaGraphExecDestroy(kv.second);@@ -878,6 +906,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&kcur_ = b.take<float>(T * NKV * HD); vcur_ = b.take<float>(T * NKV * HD);idx_raw_L_ = b.take<float>(nQ * T * ID); qidx_ = b.take<float>(T * IQ * ID);scores_ = b.take<float>(T * (uint64_t) max_blocks_); sel_ = b.take<int32_t>(T * (uint64_t) cap_);+ if (select_long_on() && strata::kernels::qsa_topk_multi_scratch_ints(T) > 0) // STRATA_SELECT_LONG only+ topk_scratch_ = b.take<int32_t>((uint64_t) strata::kernels::qsa_topk_multi_scratch_ints(T));attn_ = b.take<float>(T * NH * HD); attn32_ = b.take<float>(T * NH * HD);attn_scratch_ = b.take<float>(T * (uint64_t) attn_scratch_floats_);tail_snap_ = b.take<float>(nQ * TS);@@ -1662,6 +1692,13 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {}};auto idx_select = [&](cudaStream_t ns) {+ if (sel_long_) { // STRATA_SELECT_LONG: the long-context window graphs (the same ids)+ qsa_block_scores_long(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n,+ max_blocks_, s, scores_ + (size_t) tb * max_blocks_, ns, -1);+ qsa_block_topk_long(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_,+ cap_, s, sel_ + (size_t) tb * cap_, ns, -1, topk_scratch_);+ return;+ }qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,s, scores_ + (size_t) tb * max_blocks_, ns);qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,@@ -2452,7 +2489,7 @@ void Verifier::refresh_ar() {}bool Verifier::capture(int T, std::string& err) {- cudaGraphExec_t& exec_t = ar_off_ ? exec_nr_[T] : exec_[T];+ cudaGraphExec_t& exec_t = ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T];if (exec_t != nullptr) return true;{ // said before the capture: a process that exits inside it (#1275: Windows, 313 MiB free) leaves this line as the tracesize_t free_b = 0, total_b = 0;@@ -2839,7 +2876,8 @@ bool Verifier::chain_overlap() const {bool Verifier::prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err) {const OnDevice on_device(device_);prelaunched_ = 0;- cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[T] : exec_[T]; // #871: the variant the previous stage refreshed+ sel_long_ = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0; // the variant the previous stage captured+ cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T]; // #871: the variant the previous stage refreshedif (T < 1 || T > max_t_ || exec_t == nullptr) { err = "verify: prelaunch before the window was captured"; return false; }if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }if (pos0 + T > ss_->qsa_states[ss_->qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }@@ -2883,11 +2921,14 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,prelaunched_ = 0;if (!pre && pre_launch_ && !pre_launch_(cs_, err)) return false; // (a prelaunched window's was served with the first stage's)if (!pre) refresh_ar();+ // STRATA_SELECT_LONG: the select kernels this window's context pays off with (the same on every stage)+ sel_long_ = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0;if (!capture(T, err) || !capture_commit(err)) return false;const bool chain = chain_overlap();if (chain) { // capture the next stage's window here, on this thread, so the helper only stages and launchesconst OnDevice on_next(next_->device_);next_->refresh_ar(); // #871: the variant its prelaunch and its run() will both use+ next_->sel_long_ = sel_long_;if (!next_->capture(T, err) || !next_->capture_commit(err)) return false;}VDBG("captured; staging\n");@@ -2910,7 +2951,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,#endifconst bool test_stall = g_test_stall > 0 && windows + 1 == g_test_stall; // #267 test hook (off: false)bool flag_raised = false; // the PLE flag went up before the launch (early_ple_flag_mode)- bool& graph_launched = ar_off_ ? launched_nr_[T] : launched_[T];+ bool& graph_launched = ar_off_ ? launched_nr_[sel_long_][T] : launched_[sel_long_][T];if (!pre && ar_on() && !test_stall && (early_ple_flag_mode() == 1 || (early_ple_flag_mode() == 2 && !graph_launched))) {if (do_ple) {const Clock::time_point tp = Clock::now();@@ -2939,11 +2980,11 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");return value != nullptr && value[0] == '1';}();- launch_o.emplace(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,+ launch_o.emplace(ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T], cs_, device_, threaded_launch,[this] { (void) release_gpu_waits(5000); });const cudaError_t le = launch_o->status();#else- const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);+ const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T], cs_);#endiftrace_ev("LAUNCHED", -1, -1, (int64_t) le);graph_launched = true;@@ -3531,6 +3572,7 @@ bool Verifier::capture_batch(const int* rows, int S, int hbase, std::string& errrow_base_ = hbase;for (int t = 0; t < S; ++t) brow_[t] = rows[t];std::string rerr;+ sel_long_ = 0; // batch rows: the one-CTA selectconst bool ok = record_window(S, cs_, rerr);batch_rec_ = false;row_base_ = 0;@@ -4204,8 +4246,14 @@ bool Verifier::capture_all(std::string& err) {if (g_ == nullptr) { err = "verify: capture_all before init"; return false; }if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }- for (int T = 1; T <= max_t_; ++T)- if (!capture(T, err)) return false;+ // STRATA_SELECT_LONG: both select variants (see select_long_at): pl_launch takes the one its window's context+ // needs, and nothing can be captured once pipelined windows are in flight. Without it only [0]+ for (int sl = 0; sl < (select_long_on() ? 2 : 1); ++sl) {+ sel_long_ = sl;+ for (int T = 1; T <= max_t_; ++T)+ if (!capture(T, err)) return false;+ }+ sel_long_ = 0;if (!capture_commit(err)) return false;if ((ev_done_ == nullptr && cudaEventCreateWithFlags(&ev_done_, cudaEventDisableTiming) != cudaSuccess) ||(ev_commit_ == nullptr && cudaEventCreateWithFlags(&ev_commit_, cudaEventDisableTiming) != cudaSuccess)) {@@ -4260,7 +4308,8 @@ bool Verifier::pl_launch(int T, const int32_t* tokens, int64_t pos0, std::stringif (T < 1 || T > max_t_) { err = "verify: window size out of range"; return false; }SessionState& ss = *ss_;if (pos0 + T > ss.qsa_states[ss.qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }- if (exec_[T] == nullptr || commit_exec_ == nullptr || ev_done_ == nullptr || pl_ple_rows_.empty()) {+ const int sl = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0; // the variant this window's context pays off with+ if (exec_[sl][T] == nullptr || commit_exec_ == nullptr || ev_done_ == nullptr || pl_ple_rows_.empty()) {err = "verify: pipelined window not prepared (capture_all)";return false;}@@ -4282,7 +4331,7 @@ bool Verifier::pl_launch(int T, const int32_t* tokens, int64_t pos0, std::stringstd::atomic_thread_fence(std::memory_order_seq_cst);trace_ev("WINDOW (pipelined)", -1, -1, pos0 * 16 + T);ms_host += ms_since(t0);- const cudaError_t le = cudaGraphLaunch(exec_[T], cs_);+ const cudaError_t le = cudaGraphLaunch(exec_[sl][T], cs_);trace_ev("LAUNCHED", -1, -1, (int64_t) le);if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }if (fl_prof_) cudaMemcpyAsync(prof_pin_, prof_, prof_h_.size() * 8, cudaMemcpyDeviceToHost, cs_);diff --git a/src/kernels/cuda/qsa_select.cu b/src/kernels/cuda/qsa_select.cuindex e956dc8b9..283aba9ea 100644--- a/src/kernels/cuda/qsa_select.cu+++ b/src/kernels/cuda/qsa_select.cu@@ -1553,6 +1553,385 @@ bool scores_early_exit() {static const bool on = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();return on;}++#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+// STRATA_SELECT_LONG (verify.cpp): the long-context decode select, qsa_block_scores_long / qsa_block_topk_long.+// ---- decode block scores bit for bit block_scores_multi_kernel's, without its shuffles (STRATA_SCORES_DX, default on+// 6.x, under STRATA_SELECT_LONG). The multi kernel gives a block to a warp and pays 5 shuffles per head per (block, query) for 4 products per+// lane: on GP100 a decode window's call costs ~88 us more per query at 300K cells. Here lane i of warp h builds, for+// block i of a 32-block tile and indexer head h, the same 32-leaf tree as block_scores_multi_kernel's xor butterfly (lanes visited in+// 5-bit-reversed order, depth-first: the xor butterfly's association) for every query of the call at once, the key+// staged once per tile in shared memory and read by all of the queries; the relu sum over the heads then runs in the+// warp kernel's order (0 + r0 + r1 + r2 + r3) from shared memory. Blocks < n_bid; the tail block n_bid is+// block_scores_tail_kernel's (the multi kernel's arithmetic for it). A fixed grid strides over the tiles below the+// call's largest n_bid (capturable).+__device__ __forceinline__ int dx_lane(int j) { return (int) (__brev((unsigned) j) >> 27); } // 5-bit reversal+constexpr int DX_NB = 32; // blocks per tile: one per lane+constexpr int DX_T = 32 * IDX_HEADS; // warp h: indexer head h+constexpr int DX_KS = IDX_DIM + 4; // a key row in shared memory (a quarter warp's 16-byte reads: 32 banks)++template <int NQ>+__global__ void __launch_bounds__(DX_T) block_scores_dx_kernel(const float* __restrict__ pooled,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int64_t max_blocks,+ float* __restrict__ out) {+ extern __shared__ __align__(16) float dx_smem[];+ float* Ks = dx_smem; // [DX_NB][DX_KS]+ float* Qs = Ks + DX_NB * DX_KS; // [NQ][IDX_HEADS][IDX_DIM]+ float* Ds = Qs + NQ * IDX_HEADS * IDX_DIM; // [NQ][IDX_HEADS][DX_NB]: each head's sum, for the relu pass+ __shared__ int64_t s_nbid[NQ];+ const int t = threadIdx.x, lane = t & 31, h = t >> 5;+ if (t < NQ) s_nbid[t] = steps[t * kStepCount + kStepNBid];+ __syncthreads();+ int64_t top = 0;+#pragma unroll+ for (int q = 0; q < NQ; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if (top > max_blocks) top = max_blocks;+ if ((int64_t) blockIdx.x * DX_NB >= top) return; // CTA-uniform+ for (int i = t; i < NQ * IDX_HEADS * IDX_DIM / 4; i += DX_T)+ reinterpret_cast<float4*>(Qs)[i] = reinterpret_cast<const float4*>(q_idx)[i];+ const float* krow = Ks + lane * DX_KS;+ // the next tile's keys in registers while this one is scored (coalesced: thread t row (t + 128 i) / 32, float4 t % 32)+ constexpr int PF = DX_NB * (IDX_DIM / 4) / DX_T;+ float4 pre[PF];+ const int64_t stride = (int64_t) gridDim.x * DX_NB;+ auto fetch = [&](int64_t tb) {+#pragma unroll+ for (int i = 0; i < PF; ++i) {+ const int idx = t + i * DX_T, row = idx / (IDX_DIM / 4), c4 = idx % (IDX_DIM / 4);+ pre[i] = tb + row < top ? __ldg(reinterpret_cast<const float4*>(pooled + (tb + row) * IDX_DIM) + c4)+ : make_float4(0.f, 0.f, 0.f, 0.f);+ }+ };+ fetch((int64_t) blockIdx.x * DX_NB);+ for (int64_t b0 = (int64_t) blockIdx.x * DX_NB; b0 < top; b0 += stride) {+#pragma unroll+ for (int i = 0; i < PF; ++i) {+ const int idx = t + i * DX_T, row = idx / (IDX_DIM / 4), c4 = idx % (IDX_DIM / 4);+ *reinterpret_cast<float4*>(Ks + row * DX_KS + c4 * 4) = pre[i];+ }+ __syncthreads();+ if (b0 + stride < top) fetch(b0 + stride);+ float half0[NQ], half1[NQ];+#pragma unroll 1+ for (int c = 0; c < 4; ++c) {+ float l0[NQ], l1[NQ], l2[NQ];+#pragma unroll+ for (int jj = 0; jj < 8; ++jj) {+ const int L = dx_lane(8 * c + jj);+ const float4 k4 = *reinterpret_cast<const float4*>(krow + L * 4);+#pragma unroll+ for (int q = 0; q < NQ; ++q) {+ const float4 q4 = *reinterpret_cast<const float4*>(Qs + (q * IDX_HEADS + h) * IDX_DIM + L * 4);+ const float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ // the depth-first tree over the slice's 8 leaves (pairs, quads, the octet), then the octets+ if (jj == 0 || jj == 2 || jj == 4 || jj == 6) l0[q] = d;+ else if (jj == 1 || jj == 5) l1[q] = l0[q] + d;+ else if (jj == 3) l2[q] = l1[q] + (l0[q] + d);+ else {+ const float oct = l2[q] + (l1[q] + (l0[q] + d));+ if (c == 0) half0[q] = oct;+ else if (c == 1) half0[q] = half0[q] + oct;+ else if (c == 2) half1[q] = oct;+ else half1[q] = half1[q] + oct;+ }+ }+ }+ }+#pragma unroll+ for (int q = 0; q < NQ; ++q) Ds[(q * IDX_HEADS + h) * DX_NB + lane] = half0[q] + half1[q];+ __syncthreads(); // also: every warp is done reading Ks before the next tile+ for (int q = h; q < NQ; q += IDX_HEADS) {+ const int64_t b = b0 + lane;+ if (b >= s_nbid[q]) continue;+ float score = 0.0f;+#pragma unroll+ for (int hh = 0; hh < IDX_HEADS; ++hh) {+ const float d = Ds[(q * IDX_HEADS + hh) * DX_NB + lane];+ score += d > 0.0f ? d : 0.0f;+ }+ out[q * max_blocks + b] = score;+ }+ }+}++// ---- the decode top-k over several CTAs per query without clusters (any CUDA card; on sm_6x by default under+// STRATA_SELECT_LONG). A decode+// window's 1-5 queries otherwise run one 1,024-thread CTA each (block_topk_wide_kernel past ~135K cells) on one SM+// apiece: on GP100 0.39 ms per call at 300K cells and 1.15 ms at 1M, x12 QSA layers per window. Here each query gets+// n CTAs (a grid of n x nq), CTA r taking blocks [r * per, (r + 1) * per), and the four radix passes, the count and the+// emit are six kernels in a row on the caller's stream - capturable, no co-residency or spin waits: the kernel boundary+// is the barrier. Exchange goes through the caller's scratch (qsa_topk_multi_scratch_ints): pass 0's histograms one+// row per CTA (plain stores), passes 1-3 summed with atomics into rows the pass-0 kernel zeroes, (prefix, above) after+// each digit, and each warp's cells above / at the threshold. Nothing is carried from one call to the next.+// Warp w of a CTA takes the rows [w * rpw, (w + 1) * rpw) of 32 consecutive blocks (lane l block l of a row: coalesced),+// MT_U rows' loads in flight at a time; the loops stay rolled (GP100's instruction cache: a fully unrolled 33-row body+// doubled every kernel's time).+//+// IDENTICAL IDS, as block_topk_cluster_kernel's: thr (the width-th largest key, cells counted with their weights) and+// `above` are pure functions of the integer histograms whatever order they are summed in, and the digit rule is+// block_topk_kernel's (the largest digit d >= 1 whose cells at or above it reach `need`, else 0), here as a 256-thread+// scan. The emit writes every cell of a block above thr and the first eq_budget = width - above cells at thr, ascending+// (block_topk_wide_kernel's row emit, offset by the cells of the warps and CTAs before).+constexpr int MT_T = 256; // threads per CTA+constexpr int MT_W = MT_T / 32; // warps per CTA+constexpr int MT_U = 4; // rows of loads in flight per warp+constexpr int MT_NMAX = 32; // CTAs per query, at most+constexpr int MT_MAXQ = 16; // more queries per call fill the GPU with one CTA each already+// scratch per query (int32): pass 0's rows [MT_NMAX][256], passes 1-3 [3][256], state [5][2], counts [MT_NMAX][MT_W][2]+constexpr int MT_ACC = MT_NMAX * 256, MT_STATE = MT_ACC + 3 * 256, MT_CNT = MT_STATE + 16;+constexpr int MT_Q_INTS = MT_CNT + 2 * MT_NMAX * MT_W + 64;+static_assert(MT_NMAX * MT_W <= MT_T, "the emit sums the earlier CTAs' warp counts one per thread");++struct MtQuery {+ int64_t n_kv, n_bid, width, lo, hi, row0; // row0: the first block of this warp's first row+ int rpw, wtail;+ bool all; // n_kv <= width: every cell, nothing to select+};++__device__ __forceinline__ MtQuery mt_query(const int32_t* __restrict__ steps) {+ const int32_t* st = steps + (int64_t) blockIdx.y * kStepCount;+ MtQuery m;+ m.n_kv = st[kStepNKv];+ m.n_bid = st[kStepNBid];+ m.width = st[kStepWidth];+ m.all = m.n_kv <= m.width;+ // the tail block n_bid weighs its cells (0..3); with none it is not a candidate (the reference skips w == 0)+ m.wtail = (int) (m.n_kv - m.n_bid * R);+ const int64_t nbe = m.n_bid + (m.wtail > 0 ? 1 : 0);+ const int64_t per = (nbe + gridDim.x - 1) / gridDim.x;+ m.lo = (int64_t) blockIdx.x * per < nbe ? (int64_t) blockIdx.x * per : nbe;+ m.hi = m.lo + per < nbe ? m.lo + per : nbe;+ m.rpw = (int) (((m.hi - m.lo + 31) / 32 + MT_W - 1) / MT_W);+ m.row0 = m.lo + (int64_t) (threadIdx.x >> 5) * m.rpw * 32;+ return m;+}++// rows j0 .. j0 + MT_U - 1 of this warp: key[u] for block row0 + 32 (j0 + u) + lane (0 past hi or the warp's rows)+__device__ __forceinline__ void mt_load(const MtQuery& m, const float* __restrict__ sc, int j0, uint32_t (&key)[MT_U]) {+ const int lane = threadIdx.x & 31;+#pragma unroll+ for (int u = 0; u < MT_U; ++u) {+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;+ key[u] = (j0 + u < m.rpw && b < m.hi) ? order_key(__ldg(sc + b)) : 0u;+ }+}++// inclusive scan of one value per thread over the MT_T threads+template <typename V> __device__ __forceinline__ V mt_incl_scan(V v, V* s_w) {+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ V x = v;+#pragma unroll+ for (int o = 1; o < 32; o <<= 1) {+ const V y = __shfl_up_sync(0xffffffffu, x, o);+ if (lane >= o) x += y;+ }+ if (lane == 31) s_w[warp] = x;+ __syncthreads();+ V off = 0;+ for (int w = 0; w < warp; ++w) off += s_w[w];+ __syncthreads();+ return x + off;+}++// The digit at `shift` from the 256 counts (thread t holds digit 255 - t's): block_topk_kernel's rule.+__device__ __forceinline__ void mt_choose(int v, int64_t width, int shift, uint32_t& prefix, int& above, int* s_w,+ int* s_ex, int* s_i) {+ const int t = threadIdx.x;+ if (t == 0) *s_i = 255; // no digit >= 1 reaches it: 0, all of 1..255 above+ const int incl = mt_incl_scan(v, s_w);+ s_ex[t] = incl - v;+ const int64_t need = width - above;+ if (t < 255 && incl >= need && incl - v < need) *s_i = t; // the one t where the running count crosses `need`+ __syncthreads();+ const int i = *s_i;+ prefix |= (uint32_t) (255 - i) << shift;+ above += s_ex[i];+ __syncthreads();+}++// pass PASS's histogram of the CTA's keys under the prefix so far (PASS >= 1 first takes digit PASS - 1)+template <int PASS>+__global__ void __launch_bounds__(MT_T) block_topk_multi_hist_kernel(const float* __restrict__ scores,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, int32_t* __restrict__ scratch) {+ __shared__ int h[MT_W][256];+ __shared__ int s_w[MT_W], s_ex[256], s_i;+ const MtQuery m = mt_query(steps);+ if (m.all) return;+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;+ uint32_t key[MT_U];+ mt_load(m, sc, 0, key); // in flight while the digit is taken+ for (int i = t; i < MT_W * 256; i += MT_T) (&h[0][0])[i] = 0;+ int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;+ uint32_t prefix = 0;+ int above = 0;+ if (PASS == 0) {+ if (blockIdx.x == 0) // passes 1-3 sum into these: zero them for this call+ for (int i = t; i < 3 * 256; i += MT_T) qs[MT_ACC + i] = 0;+ } else {+ int v = 0;+ if (PASS == 1) { // pass 0: one row per CTA+#pragma unroll 8+ for (unsigned r = 0; r < gridDim.x; ++r) v += qs[r * 256 + 255 - t];+ } else {+ prefix = (uint32_t) qs[MT_STATE + 2 * (PASS - 1)];+ above = qs[MT_STATE + 2 * (PASS - 1) + 1];+ v = qs[MT_ACC + (PASS - 2) * 256 + 255 - t];+ }+ mt_choose(v, m.width, 32 - 8 * PASS, prefix, above, s_w, s_ex, &s_i);+ if (blockIdx.x == 0 && t == 0) {+ qs[MT_STATE + 2 * PASS] = (int) prefix;+ qs[MT_STATE + 2 * PASS + 1] = above;+ }+ }+ __syncthreads();+ const int shift = 24 - 8 * PASS;+ const uint32_t hmask = PASS == 0 ? 0u : (0xffffffffu << (shift + 8));+#pragma unroll 1+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) { // uniform over the warp: the ballots see all its lanes+ if (j0 > 0) mt_load(m, sc, j0, key);+#pragma unroll+ for (int u = 0; u < MT_U; ++u) {+ if (j0 + u >= m.rpw) break;+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;+ const int bin = (b < m.hi && (key[u] & hmask) == prefix) ? (int) ((key[u] >> shift) & 255) : -1;+ // the scores share their top bits: the lanes on the first lane's bin in one atomic (no __match_any_sync on 6.x)+ const unsigned pend = __ballot_sync(0xffffffffu, bin >= 0);+ if (pend) {+ const int leader = __ffs(pend) - 1;+ const int lb = __shfl_sync(0xffffffffu, bin, leader);+ const unsigned same = __ballot_sync(0xffffffffu, bin == lb);+ if (lane == leader) atomicAdd(&h[warp][lb], __popc(same) * R);+ else if (bin >= 0 && bin != lb) atomicAdd(&h[warp][bin], R);+ }+ if (bin >= 0 && b == m.n_bid) atomicAdd(&h[warp][bin], m.wtail - R);+ }+ }+ __syncthreads();+ int sum = 0;+#pragma unroll+ for (int w = 0; w < MT_W; ++w) sum += h[w][t];+ if (PASS == 0) qs[blockIdx.x * 256 + t] = sum;+ else if (sum != 0) atomicAdd(&qs[MT_ACC + (PASS - 1) * 256 + t], sum);+}++// digit 3 (the threshold), then each warp's cells above it and at it+__global__ void __launch_bounds__(MT_T) block_topk_multi_count_kernel(const float* __restrict__ scores,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, int32_t* __restrict__ scratch) {+ __shared__ int s_w[MT_W], s_ex[256], s_i;+ const MtQuery m = mt_query(steps);+ if (m.all) return;+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;+ uint32_t key[MT_U];+ mt_load(m, sc, 0, key);+ int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;+ uint32_t prefix = (uint32_t) qs[MT_STATE + 6];+ int above = qs[MT_STATE + 7];+ mt_choose(qs[MT_ACC + 2 * 256 + 255 - t], m.width, 0, prefix, above, s_w, s_ex, &s_i);+ if (blockIdx.x == 0 && t == 0) {+ qs[MT_STATE + 8] = (int) prefix;+ qs[MT_STATE + 9] = above;+ }+ const uint32_t thr = prefix;+ int gt = 0, eq = 0;+#pragma unroll 1+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) {+ if (j0 > 0) mt_load(m, sc, j0, key);+#pragma unroll+ for (int u = 0; u < MT_U; ++u) {+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;+ if (j0 + u >= m.rpw || b >= m.hi) continue;+ const int w = b == m.n_bid ? m.wtail : R;+ if (key[u] > thr) gt += w;+ else if (key[u] == thr) eq += w;+ }+ }+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) {+ gt += __shfl_xor_sync(0xffffffffu, gt, o);+ eq += __shfl_xor_sync(0xffffffffu, eq, o);+ }+ if (lane == 0) {+ qs[MT_CNT + 2 * (blockIdx.x * MT_W + warp)] = gt;+ qs[MT_CNT + 2 * (blockIdx.x * MT_W + warp) + 1] = eq;+ }+}++// the cells, ascending: a warp's first cell goes after the cells the CTAs and warps before it selected+__global__ void __launch_bounds__(MT_T) block_topk_multi_emit_kernel(const float* __restrict__ scores,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, int64_t cap,+ int32_t* __restrict__ ids,+ const int32_t* __restrict__ scratch) {+ __shared__ unsigned long long s_w[MT_W];+ const MtQuery m = mt_query(steps);+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;+ int32_t* out = ids + (int64_t) blockIdx.y * cap;+ if (m.all) { // the identity, split over the CTAs+ for (int64_t j = (int64_t) blockIdx.x * MT_T + t; j < m.n_kv; j += (int64_t) gridDim.x * MT_T) out[j] = (int32_t) j;+ return;+ }+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;+ uint32_t key[MT_U];+ mt_load(m, sc, 0, key);+ const int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;+ const uint32_t thr = (uint32_t) qs[MT_STATE + 8];+ const int eq_budget = (int) (m.width - qs[MT_STATE + 9]); // cells at thr that fit, lowest index first+ // the cells of the warps before this one: the earlier CTAs' (one warp count per thread), then this CTA's+ unsigned long long c = 0;+ if (t < (int) blockIdx.x * MT_W)+ c = (unsigned long long) (unsigned) qs[MT_CNT + 2 * t] | ((unsigned long long) (unsigned) qs[MT_CNT + 2 * t + 1] << 32);+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) c += __shfl_xor_sync(0xffffffffu, c, o);+ if (lane == 0) s_w[warp] = c;+ __syncthreads();+ unsigned long long before = 0;+ for (int w = 0; w < MT_W; ++w) before += s_w[w];+ for (int w = 0; w < warp; ++w) {+ const int i = (int) blockIdx.x * MT_W + w;+ before += (unsigned long long) (unsigned) qs[MT_CNT + 2 * i] |+ ((unsigned long long) (unsigned) qs[MT_CNT + 2 * i + 1] << 32);+ }+ const int gt_before = (int) (before & 0xffffffffu), eq_before = (int) (before >> 32);+ int run_sel = gt_before + (eq_before < eq_budget ? eq_before : eq_budget), run_eq = eq_before;+#pragma unroll 1+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) {+ if (j0 > 0) mt_load(m, sc, j0, key);+#pragma unroll+ for (int u = 0; u < MT_U; ++u) {+ if (j0 + u >= m.rpw) break;+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;+ const int w = b < m.hi ? (b == m.n_bid ? m.wtail : R) : 0;+ const uint32_t k = key[u];+ const int my_gt = (w && k > thr) ? w : 0, my_eq = (w && k == thr) ? w : 0;+ if (__ballot_sync(0xffffffffu, (my_gt | my_eq) != 0) == 0u) continue; // a row without a selected block+ int pe = my_eq; // tied cells up to and with this lane+#pragma unroll+ for (int o = 1; o < 32; o <<= 1) {+ const int y = __shfl_up_sync(0xffffffffu, pe, o);+ if (lane >= o) pe += y;+ }+ const int left = eq_budget - (run_eq + pe - my_eq);+ const int take = left < 0 ? 0 : (left > my_eq ? my_eq : left);+ const int my_sel = my_gt + take;+ int ps = my_sel;+#pragma unroll+ for (int o = 1; o < 32; o <<= 1) {+ const int y = __shfl_up_sync(0xffffffffu, ps, o);+ if (lane >= o) ps += y;+ }+ int32_t* dst = out + run_sel + ps - my_sel;+ for (int cc = 0; cc < my_sel; ++cc) dst[cc] = (int32_t) (b * R + cc);+ run_eq += __shfl_sync(0xffffffffu, pe, 31);+ run_sel += __shfl_sync(0xffffffffu, ps, 31);+ }+ }+}+#endif // !__HIPCC__ && STRATA_EXPERIMENTAL_SM60} // namespacebool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,@@ -2584,4 +2963,153 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }}+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+// STRATA_SCORES_DX: 0 off, 1 on for any CUDA card; unset: on for compute capability 6.x (block_scores_dx_kernel).+// `sms`: the device's multiprocessors (the grid: as many CTAs as fit on each).+static bool scores_dx_device(int& sms) {+ static const int env = [] {+ const char* v = std::getenv("STRATA_SCORES_DX");+ return v == nullptr || *v == '\0' ? -1 : std::atoi(v);+ }();+ if (env == 0) return false;+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess) { cudaGetLastError(); return false; }+ static thread_local int cached_device = -1, cached_sms = 0;+ static thread_local bool pascal = false;+ if (dev != cached_device) {+ int major = 0;+ pascal = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ strata::cc_major_of(major) == 6;+ if (cudaDeviceGetAttribute(&cached_sms, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess) cached_sms = 0;+ cudaGetLastError();+ cached_device = dev;+ }+ sms = cached_sms;+ return sms > 0 && (env > 0 || pascal);+}++template <int NQ>+static void scores_dx_launch(const float* pooled, const float* q_idx, const int32_t* steps, int64_t max_blocks,+ float* scores, int sms, cudaStream_t st) {+ const size_t smem = (size_t) (DX_NB * DX_KS + NQ * IDX_HEADS * IDX_DIM + NQ * IDX_HEADS * DX_NB) * sizeof(float);+ static thread_local int per_sm[64][9] = {}; // CTAs per multiprocessor, by device and NQ+ int dev = 0;+ cudaGetDevice(&dev);+ int& occ = per_sm[dev & 63][NQ];+ if (occ == 0 && (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&occ, block_scores_dx_kernel<NQ>, DX_T, smem) !=+ cudaSuccess || occ < 1)) {+ cudaGetLastError();+ occ = 2;+ }+ const int64_t tiles = (max_blocks + DX_NB - 1) / DX_NB, want = (int64_t) occ * sms;+ const unsigned grid = (unsigned) (want < tiles ? want : tiles);+ block_scores_dx_kernel<NQ><<<grid, DX_T, smem, st>>>(pooled, q_idx, steps, max_blocks, scores);+}++static bool qsa_block_scores_dx(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps,+ int64_t nq, int64_t max_blocks, float* scores, void* stream) {+ int sms = 0;+ if (nq < 1 || nq > MQ || !scores_dx_device(sms)) return false;+ const cudaStream_t st = (cudaStream_t) stream;+ switch (nq) {+ case 1: scores_dx_launch<1>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 2: scores_dx_launch<2>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 3: scores_dx_launch<3>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 4: scores_dx_launch<4>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 5: scores_dx_launch<5>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 6: scores_dx_launch<6>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ case 7: scores_dx_launch<7>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ default: scores_dx_launch<8>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;+ }+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, st>>>(dead, q_idx, steps, max_blocks, scores);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (dx): %s\n", cudaGetErrorString(e)); std::exit(1); }+ return true;+}+#endif++#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+// STRATA_TOPK_MULTI: 0 off, 1 on for any CUDA card and capacity; unset: on for compute capability 6.x past the register+// kernel's reach (GP100: 0.39 -> 0.04 ms per call at 300K cells; see block_topk_multi_hist_kernel)+static int topk_multi_mode() {+ static const int env = [] {+ const char* v = std::getenv("STRATA_TOPK_MULTI");+ return v == nullptr || *v == '\0' ? -1 : std::atoi(v);+ }();+ if (env >= 0) return env != 0 ? 1 : 0;+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess) { cudaGetLastError(); return 0; }+ static thread_local int cached_device = -1;+ static thread_local int mode = 0;+ if (dev != cached_device) {+ int major = 0;+ mode = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ strata::cc_major_of(major) == 6 ? 2 : 0;+ cudaGetLastError();+ cached_device = dev;+ }+ return mode;+}++static bool qsa_block_topk_multi(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,+ int32_t* ids, void* stream, int32_t* scratch) {+ // CTAs per query from the capacity (the grid is fixed in a captured graph; each CTA's share follows n_bid): one per+ // 2,048 blocks, at most MT_NMAX. STRATA_TOPK_MULTI_N: a fixed count (tests)+ static const int n_env = [] { const char* v = std::getenv("STRATA_TOPK_MULTI_N"); return v ? std::atoi(v) : 0; }();+ int64_t n = n_env > 0 ? n_env : (max_blocks + 2047) / 2048;+ n = n < 1 ? 1 : n > MT_NMAX ? MT_NMAX : n;+ const dim3 grid((unsigned) n, (unsigned) nq);+ const cudaStream_t st = (cudaStream_t) stream;+ block_topk_multi_hist_kernel<0><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);+ block_topk_multi_hist_kernel<1><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);+ block_topk_multi_hist_kernel<2><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);+ block_topk_multi_hist_kernel<3><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);+ block_topk_multi_count_kernel<<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);+ block_topk_multi_emit_kernel<<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, cap, ids, scratch);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk multi: %s\n", cudaGetErrorString(e)); std::exit(1); }+ return true;+}+#endif++// ---- STRATA_SELECT_LONG (verify.cpp, opt-in): the decode select past ~64K cells. Outside the SM60 build (and on HIP)+// these are the plain calls; the scratch is never asked for.+int64_t qsa_topk_multi_scratch_ints(int64_t nq) {+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+ return (nq < MT_MAXQ ? nq : MT_MAXQ) * (int64_t) MT_Q_INTS;+#else+ (void) nq;+ return 0;+#endif+}++void qsa_block_topk_long(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,+ const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks, int32_t* multi_scratch) {+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+ // several CTAs per query through the caller's scratch (decode calls: no active count, a few queries)+ static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;+ if (nq > 0 && multi_scratch != nullptr && !old && active_blocks <= 0 && nq <= MT_MAXQ && s.idx_block == R &&+ cap >= qsa_selection_width(kTopkMaxCells, s)) {+ const int mode = topk_multi_mode();+ if ((mode == 1 || (mode == 2 && max_blocks > (int64_t) TK_T * TK_PER)) &&+ qsa_block_topk_multi(scores, steps, nq, max_blocks, cap, ids, stream, multi_scratch))+ return;+ }+#else+ (void) multi_scratch;+#endif+ qsa_block_topk(scores, steps, nq, max_blocks, cap, s, ids, stream, active_blocks);+}++void qsa_block_scores_long(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)+ if (nq > 0 && s.idx_dim == IDX_DIM && s.idx_n_head == IDX_HEADS && s.idx_block == R && scores_multi_on() &&+ nq <= MQ && active_blocks <= 0 &&+ qsa_block_scores_dx(pooled, dead, q_idx, steps, nq, max_blocks, scores, stream))+ return;+#endif+ qsa_block_scores(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream, active_blocks);+}+} // namespace strata::kernelsdiff --git a/src/kernels/qsa_select_long_parity.cpp b/src/kernels/qsa_select_long_parity.cppnew file mode 100644index 000000000..affb41ac0--- /dev/null+++ b/src/kernels/qsa_select_long_parity.cpp@@ -0,0 +1,214 @@+// src/kernels/qsa_select_long_parity.cpp - STRATA_SELECT_LONG's decode select (PR #3820): qsa_block_scores_long+// (block_scores_dx_kernel) and qsa_block_topk_long (the multi-CTA top-k) against qsa_block_scores and qsa_block_topk,+// bit for bit (GPU, synthetic data, no model).+//+// Needs the CUDA build with STRATA_EXPERIMENTAL_SM60 (elsewhere the _long calls forward to the plain ones and there is+// nothing to compare; the test skips with 77) and STRATA_SCORES_DX=1 STRATA_TOPK_MULTI=1 in the environment, which make+// the new kernels run on any card (on compute capability 6.x they are the default). STRATA_TOPK_MULTI_N=<n> fixes the+// CTAs per query (the ctest "n3" case), otherwise one per 2,048 blocks of capacity.+//+// scores: random pooled keys, queries and a dead key; nq 1..8 queries with their own n_bid (a decode window's+// consecutive positions, so tail blocks of 0..3 cells), n_bid 0 / 1 / 31 / 32 / 33 / 1000 / 70001 ...; the+// scores of every block 0..n_bid of every query are compared by memcmp (the tail block n_bid included).+// top-k: scores that are random, all equal, 7 distinct values (many ties across CTAs, so the "first eq_budget cells+// at the threshold" rule is exercised) and random with a boosted tail block; contexts 1,000 (width equals+// n_kv: the identity) up to 524,288 cells; nq 1, 2, 5, 8, 16; capacity max_blocks = the largest context / 4 + 2.+// The selected ids of every query are compared on the first `width` entries. The multi-CTA scratch is filled+// with junk before every call (the contract says no initial value) and must have been written (the new+// kernels ran; a forwarded call would leave it untouched).+//+// qsa_select_long_parity [--device N] exit 0: all cases identical; 77: no GPU or a build without the kernels+#include "strata/kernels/qsa.hpp"+#include "strata/kernels/qsa_select.hpp"++#include <cuda_runtime.h>++#include <algorithm>+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <vector>++namespace k = strata::kernels;++namespace {+void ck(cudaError_t e, const char* what) {+ if (e != cudaSuccess) {+ std::fprintf(stderr, "qsa_select_long_parity: %s: %s\n", what, cudaGetErrorString(e));+ std::exit(2);+ }+}+template <typename T> T* dalloc(size_t n) {+ T* p = nullptr;+ ck(cudaMalloc((void**) &p, n * sizeof(T) + 64), "cudaMalloc");+ ck(cudaMemset(p, 0, n * sizeof(T) + 64), "cudaMemset");+ return p;+}+template <typename T> T* upload(const std::vector<T>& h) {+ T* d = dalloc<T>(h.size());+ ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload");+ return d;+}++// steps of a decode window: query i at position pos0 + i+std::vector<int32_t> make_steps(int nq, int64_t pos0, const k::QsaShapes& s) {+ std::vector<int32_t> st((size_t) nq * k::kStepCount, 0);+ for (int i = 0; i < nq; ++i) {+ int32_t* p = st.data() + (size_t) i * k::kStepCount;+ const int64_t pos = pos0 + i;+ p[k::kStepPos] = (int32_t) pos;+ p[k::kStepNKv] = (int32_t) (pos + 1);+ p[k::kStepNBid] = (int32_t) ((pos + 1) / s.idx_block);+ p[k::kStepWidth] = (int32_t) k::qsa_selection_width(pos + 1, s);+ }+ return st;+}++long long g_cases = 0, g_failed = 0;++void check_scores(const k::QsaShapes& s, int nq, int64_t n_bid0, std::mt19937& rng) {+ const int64_t D = s.idx_dim, H = s.idx_n_head;+ const int64_t max_blocks = n_bid0 + nq + 8;+ std::normal_distribution<float> nd(0.0f, 1.0f);+ std::vector<float> pooled((size_t) (max_blocks * D)), dead((size_t) D), q((size_t) (nq * H * D));+ for (auto& x : pooled) x = nd(rng);+ for (auto& x : dead) x = nd(rng);+ for (auto& x : q) x = nd(rng);+ // the window's last query has n_bid0 blocks; the others one position earlier each (their tails differ)+ const std::vector<int32_t> steps = make_steps(nq, n_bid0 * s.idx_block - 1 + (int64_t) (rng() % 4u) - (nq - 1), s);+ float *dp = upload(pooled), *dd = upload(dead), *dq = upload(q);+ int32_t* dst = upload(steps);+ float* a = dalloc<float>((size_t) (nq * max_blocks));+ float* b = dalloc<float>((size_t) (nq * max_blocks));+ ck(cudaMemset(a, 0xFF, (size_t) (nq * max_blocks) * 4), "fill a");+ ck(cudaMemset(b, 0xFF, (size_t) (nq * max_blocks) * 4), "fill b");+ k::qsa_block_scores(dp, dd, dq, dst, nq, max_blocks, s, a, nullptr);+ k::qsa_block_scores_long(dp, dd, dq, dst, nq, max_blocks, s, b, nullptr, -1);+ ck(cudaDeviceSynchronize(), "scores");+ std::vector<float> ha((size_t) (nq * max_blocks)), hb(ha.size());+ ck(cudaMemcpy(ha.data(), a, ha.size() * 4, cudaMemcpyDeviceToHost), "read a");+ ck(cudaMemcpy(hb.data(), b, hb.size() * 4, cudaMemcpyDeviceToHost), "read b");+ bool ok = true;+ for (int i = 0; i < nq && ok; ++i) {+ const int64_t nb = steps[(size_t) i * k::kStepCount + k::kStepNBid];+ for (int64_t j = 0; j <= nb; ++j) { // block n_bid is the tail+ if (std::memcmp(&ha[(size_t) (i * max_blocks + j)], &hb[(size_t) (i * max_blocks + j)], 4) != 0) {+ std::printf("FAIL scores nq %d n_bid0 %lld: query %d block %lld: %.9g vs %.9g\n", nq, (long long) n_bid0, i,+ (long long) j, ha[(size_t) (i * max_blocks + j)], hb[(size_t) (i * max_blocks + j)]);+ ok = false;+ break;+ }+ }+ }+ ++g_cases;+ if (!ok) ++g_failed;+ for (void* p : {(void*) dp, (void*) dd, (void*) dq, (void*) dst, (void*) a, (void*) b}) cudaFree(p);+}++void check_topk(const k::QsaShapes& s, int nq, int64_t ctx, int mode, int64_t max_blocks, std::mt19937& rng, bool& multi_ran) {+ const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);+ const std::vector<int32_t> steps = make_steps(nq, ctx - nq, s);+ std::vector<float> sc((size_t) (nq * max_blocks), -999.0f);+ std::uniform_real_distribution<float> ur(0.0f, 200.0f);+ for (int i = 0; i < nq; ++i) {+ const int32_t nb = steps[(size_t) i * k::kStepCount + k::kStepNBid];+ const bool tail = steps[(size_t) i * k::kStepCount + k::kStepNKv] % s.idx_block != 0;+ for (int64_t j = 0; j <= nb; ++j) {+ float v = mode == 1 ? 3.0f : mode == 2 ? (float) (rng() % 7u) : ur(rng);+ sc[(size_t) (i * max_blocks + j)] = v;+ }+ if (tail && mode != 1) sc[(size_t) (i * max_blocks + nb)] += 1e9f;+ }+ float* ds = upload(sc);+ int32_t* dst = upload(steps);+ int32_t* ia = dalloc<int32_t>((size_t) (nq * cap));+ int32_t* ib = dalloc<int32_t>((size_t) (nq * cap));+ const int64_t scratch_ints = k::qsa_topk_multi_scratch_ints(nq);+ int32_t* scratch = dalloc<int32_t>((size_t) scratch_ints);+ ck(cudaMemset(scratch, 0xFF, (size_t) scratch_ints * 4), "scratch junk");+ ck(cudaMemset(ia, 0xFF, (size_t) (nq * cap) * 4), "fill ia");+ ck(cudaMemset(ib, 0xFF, (size_t) (nq * cap) * 4), "fill ib");+ k::qsa_block_topk(ds, dst, nq, max_blocks, cap, s, ia, nullptr);+ k::qsa_block_topk_long(ds, dst, nq, max_blocks, cap, s, ib, nullptr, -1, scratch);+ ck(cudaDeviceSynchronize(), "topk");+ std::vector<int32_t> ha((size_t) (nq * cap)), hb(ha.size()), hs((size_t) scratch_ints);+ ck(cudaMemcpy(ha.data(), ia, ha.size() * 4, cudaMemcpyDeviceToHost), "read ia");+ ck(cudaMemcpy(hb.data(), ib, hb.size() * 4, cudaMemcpyDeviceToHost), "read ib");+ ck(cudaMemcpy(hs.data(), scratch, hs.size() * 4, cudaMemcpyDeviceToHost), "read scratch");+ // the multi kernels write the scratch (pass 0's histogram rows) when the contexts are above the identity width+ const bool touched = std::any_of(hs.begin(), hs.end(), [](int32_t v) { return v != -1; });+ if (touched) multi_ran = true;+ bool ok = true;+ for (int i = 0; i < nq && ok; ++i) {+ const int64_t w = steps[(size_t) i * k::kStepCount + k::kStepWidth];+ if (!std::equal(ha.begin() + i * cap, ha.begin() + i * cap + w, hb.begin() + i * cap)) {+ size_t first = 0;+ while (ha[(size_t) (i * cap) + first] == hb[(size_t) (i * cap) + first]) ++first;+ std::printf("FAIL topk nq %d ctx %lld mode %d max_blocks %lld: query %d entry %zu: %d vs %d\n", nq,+ (long long) ctx, mode, (long long) max_blocks, i, first, ha[(size_t) (i * cap) + first],+ hb[(size_t) (i * cap) + first]);+ ok = false;+ }+ }+ ++g_cases;+ if (!ok) ++g_failed;+ for (void* p : {(void*) ds, (void*) dst, (void*) ia, (void*) ib, (void*) scratch}) cudaFree(p);+}+} // namespace++int main(int argc, char** argv) {+ int device = 0;+ for (int i = 1; i < argc; ++i) {+ if (std::strcmp(argv[i], "--device") == 0 && i + 1 < argc) device = std::atoi(argv[++i]);+ else {+ std::fprintf(stderr, "usage: qsa_select_long_parity [--device N]\n");+ return 2;+ }+ }+ int devices = 0;+ if (cudaGetDeviceCount(&devices) != cudaSuccess || devices == 0) {+ std::puts("SKIP: no GPU");+ return 77;+ }+ ck(cudaSetDevice(device), "cudaSetDevice");+ if (k::qsa_topk_multi_scratch_ints(1) <= 0) {+ std::puts("SKIP: this build has no multi-CTA select (CUDA with STRATA_EXPERIMENTAL_SM60)");+ return 77;+ }+ const char* dx = std::getenv("STRATA_SCORES_DX");+ const char* mt = std::getenv("STRATA_TOPK_MULTI");+ if (dx == nullptr || dx[0] != '1' || mt == nullptr || mt[0] != '1') {+ std::fprintf(stderr, "qsa_select_long_parity: set STRATA_SCORES_DX=1 and STRATA_TOPK_MULTI=1 (the kernels are "+ "otherwise default only on compute capability 6.x)\n");+ return 2;+ }+ cudaDeviceProp prop{};+ ck(cudaGetDeviceProperties(&prop, device), "cudaGetDeviceProperties");+ std::printf("qsa_select_long_parity on device %d (%s, %d.%d), STRATA_TOPK_MULTI_N=%s\n", device, prop.name, prop.major,+ prop.minor, std::getenv("STRATA_TOPK_MULTI_N") ? std::getenv("STRATA_TOPK_MULTI_N") : "(per capacity)");++ const k::QsaShapes s = k::qsa_real_shapes();+ std::mt19937 rng(20261026u);+ for (int nq : {1, 2, 3, 5, 8})+ for (int64_t nb : {(int64_t) 8, (int64_t) 31, (int64_t) 32, (int64_t) 33, (int64_t) 1000, (int64_t) 70001, (int64_t) 131072})+ check_scores(s, nq, nb, rng);++ bool multi_ran = false;+ const int64_t max_ctx = 524288, max_blocks = max_ctx / s.idx_block + 2;+ for (int nq : {1, 2, 5, 8, 16})+ for (int64_t ctx : {(int64_t) 1000, (int64_t) 2051, (int64_t) 2055, (int64_t) 8192, (int64_t) 70000, (int64_t) 135164,+ (int64_t) 135168, (int64_t) 300003, max_ctx})+ for (int mode = 0; mode < 3; ++mode)+ check_topk(s, nq, ctx, mode, max_blocks, rng, multi_ran);+ // a smaller capacity than the context asks for (max_blocks only bounds the row stride)+ for (int nq : {1, 4}) check_topk(s, nq, 40000, 0, 40000 / s.idx_block + 2, rng, multi_ran);+ if (!multi_ran) {+ std::printf("FAIL: the multi-CTA top-k never wrote its scratch: the new kernels did not run\n");+ ++g_failed;+ }+ std::printf("qsa_select_long_parity: %lld cases, %lld differ -> %s\n", g_cases, g_failed, g_failed == 0 ? "PASS" : "FAIL");+ return g_failed == 0 ? 0 : 1;+}