Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

qsa_select: multi-CTA decode top-k and a shuffle-free decode scorer, opt-in STRATA_SELECT_LONG=1 (#3820)

ruibeikaab434a8aa6e2f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ff5494d28..3628d1443 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -470,6 +470,19 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_test(NAME qsa_pattn_parity COMMAND qsa_pattn_parity)
set_tests_properties(qsa_pattn_parity PROPERTIES SKIP_RETURN_CODE 77)
endif()
+ # PR #3820: STRATA_SELECT_LONG's multi-CTA top-k and shuffle-free decode scorer against qsa_block_topk and
+ # qsa_block_scores, bitwise, on random scores (GPU, synthetic, no model). The switches force the new kernels on any
+ # card; the "n3" case fixes three CTAs a query. Only in the experimental sm_60 build (skips with 77 otherwise).
+ if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)
+ add_executable(qsa_select_long_parity src/kernels/qsa_select_long_parity.cpp)
+ target_include_directories(qsa_select_long_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_select_long_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_select_long_parity COMMAND qsa_select_long_parity)
+ add_test(NAME qsa_select_long_parity_n3 COMMAND qsa_select_long_parity)
+ set_tests_properties(qsa_select_long_parity qsa_select_long_parity_n3 PROPERTIES SKIP_RETURN_CODE 77)
+ set_tests_properties(qsa_select_long_parity PROPERTIES ENVIRONMENT "STRATA_SCORES_DX=1;STRATA_TOPK_MULTI=1")
+ set_tests_properties(qsa_select_long_parity_n3 PROPERTIES ENVIRONMENT "STRATA_SCORES_DX=1;STRATA_TOPK_MULTI=1;STRATA_TOPK_MULTI_N=3")
+ endif()
# PR #600: the Volta (sm_70) prompt attention's latency pipeline against the FP32 kernel it replaces. Only in the
# experimental sm_60/sm_70 build, where prompt_attn_v70_kernel is compiled in.
if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND STRATA_EXPERIMENTAL_SM60)
diff --git a/docs/OLDER_GPUS.md b/docs/OLDER_GPUS.md
index 93a0aa7c3..149cb09f1 100644
--- a/docs/OLDER_GPUS.md
+++ b/docs/OLDER_GPUS.md
@@ -129,7 +129,7 @@ in [DETAILS.md](DETAILS.md#rtx-30-series-sm_86-and-tesla-p100-sm_60-bitwise-deco
## Pascal GP100: a Q8_0 decode kernel (opt-in)
GP100 (P100, PH402) is the one Pascal chip without `__dp4a`. Its dense projections read their weights at 40-80 GB/s of about
-730, decoding K- and i-quant blocks one int at a time. Four opt-in changes, all measured on a PH402 only:
+730, decoding K- and i-quant blocks one int at a time. Five opt-in changes, all measured on a PH402 only:
- **`STRATA_Q8_SM60=1`** packs every Q8_0 dense matrix and the head into the planes of `STRATA_Q8_PACKED` and runs them on
`src/kernels/cuda/q8_sm60.cuh`: a warp per row, 16-byte weight loads, activations copied once per workgroup into shared memory,
@@ -155,6 +155,14 @@ GP100 (P100, PH402) is the one Pascal chip without `__dp4a`. Its dense projectio
chunk's attention at 43.3 -> 30.8 ms per QSA layer at 26K context on one GP100 die, and 2-3% less time on 8K-150K token prompts
on four dies; we have not measured it here, and nobody has measured compute capability 6.1. `qsa_pattn_parity` (built with the
tests in this build) checks the bytes on random data.
+- **`STRATA_SELECT_LONG=1`** (decode, CUDA build with `STRATA_EXPERIMENTAL_SM60`, compute capability 6.x by default): past
+ `STRATA_SELECT_LONG_AT` cells (default 65,536) a decode window's QSA select runs a top-k over several CTAs a query (six kernels)
+ and a decode block scorer without warp shuffles, in place of one CTA a query. They return the same ids and the same score bits
+ (`qsa_select_long_parity` checks this on random data). Each costs a few more kernels a call, which only a long context pays
+ back, so the verifier keeps a second set of window graphs for it: expect some more VRAM for graphs, and nothing changes below
+ the threshold. `STRATA_TOPK_MULTI=0` and `STRATA_SCORES_DX=0` turn one half off. The author measured, on four GP100 dies with
+ every expert resident and `--pipeline-windows 3`, 51.0 -> 52.5 tok/s (+2.9%) decoding 400 tokens at 115K context, and no change
+ below 64K; we have not measured it. A build without the sm_6x kernels ignores the switch with a warning.
None of these changes is measured on compute capability 6.1, which has `__dp4a`. Two practical notes for GP100 cards: the default
application clock is 759 MHz and a layer split leaves each die idle most of a window, so lock the clocks
diff --git a/include/strata/core/verify.hpp b/include/strata/core/verify.hpp
index dc6613da7..4ed588087 100644
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@ -544,9 +544,12 @@ private:
cudaStream_t df_side_[2] = {};
cudaEvent_t df_fork_ = nullptr;
cudaEvent_t df_join_[2] = {};
- cudaGraphExec_t exec_[9] = {};
- cudaGraphExec_t exec_nr_[9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise
- bool launched_[9] = {}, launched_nr_[9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)
+ // [sel_long_][T]: 1 is the long-context QSA select (STRATA_SELECT_LONG=1: multi-CTA top-k, shuffle-free decode
+ // scores; see run()); without it only [0] is ever captured
+ cudaGraphExec_t exec_[2][9] = {};
+ cudaGraphExec_t exec_nr_[2][9] = {}; // #871: the doorbell variant of a stage that is all-resident otherwise
+ bool launched_[2][9] = {}, launched_nr_[2][9] = {}; ///< this graph has been launched once (early_ple_flag: a cold first launch raises the PLE flag first)
+ int sel_long_ = 0;
// fork 34a35db (STRATA_SIDE_MIXER=1): the GDN gates and the QSA indexer on a branch of the window's graph beside the
// mixer's projections, joined before the recurrence / the attention
cudaStream_t side_ = nullptr;
@@ -630,6 +633,7 @@ private:
float *qidx_ = nullptr, *scores_ = nullptr, *attn_ = nullptr, *attn32_ = nullptr, *attn_scratch_ = nullptr;
float* tail_snap_ = nullptr; // per QSA layer
int32_t* sel_ = nullptr;
+ int32_t* topk_scratch_ = nullptr; // qsa_block_topk_long's multi-CTA exchange (STRATA_SELECT_LONG=1 only)
float *logits_ = nullptr, *w_ = nullptr, *shared_ = nullptr, *parts_ = nullptr, *hit_out_ = nullptr;
int32_t *ids_ = nullptr, *hit_slot_ = nullptr, *hit_dst_ = nullptr, *hit_count_ = nullptr;
int32_t* plan_ = nullptr; // device copy of the plan block
diff --git a/include/strata/kernels/qsa_select.hpp b/include/strata/kernels/qsa_select.hpp
index e8beacce3..35ee7a586 100644
--- a/include/strata/kernels/qsa_select.hpp
+++ b/include/strata/kernels/qsa_select.hpp
@@ -78,4 +78,20 @@ bool qsa_block_topk_cluster(const float* scores, const int32_t* steps, int64_t n
void qsa_block_topk_ref(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
const QsaShapes& s, int32_t* ids, void* stream);
+/// The long-context decode select (the verifier's STRATA_SELECT_LONG=1 windows; CUDA builds with
+/// STRATA_EXPERIMENTAL_SM60, elsewhere the plain calls). qsa_block_topk_long: qsa_block_topk's ids; a call with
+/// `multi_scratch` (qsa_topk_multi_scratch_ints(nq) device ints, one call at a time per scratch, no initial value), no
+/// active count and <= 16 queries runs several CTAs per query (block_topk_multi_*: STRATA_TOPK_MULTI=0 off, =1 on any
+/// card, default compute capability 6.x past the register kernel's reach); otherwise it is qsa_block_topk.
+/// qsa_block_scores_long: qsa_block_scores's bits; a call of <= 8 queries with no active count runs
+/// block_scores_dx_kernel (STRATA_SCORES_DX=0 off, =1 on any card, default compute capability 6.x); otherwise it is
+/// qsa_block_scores.
+void qsa_block_topk_long(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+ const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks, int32_t* multi_scratch);
+void qsa_block_scores_long(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks);
+/// Device ints a qsa_block_topk_long call of `nq` queries may use as multi_scratch (0 on a build without the multi-CTA
+/// path: pass null).
+int64_t qsa_topk_multi_scratch_ints(int64_t nq);
+
} // namespace strata::kernels
diff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cpp
index 351665ecd..8aa89f3d5 100644
--- a/metal/src/engine/stubs_0145.cpp
+++ b/metal/src/engine/stubs_0145.cpp
@@ -150,5 +150,9 @@ namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }
// the MTP drafter's STRATA_MTP_Q8_SM60 packing (mtp.cpp): nothing packed on Metal
namespace strata::kernels { bool native_q8_0_sm60_pack_quiet(void const* p0, int p1, int p2, char const* p3) { return false; } }
namespace strata::kernels { bool native_q8_0_sm60_pack_in_place(void* p0, int p1, int p2, char const* p3) { return false; } }
+// STRATA_SELECT_LONG (verify.cpp's long-context decode select): CUDA only; never reached on Metal (no scratch asked for)
+namespace strata::kernels { long long qsa_topk_multi_scratch_ints(long long p0) { return 0; } }
+namespace strata::kernels { void qsa_block_topk_long(float const* p0, int const* p1, long long p2, long long p3, long long p4, strata::kernels::QsaShapes const& p5, int* p6, void* p7, long long p8, int* p9) { strata::metal_engine::missing("strata::kernels::qsa_block_topk_long"); } }
+namespace strata::kernels { void qsa_block_scores_long(float const* p0, float const* p1, float const* p2, int const* p3, long long p4, long long p5, strata::kernels::QsaShapes const& p6, float* p7, void* p8, long long p9) { strata::metal_engine::missing("strata::kernels::qsa_block_scores_long"); } }
// only reached with pf_wmma (off on Metal)
namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index b82181c25..06306a239 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -166,6 +166,32 @@ inline bool g_qdedup() { static const bool on = [] { const char* v = std::getenv
// fork F4 (Eddoursul): 2-4 token dense projections read an interleaved copy of the q8_1 activations (bitwise the same
// outputs); STRATA_MMVQ_IL=0 keeps native_mmvq's multi-column kernels
inline bool g_mmvq_il() { static const bool on = [] { const char* v = std::getenv("STRATA_MMVQ_IL"); return v == nullptr || v[0] != '0'; }(); return on; }
+// STRATA_SELECT_LONG=1 (opt-in): a window graph past select_long_at() cells takes the long-context QSA select
+// (qsa_block_topk_long's multi-CTA top-k through topk_scratch_, qsa_block_scores_long's shuffle-free decode scorer;
+// both compute capability 6.x by default, the same ids and score bits). Each costs ~6 more kernels a call, which only
+// a longer context pays back (4x GP100: 0.5 ms a window more at 1K, even at 30K, 4.0-4.7 ms less at 300K), so the
+// verifier keeps a second set of window graphs for it. Off: no scratch, no second graphs.
+inline bool select_long_on() {
+ static const bool on = [] {
+ const char* v = std::getenv("STRATA_SELECT_LONG");
+ if (v == nullptr || v[0] != '1') return false;
+ if (strata::kernels::qsa_topk_multi_scratch_ints(1) <= 0) { // a build without the multi-CTA kernels: no second graphs
+ std::fprintf(stderr, "strata verify: STRATA_SELECT_LONG=1 needs the CUDA build with STRATA_EXPERIMENTAL_SM60; "
+ "ignored\n");
+ return false;
+ }
+ return true;
+ }();
+ return on;
+}
+// STRATA_SELECT_LONG_AT: the threshold in cells (default 65536; 0: always)
+inline int64_t select_long_at() {
+ static const int64_t at = [] {
+ const char* v = std::getenv("STRATA_SELECT_LONG_AT");
+ return v != nullptr && v[0] != 0 ? (int64_t) std::atoll(v) : (int64_t) 65536;
+ }();
+ return at;
+}
// 143-amdk: STRATA_QFUSE_HC - only the hyper-connection read writes its consumer's q8_1 image (not the GDN recurrence): gfx12 default (=0 / 1 forces off / on)
inline bool g_qfuse_hc() {
#if defined(STRATA_USE_HIP)
@@ -617,10 +643,12 @@ Verifier::~Verifier() {
}
if (cs_) cudaStreamSynchronize(cs_);
if (sh_cs_) cudaStreamSynchronize(sh_cs_);
- for (auto& e : exec_)
- if (e) cudaGraphExecDestroy(e);
- for (auto& e : exec_nr_)
- if (e) cudaGraphExecDestroy(e);
+ for (auto& row : exec_)
+ for (auto& e : row)
+ if (e) cudaGraphExecDestroy(e);
+ for (auto& row : exec_nr_)
+ for (auto& e : row)
+ if (e) cudaGraphExecDestroy(e);
if (commit_exec_) cudaGraphExecDestroy(commit_exec_);
for (auto& kv : exec_bm_)
if (kv.second) cudaGraphExecDestroy(kv.second);
@@ -878,6 +906,8 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
kcur_ = b.take<float>(T * NKV * HD); vcur_ = b.take<float>(T * NKV * HD);
idx_raw_L_ = b.take<float>(nQ * T * ID); qidx_ = b.take<float>(T * IQ * ID);
scores_ = b.take<float>(T * (uint64_t) max_blocks_); sel_ = b.take<int32_t>(T * (uint64_t) cap_);
+ if (select_long_on() && strata::kernels::qsa_topk_multi_scratch_ints(T) > 0) // STRATA_SELECT_LONG only
+ topk_scratch_ = b.take<int32_t>((uint64_t) strata::kernels::qsa_topk_multi_scratch_ints(T));
attn_ = b.take<float>(T * NH * HD); attn32_ = b.take<float>(T * NH * HD);
attn_scratch_ = b.take<float>(T * (uint64_t) attn_scratch_floats_);
tail_snap_ = b.take<float>(nQ * TS);
@@ -1662,6 +1692,13 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
}
};
auto idx_select = [&](cudaStream_t ns) {
+ if (sel_long_) { // STRATA_SELECT_LONG: the long-context window graphs (the same ids)
+ qsa_block_scores_long(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n,
+ max_blocks_, s, scores_ + (size_t) tb * max_blocks_, ns, -1);
+ qsa_block_topk_long(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_,
+ cap_, s, sel_ + (size_t) tb * cap_, ns, -1, topk_scratch_);
+ return;
+ }
qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,
s, scores_ + (size_t) tb * max_blocks_, ns);
qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,
@@ -2452,7 +2489,7 @@ void Verifier::refresh_ar() {
}
bool Verifier::capture(int T, std::string& err) {
- cudaGraphExec_t& exec_t = ar_off_ ? exec_nr_[T] : exec_[T];
+ cudaGraphExec_t& exec_t = ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T];
if (exec_t != nullptr) return true;
{ // said before the capture: a process that exits inside it (#1275: Windows, 313 MiB free) leaves this line as the trace
size_t free_b = 0, total_b = 0;
@@ -2839,7 +2876,8 @@ bool Verifier::chain_overlap() const {
bool Verifier::prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err) {
const OnDevice on_device(device_);
prelaunched_ = 0;
- cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[T] : exec_[T]; // #871: the variant the previous stage refreshed
+ sel_long_ = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0; // the variant the previous stage captured
+ cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T]; // #871: the variant the previous stage refreshed
if (T < 1 || T > max_t_ || exec_t == nullptr) { err = "verify: prelaunch before the window was captured"; return false; }
if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }
if (pos0 + T > ss_->qsa_states[ss_->qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }
@@ -2883,11 +2921,14 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
prelaunched_ = 0;
if (!pre && pre_launch_ && !pre_launch_(cs_, err)) return false; // (a prelaunched window's was served with the first stage's)
if (!pre) refresh_ar();
+ // STRATA_SELECT_LONG: the select kernels this window's context pays off with (the same on every stage)
+ sel_long_ = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0;
if (!capture(T, err) || !capture_commit(err)) return false;
const bool chain = chain_overlap();
if (chain) { // capture the next stage's window here, on this thread, so the helper only stages and launches
const OnDevice on_next(next_->device_);
next_->refresh_ar(); // #871: the variant its prelaunch and its run() will both use
+ next_->sel_long_ = sel_long_;
if (!next_->capture(T, err) || !next_->capture_commit(err)) return false;
}
VDBG("captured; staging\n");
@@ -2910,7 +2951,7 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
#endif
const bool test_stall = g_test_stall > 0 && windows + 1 == g_test_stall; // #267 test hook (off: false)
bool flag_raised = false; // the PLE flag went up before the launch (early_ple_flag_mode)
- bool& graph_launched = ar_off_ ? launched_nr_[T] : launched_[T];
+ bool& graph_launched = ar_off_ ? launched_nr_[sel_long_][T] : launched_[sel_long_][T];
if (!pre && ar_on() && !test_stall && (early_ple_flag_mode() == 1 || (early_ple_flag_mode() == 2 && !graph_launched))) {
if (do_ple) {
const Clock::time_point tp = Clock::now();
@@ -2939,11 +2980,11 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");
return value != nullptr && value[0] == '1';
}();
- launch_o.emplace(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,
+ launch_o.emplace(ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T], cs_, device_, threaded_launch,
[this] { (void) release_gpu_waits(5000); });
const cudaError_t le = launch_o->status();
#else
- const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);
+ const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[sel_long_][T] : exec_[sel_long_][T], cs_);
#endif
trace_ev("LAUNCHED", -1, -1, (int64_t) le);
graph_launched = true;
@@ -3531,6 +3572,7 @@ bool Verifier::capture_batch(const int* rows, int S, int hbase, std::string& err
row_base_ = hbase;
for (int t = 0; t < S; ++t) brow_[t] = rows[t];
std::string rerr;
+ sel_long_ = 0; // batch rows: the one-CTA select
const bool ok = record_window(S, cs_, rerr);
batch_rec_ = false;
row_base_ = 0;
@@ -4204,8 +4246,14 @@ bool Verifier::capture_all(std::string& err) {
if (g_ == nullptr) { err = "verify: capture_all before init"; return false; }
if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }
if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }
- for (int T = 1; T <= max_t_; ++T)
- if (!capture(T, err)) return false;
+ // STRATA_SELECT_LONG: both select variants (see select_long_at): pl_launch takes the one its window's context
+ // needs, and nothing can be captured once pipelined windows are in flight. Without it only [0]
+ for (int sl = 0; sl < (select_long_on() ? 2 : 1); ++sl) {
+ sel_long_ = sl;
+ for (int T = 1; T <= max_t_; ++T)
+ if (!capture(T, err)) return false;
+ }
+ sel_long_ = 0;
if (!capture_commit(err)) return false;
if ((ev_done_ == nullptr && cudaEventCreateWithFlags(&ev_done_, cudaEventDisableTiming) != cudaSuccess) ||
(ev_commit_ == nullptr && cudaEventCreateWithFlags(&ev_commit_, cudaEventDisableTiming) != cudaSuccess)) {
@@ -4260,7 +4308,8 @@ bool Verifier::pl_launch(int T, const int32_t* tokens, int64_t pos0, std::string
if (T < 1 || T > max_t_) { err = "verify: window size out of range"; return false; }
SessionState& ss = *ss_;
if (pos0 + T > ss.qsa_states[ss.qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }
- if (exec_[T] == nullptr || commit_exec_ == nullptr || ev_done_ == nullptr || pl_ple_rows_.empty()) {
+ const int sl = select_long_on() && pos0 + T >= select_long_at() ? 1 : 0; // the variant this window's context pays off with
+ if (exec_[sl][T] == nullptr || commit_exec_ == nullptr || ev_done_ == nullptr || pl_ple_rows_.empty()) {
err = "verify: pipelined window not prepared (capture_all)";
return false;
}
@@ -4282,7 +4331,7 @@ bool Verifier::pl_launch(int T, const int32_t* tokens, int64_t pos0, std::string
std::atomic_thread_fence(std::memory_order_seq_cst);
trace_ev("WINDOW (pipelined)", -1, -1, pos0 * 16 + T);
ms_host += ms_since(t0);
- const cudaError_t le = cudaGraphLaunch(exec_[T], cs_);
+ const cudaError_t le = cudaGraphLaunch(exec_[sl][T], cs_);
trace_ev("LAUNCHED", -1, -1, (int64_t) le);
if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }
if (fl_prof_) cudaMemcpyAsync(prof_pin_, prof_, prof_h_.size() * 8, cudaMemcpyDeviceToHost, cs_);
diff --git a/src/kernels/cuda/qsa_select.cu b/src/kernels/cuda/qsa_select.cu
index e956dc8b9..283aba9ea 100644
--- a/src/kernels/cuda/qsa_select.cu
+++ b/src/kernels/cuda/qsa_select.cu
@@ -1553,6 +1553,385 @@ bool scores_early_exit() {
static const bool on = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();
return on;
}
+
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+// STRATA_SELECT_LONG (verify.cpp): the long-context decode select, qsa_block_scores_long / qsa_block_topk_long.
+// ---- decode block scores bit for bit block_scores_multi_kernel's, without its shuffles (STRATA_SCORES_DX, default on
+// 6.x, under STRATA_SELECT_LONG). The multi kernel gives a block to a warp and pays 5 shuffles per head per (block, query) for 4 products per
+// lane: on GP100 a decode window's call costs ~88 us more per query at 300K cells. Here lane i of warp h builds, for
+// block i of a 32-block tile and indexer head h, the same 32-leaf tree as block_scores_multi_kernel's xor butterfly (lanes visited in
+// 5-bit-reversed order, depth-first: the xor butterfly's association) for every query of the call at once, the key
+// staged once per tile in shared memory and read by all of the queries; the relu sum over the heads then runs in the
+// warp kernel's order (0 + r0 + r1 + r2 + r3) from shared memory. Blocks < n_bid; the tail block n_bid is
+// block_scores_tail_kernel's (the multi kernel's arithmetic for it). A fixed grid strides over the tiles below the
+// call's largest n_bid (capturable).
+__device__ __forceinline__ int dx_lane(int j) { return (int) (__brev((unsigned) j) >> 27); } // 5-bit reversal
+constexpr int DX_NB = 32; // blocks per tile: one per lane
+constexpr int DX_T = 32 * IDX_HEADS; // warp h: indexer head h
+constexpr int DX_KS = IDX_DIM + 4; // a key row in shared memory (a quarter warp's 16-byte reads: 32 banks)
+
+template <int NQ>
+__global__ void __launch_bounds__(DX_T) block_scores_dx_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int64_t max_blocks,
+ float* __restrict__ out) {
+ extern __shared__ __align__(16) float dx_smem[];
+ float* Ks = dx_smem; // [DX_NB][DX_KS]
+ float* Qs = Ks + DX_NB * DX_KS; // [NQ][IDX_HEADS][IDX_DIM]
+ float* Ds = Qs + NQ * IDX_HEADS * IDX_DIM; // [NQ][IDX_HEADS][DX_NB]: each head's sum, for the relu pass
+ __shared__ int64_t s_nbid[NQ];
+ const int t = threadIdx.x, lane = t & 31, h = t >> 5;
+ if (t < NQ) s_nbid[t] = steps[t * kStepCount + kStepNBid];
+ __syncthreads();
+ int64_t top = 0;
+#pragma unroll
+ for (int q = 0; q < NQ; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if (top > max_blocks) top = max_blocks;
+ if ((int64_t) blockIdx.x * DX_NB >= top) return; // CTA-uniform
+ for (int i = t; i < NQ * IDX_HEADS * IDX_DIM / 4; i += DX_T)
+ reinterpret_cast<float4*>(Qs)[i] = reinterpret_cast<const float4*>(q_idx)[i];
+ const float* krow = Ks + lane * DX_KS;
+ // the next tile's keys in registers while this one is scored (coalesced: thread t row (t + 128 i) / 32, float4 t % 32)
+ constexpr int PF = DX_NB * (IDX_DIM / 4) / DX_T;
+ float4 pre[PF];
+ const int64_t stride = (int64_t) gridDim.x * DX_NB;
+ auto fetch = [&](int64_t tb) {
+#pragma unroll
+ for (int i = 0; i < PF; ++i) {
+ const int idx = t + i * DX_T, row = idx / (IDX_DIM / 4), c4 = idx % (IDX_DIM / 4);
+ pre[i] = tb + row < top ? __ldg(reinterpret_cast<const float4*>(pooled + (tb + row) * IDX_DIM) + c4)
+ : make_float4(0.f, 0.f, 0.f, 0.f);
+ }
+ };
+ fetch((int64_t) blockIdx.x * DX_NB);
+ for (int64_t b0 = (int64_t) blockIdx.x * DX_NB; b0 < top; b0 += stride) {
+#pragma unroll
+ for (int i = 0; i < PF; ++i) {
+ const int idx = t + i * DX_T, row = idx / (IDX_DIM / 4), c4 = idx % (IDX_DIM / 4);
+ *reinterpret_cast<float4*>(Ks + row * DX_KS + c4 * 4) = pre[i];
+ }
+ __syncthreads();
+ if (b0 + stride < top) fetch(b0 + stride);
+ float half0[NQ], half1[NQ];
+#pragma unroll 1
+ for (int c = 0; c < 4; ++c) {
+ float l0[NQ], l1[NQ], l2[NQ];
+#pragma unroll
+ for (int jj = 0; jj < 8; ++jj) {
+ const int L = dx_lane(8 * c + jj);
+ const float4 k4 = *reinterpret_cast<const float4*>(krow + L * 4);
+#pragma unroll
+ for (int q = 0; q < NQ; ++q) {
+ const float4 q4 = *reinterpret_cast<const float4*>(Qs + (q * IDX_HEADS + h) * IDX_DIM + L * 4);
+ const float d = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ // the depth-first tree over the slice's 8 leaves (pairs, quads, the octet), then the octets
+ if (jj == 0 || jj == 2 || jj == 4 || jj == 6) l0[q] = d;
+ else if (jj == 1 || jj == 5) l1[q] = l0[q] + d;
+ else if (jj == 3) l2[q] = l1[q] + (l0[q] + d);
+ else {
+ const float oct = l2[q] + (l1[q] + (l0[q] + d));
+ if (c == 0) half0[q] = oct;
+ else if (c == 1) half0[q] = half0[q] + oct;
+ else if (c == 2) half1[q] = oct;
+ else half1[q] = half1[q] + oct;
+ }
+ }
+ }
+ }
+#pragma unroll
+ for (int q = 0; q < NQ; ++q) Ds[(q * IDX_HEADS + h) * DX_NB + lane] = half0[q] + half1[q];
+ __syncthreads(); // also: every warp is done reading Ks before the next tile
+ for (int q = h; q < NQ; q += IDX_HEADS) {
+ const int64_t b = b0 + lane;
+ if (b >= s_nbid[q]) continue;
+ float score = 0.0f;
+#pragma unroll
+ for (int hh = 0; hh < IDX_HEADS; ++hh) {
+ const float d = Ds[(q * IDX_HEADS + hh) * DX_NB + lane];
+ score += d > 0.0f ? d : 0.0f;
+ }
+ out[q * max_blocks + b] = score;
+ }
+ }
+}
+
+// ---- the decode top-k over several CTAs per query without clusters (any CUDA card; on sm_6x by default under
+// STRATA_SELECT_LONG). A decode
+// window's 1-5 queries otherwise run one 1,024-thread CTA each (block_topk_wide_kernel past ~135K cells) on one SM
+// apiece: on GP100 0.39 ms per call at 300K cells and 1.15 ms at 1M, x12 QSA layers per window. Here each query gets
+// n CTAs (a grid of n x nq), CTA r taking blocks [r * per, (r + 1) * per), and the four radix passes, the count and the
+// emit are six kernels in a row on the caller's stream - capturable, no co-residency or spin waits: the kernel boundary
+// is the barrier. Exchange goes through the caller's scratch (qsa_topk_multi_scratch_ints): pass 0's histograms one
+// row per CTA (plain stores), passes 1-3 summed with atomics into rows the pass-0 kernel zeroes, (prefix, above) after
+// each digit, and each warp's cells above / at the threshold. Nothing is carried from one call to the next.
+// Warp w of a CTA takes the rows [w * rpw, (w + 1) * rpw) of 32 consecutive blocks (lane l block l of a row: coalesced),
+// MT_U rows' loads in flight at a time; the loops stay rolled (GP100's instruction cache: a fully unrolled 33-row body
+// doubled every kernel's time).
+//
+// IDENTICAL IDS, as block_topk_cluster_kernel's: thr (the width-th largest key, cells counted with their weights) and
+// `above` are pure functions of the integer histograms whatever order they are summed in, and the digit rule is
+// block_topk_kernel's (the largest digit d >= 1 whose cells at or above it reach `need`, else 0), here as a 256-thread
+// scan. The emit writes every cell of a block above thr and the first eq_budget = width - above cells at thr, ascending
+// (block_topk_wide_kernel's row emit, offset by the cells of the warps and CTAs before).
+constexpr int MT_T = 256; // threads per CTA
+constexpr int MT_W = MT_T / 32; // warps per CTA
+constexpr int MT_U = 4; // rows of loads in flight per warp
+constexpr int MT_NMAX = 32; // CTAs per query, at most
+constexpr int MT_MAXQ = 16; // more queries per call fill the GPU with one CTA each already
+// scratch per query (int32): pass 0's rows [MT_NMAX][256], passes 1-3 [3][256], state [5][2], counts [MT_NMAX][MT_W][2]
+constexpr int MT_ACC = MT_NMAX * 256, MT_STATE = MT_ACC + 3 * 256, MT_CNT = MT_STATE + 16;
+constexpr int MT_Q_INTS = MT_CNT + 2 * MT_NMAX * MT_W + 64;
+static_assert(MT_NMAX * MT_W <= MT_T, "the emit sums the earlier CTAs' warp counts one per thread");
+
+struct MtQuery {
+ int64_t n_kv, n_bid, width, lo, hi, row0; // row0: the first block of this warp's first row
+ int rpw, wtail;
+ bool all; // n_kv <= width: every cell, nothing to select
+};
+
+__device__ __forceinline__ MtQuery mt_query(const int32_t* __restrict__ steps) {
+ const int32_t* st = steps + (int64_t) blockIdx.y * kStepCount;
+ MtQuery m;
+ m.n_kv = st[kStepNKv];
+ m.n_bid = st[kStepNBid];
+ m.width = st[kStepWidth];
+ m.all = m.n_kv <= m.width;
+ // the tail block n_bid weighs its cells (0..3); with none it is not a candidate (the reference skips w == 0)
+ m.wtail = (int) (m.n_kv - m.n_bid * R);
+ const int64_t nbe = m.n_bid + (m.wtail > 0 ? 1 : 0);
+ const int64_t per = (nbe + gridDim.x - 1) / gridDim.x;
+ m.lo = (int64_t) blockIdx.x * per < nbe ? (int64_t) blockIdx.x * per : nbe;
+ m.hi = m.lo + per < nbe ? m.lo + per : nbe;
+ m.rpw = (int) (((m.hi - m.lo + 31) / 32 + MT_W - 1) / MT_W);
+ m.row0 = m.lo + (int64_t) (threadIdx.x >> 5) * m.rpw * 32;
+ return m;
+}
+
+// rows j0 .. j0 + MT_U - 1 of this warp: key[u] for block row0 + 32 (j0 + u) + lane (0 past hi or the warp's rows)
+__device__ __forceinline__ void mt_load(const MtQuery& m, const float* __restrict__ sc, int j0, uint32_t (&key)[MT_U]) {
+ const int lane = threadIdx.x & 31;
+#pragma unroll
+ for (int u = 0; u < MT_U; ++u) {
+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;
+ key[u] = (j0 + u < m.rpw && b < m.hi) ? order_key(__ldg(sc + b)) : 0u;
+ }
+}
+
+// inclusive scan of one value per thread over the MT_T threads
+template <typename V> __device__ __forceinline__ V mt_incl_scan(V v, V* s_w) {
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ V x = v;
+#pragma unroll
+ for (int o = 1; o < 32; o <<= 1) {
+ const V y = __shfl_up_sync(0xffffffffu, x, o);
+ if (lane >= o) x += y;
+ }
+ if (lane == 31) s_w[warp] = x;
+ __syncthreads();
+ V off = 0;
+ for (int w = 0; w < warp; ++w) off += s_w[w];
+ __syncthreads();
+ return x + off;
+}
+
+// The digit at `shift` from the 256 counts (thread t holds digit 255 - t's): block_topk_kernel's rule.
+__device__ __forceinline__ void mt_choose(int v, int64_t width, int shift, uint32_t& prefix, int& above, int* s_w,
+ int* s_ex, int* s_i) {
+ const int t = threadIdx.x;
+ if (t == 0) *s_i = 255; // no digit >= 1 reaches it: 0, all of 1..255 above
+ const int incl = mt_incl_scan(v, s_w);
+ s_ex[t] = incl - v;
+ const int64_t need = width - above;
+ if (t < 255 && incl >= need && incl - v < need) *s_i = t; // the one t where the running count crosses `need`
+ __syncthreads();
+ const int i = *s_i;
+ prefix |= (uint32_t) (255 - i) << shift;
+ above += s_ex[i];
+ __syncthreads();
+}
+
+// pass PASS's histogram of the CTA's keys under the prefix so far (PASS >= 1 first takes digit PASS - 1)
+template <int PASS>
+__global__ void __launch_bounds__(MT_T) block_topk_multi_hist_kernel(const float* __restrict__ scores,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, int32_t* __restrict__ scratch) {
+ __shared__ int h[MT_W][256];
+ __shared__ int s_w[MT_W], s_ex[256], s_i;
+ const MtQuery m = mt_query(steps);
+ if (m.all) return;
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;
+ uint32_t key[MT_U];
+ mt_load(m, sc, 0, key); // in flight while the digit is taken
+ for (int i = t; i < MT_W * 256; i += MT_T) (&h[0][0])[i] = 0;
+ int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;
+ uint32_t prefix = 0;
+ int above = 0;
+ if (PASS == 0) {
+ if (blockIdx.x == 0) // passes 1-3 sum into these: zero them for this call
+ for (int i = t; i < 3 * 256; i += MT_T) qs[MT_ACC + i] = 0;
+ } else {
+ int v = 0;
+ if (PASS == 1) { // pass 0: one row per CTA
+#pragma unroll 8
+ for (unsigned r = 0; r < gridDim.x; ++r) v += qs[r * 256 + 255 - t];
+ } else {
+ prefix = (uint32_t) qs[MT_STATE + 2 * (PASS - 1)];
+ above = qs[MT_STATE + 2 * (PASS - 1) + 1];
+ v = qs[MT_ACC + (PASS - 2) * 256 + 255 - t];
+ }
+ mt_choose(v, m.width, 32 - 8 * PASS, prefix, above, s_w, s_ex, &s_i);
+ if (blockIdx.x == 0 && t == 0) {
+ qs[MT_STATE + 2 * PASS] = (int) prefix;
+ qs[MT_STATE + 2 * PASS + 1] = above;
+ }
+ }
+ __syncthreads();
+ const int shift = 24 - 8 * PASS;
+ const uint32_t hmask = PASS == 0 ? 0u : (0xffffffffu << (shift + 8));
+#pragma unroll 1
+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) { // uniform over the warp: the ballots see all its lanes
+ if (j0 > 0) mt_load(m, sc, j0, key);
+#pragma unroll
+ for (int u = 0; u < MT_U; ++u) {
+ if (j0 + u >= m.rpw) break;
+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;
+ const int bin = (b < m.hi && (key[u] & hmask) == prefix) ? (int) ((key[u] >> shift) & 255) : -1;
+ // the scores share their top bits: the lanes on the first lane's bin in one atomic (no __match_any_sync on 6.x)
+ const unsigned pend = __ballot_sync(0xffffffffu, bin >= 0);
+ if (pend) {
+ const int leader = __ffs(pend) - 1;
+ const int lb = __shfl_sync(0xffffffffu, bin, leader);
+ const unsigned same = __ballot_sync(0xffffffffu, bin == lb);
+ if (lane == leader) atomicAdd(&h[warp][lb], __popc(same) * R);
+ else if (bin >= 0 && bin != lb) atomicAdd(&h[warp][bin], R);
+ }
+ if (bin >= 0 && b == m.n_bid) atomicAdd(&h[warp][bin], m.wtail - R);
+ }
+ }
+ __syncthreads();
+ int sum = 0;
+#pragma unroll
+ for (int w = 0; w < MT_W; ++w) sum += h[w][t];
+ if (PASS == 0) qs[blockIdx.x * 256 + t] = sum;
+ else if (sum != 0) atomicAdd(&qs[MT_ACC + (PASS - 1) * 256 + t], sum);
+}
+
+// digit 3 (the threshold), then each warp's cells above it and at it
+__global__ void __launch_bounds__(MT_T) block_topk_multi_count_kernel(const float* __restrict__ scores,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, int32_t* __restrict__ scratch) {
+ __shared__ int s_w[MT_W], s_ex[256], s_i;
+ const MtQuery m = mt_query(steps);
+ if (m.all) return;
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;
+ uint32_t key[MT_U];
+ mt_load(m, sc, 0, key);
+ int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;
+ uint32_t prefix = (uint32_t) qs[MT_STATE + 6];
+ int above = qs[MT_STATE + 7];
+ mt_choose(qs[MT_ACC + 2 * 256 + 255 - t], m.width, 0, prefix, above, s_w, s_ex, &s_i);
+ if (blockIdx.x == 0 && t == 0) {
+ qs[MT_STATE + 8] = (int) prefix;
+ qs[MT_STATE + 9] = above;
+ }
+ const uint32_t thr = prefix;
+ int gt = 0, eq = 0;
+#pragma unroll 1
+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) {
+ if (j0 > 0) mt_load(m, sc, j0, key);
+#pragma unroll
+ for (int u = 0; u < MT_U; ++u) {
+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;
+ if (j0 + u >= m.rpw || b >= m.hi) continue;
+ const int w = b == m.n_bid ? m.wtail : R;
+ if (key[u] > thr) gt += w;
+ else if (key[u] == thr) eq += w;
+ }
+ }
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) {
+ gt += __shfl_xor_sync(0xffffffffu, gt, o);
+ eq += __shfl_xor_sync(0xffffffffu, eq, o);
+ }
+ if (lane == 0) {
+ qs[MT_CNT + 2 * (blockIdx.x * MT_W + warp)] = gt;
+ qs[MT_CNT + 2 * (blockIdx.x * MT_W + warp) + 1] = eq;
+ }
+}
+
+// the cells, ascending: a warp's first cell goes after the cells the CTAs and warps before it selected
+__global__ void __launch_bounds__(MT_T) block_topk_multi_emit_kernel(const float* __restrict__ scores,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, int64_t cap,
+ int32_t* __restrict__ ids,
+ const int32_t* __restrict__ scratch) {
+ __shared__ unsigned long long s_w[MT_W];
+ const MtQuery m = mt_query(steps);
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ int32_t* out = ids + (int64_t) blockIdx.y * cap;
+ if (m.all) { // the identity, split over the CTAs
+ for (int64_t j = (int64_t) blockIdx.x * MT_T + t; j < m.n_kv; j += (int64_t) gridDim.x * MT_T) out[j] = (int32_t) j;
+ return;
+ }
+ const float* sc = scores + (int64_t) blockIdx.y * max_blocks;
+ uint32_t key[MT_U];
+ mt_load(m, sc, 0, key);
+ const int32_t* qs = scratch + (int64_t) blockIdx.y * MT_Q_INTS;
+ const uint32_t thr = (uint32_t) qs[MT_STATE + 8];
+ const int eq_budget = (int) (m.width - qs[MT_STATE + 9]); // cells at thr that fit, lowest index first
+ // the cells of the warps before this one: the earlier CTAs' (one warp count per thread), then this CTA's
+ unsigned long long c = 0;
+ if (t < (int) blockIdx.x * MT_W)
+ c = (unsigned long long) (unsigned) qs[MT_CNT + 2 * t] | ((unsigned long long) (unsigned) qs[MT_CNT + 2 * t + 1] << 32);
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) c += __shfl_xor_sync(0xffffffffu, c, o);
+ if (lane == 0) s_w[warp] = c;
+ __syncthreads();
+ unsigned long long before = 0;
+ for (int w = 0; w < MT_W; ++w) before += s_w[w];
+ for (int w = 0; w < warp; ++w) {
+ const int i = (int) blockIdx.x * MT_W + w;
+ before += (unsigned long long) (unsigned) qs[MT_CNT + 2 * i] |
+ ((unsigned long long) (unsigned) qs[MT_CNT + 2 * i + 1] << 32);
+ }
+ const int gt_before = (int) (before & 0xffffffffu), eq_before = (int) (before >> 32);
+ int run_sel = gt_before + (eq_before < eq_budget ? eq_before : eq_budget), run_eq = eq_before;
+#pragma unroll 1
+ for (int j0 = 0; j0 < m.rpw; j0 += MT_U) {
+ if (j0 > 0) mt_load(m, sc, j0, key);
+#pragma unroll
+ for (int u = 0; u < MT_U; ++u) {
+ if (j0 + u >= m.rpw) break;
+ const int64_t b = m.row0 + 32 * (j0 + u) + lane;
+ const int w = b < m.hi ? (b == m.n_bid ? m.wtail : R) : 0;
+ const uint32_t k = key[u];
+ const int my_gt = (w && k > thr) ? w : 0, my_eq = (w && k == thr) ? w : 0;
+ if (__ballot_sync(0xffffffffu, (my_gt | my_eq) != 0) == 0u) continue; // a row without a selected block
+ int pe = my_eq; // tied cells up to and with this lane
+#pragma unroll
+ for (int o = 1; o < 32; o <<= 1) {
+ const int y = __shfl_up_sync(0xffffffffu, pe, o);
+ if (lane >= o) pe += y;
+ }
+ const int left = eq_budget - (run_eq + pe - my_eq);
+ const int take = left < 0 ? 0 : (left > my_eq ? my_eq : left);
+ const int my_sel = my_gt + take;
+ int ps = my_sel;
+#pragma unroll
+ for (int o = 1; o < 32; o <<= 1) {
+ const int y = __shfl_up_sync(0xffffffffu, ps, o);
+ if (lane >= o) ps += y;
+ }
+ int32_t* dst = out + run_sel + ps - my_sel;
+ for (int cc = 0; cc < my_sel; ++cc) dst[cc] = (int32_t) (b * R + cc);
+ run_eq += __shfl_sync(0xffffffffu, pe, 31);
+ run_sel += __shfl_sync(0xffffffffu, ps, 31);
+ }
+ }
+}
+#endif // !__HIPCC__ && STRATA_EXPERIMENTAL_SM60
} // namespace
bool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,
@@ -2584,4 +2963,153 @@ void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64
if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk: %s\n", cudaGetErrorString(e)); std::exit(1); }
}
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+// STRATA_SCORES_DX: 0 off, 1 on for any CUDA card; unset: on for compute capability 6.x (block_scores_dx_kernel).
+// `sms`: the device's multiprocessors (the grid: as many CTAs as fit on each).
+static bool scores_dx_device(int& sms) {
+ static const int env = [] {
+ const char* v = std::getenv("STRATA_SCORES_DX");
+ return v == nullptr || *v == '\0' ? -1 : std::atoi(v);
+ }();
+ if (env == 0) return false;
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess) { cudaGetLastError(); return false; }
+ static thread_local int cached_device = -1, cached_sms = 0;
+ static thread_local bool pascal = false;
+ if (dev != cached_device) {
+ int major = 0;
+ pascal = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+ strata::cc_major_of(major) == 6;
+ if (cudaDeviceGetAttribute(&cached_sms, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess) cached_sms = 0;
+ cudaGetLastError();
+ cached_device = dev;
+ }
+ sms = cached_sms;
+ return sms > 0 && (env > 0 || pascal);
+}
+
+template <int NQ>
+static void scores_dx_launch(const float* pooled, const float* q_idx, const int32_t* steps, int64_t max_blocks,
+ float* scores, int sms, cudaStream_t st) {
+ const size_t smem = (size_t) (DX_NB * DX_KS + NQ * IDX_HEADS * IDX_DIM + NQ * IDX_HEADS * DX_NB) * sizeof(float);
+ static thread_local int per_sm[64][9] = {}; // CTAs per multiprocessor, by device and NQ
+ int dev = 0;
+ cudaGetDevice(&dev);
+ int& occ = per_sm[dev & 63][NQ];
+ if (occ == 0 && (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&occ, block_scores_dx_kernel<NQ>, DX_T, smem) !=
+ cudaSuccess || occ < 1)) {
+ cudaGetLastError();
+ occ = 2;
+ }
+ const int64_t tiles = (max_blocks + DX_NB - 1) / DX_NB, want = (int64_t) occ * sms;
+ const unsigned grid = (unsigned) (want < tiles ? want : tiles);
+ block_scores_dx_kernel<NQ><<<grid, DX_T, smem, st>>>(pooled, q_idx, steps, max_blocks, scores);
+}
+
+static bool qsa_block_scores_dx(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps,
+ int64_t nq, int64_t max_blocks, float* scores, void* stream) {
+ int sms = 0;
+ if (nq < 1 || nq > MQ || !scores_dx_device(sms)) return false;
+ const cudaStream_t st = (cudaStream_t) stream;
+ switch (nq) {
+ case 1: scores_dx_launch<1>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 2: scores_dx_launch<2>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 3: scores_dx_launch<3>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 4: scores_dx_launch<4>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 5: scores_dx_launch<5>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 6: scores_dx_launch<6>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ case 7: scores_dx_launch<7>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ default: scores_dx_launch<8>(pooled, q_idx, steps, max_blocks, scores, sms, st); break;
+ }
+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, st>>>(dead, q_idx, steps, max_blocks, scores);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (dx): %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return true;
+}
+#endif
+
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+// STRATA_TOPK_MULTI: 0 off, 1 on for any CUDA card and capacity; unset: on for compute capability 6.x past the register
+// kernel's reach (GP100: 0.39 -> 0.04 ms per call at 300K cells; see block_topk_multi_hist_kernel)
+static int topk_multi_mode() {
+ static const int env = [] {
+ const char* v = std::getenv("STRATA_TOPK_MULTI");
+ return v == nullptr || *v == '\0' ? -1 : std::atoi(v);
+ }();
+ if (env >= 0) return env != 0 ? 1 : 0;
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess) { cudaGetLastError(); return 0; }
+ static thread_local int cached_device = -1;
+ static thread_local int mode = 0;
+ if (dev != cached_device) {
+ int major = 0;
+ mode = cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+ strata::cc_major_of(major) == 6 ? 2 : 0;
+ cudaGetLastError();
+ cached_device = dev;
+ }
+ return mode;
+}
+
+static bool qsa_block_topk_multi(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+ int32_t* ids, void* stream, int32_t* scratch) {
+ // CTAs per query from the capacity (the grid is fixed in a captured graph; each CTA's share follows n_bid): one per
+ // 2,048 blocks, at most MT_NMAX. STRATA_TOPK_MULTI_N: a fixed count (tests)
+ static const int n_env = [] { const char* v = std::getenv("STRATA_TOPK_MULTI_N"); return v ? std::atoi(v) : 0; }();
+ int64_t n = n_env > 0 ? n_env : (max_blocks + 2047) / 2048;
+ n = n < 1 ? 1 : n > MT_NMAX ? MT_NMAX : n;
+ const dim3 grid((unsigned) n, (unsigned) nq);
+ const cudaStream_t st = (cudaStream_t) stream;
+ block_topk_multi_hist_kernel<0><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);
+ block_topk_multi_hist_kernel<1><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);
+ block_topk_multi_hist_kernel<2><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);
+ block_topk_multi_hist_kernel<3><<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);
+ block_topk_multi_count_kernel<<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, scratch);
+ block_topk_multi_emit_kernel<<<grid, MT_T, 0, st>>>(scores, steps, max_blocks, cap, ids, scratch);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_topk multi: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return true;
+}
+#endif
+
+// ---- STRATA_SELECT_LONG (verify.cpp, opt-in): the decode select past ~64K cells. Outside the SM60 build (and on HIP)
+// these are the plain calls; the scratch is never asked for.
+int64_t qsa_topk_multi_scratch_ints(int64_t nq) {
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+ return (nq < MT_MAXQ ? nq : MT_MAXQ) * (int64_t) MT_Q_INTS;
+#else
+ (void) nq;
+ return 0;
+#endif
+}
+
+void qsa_block_topk_long(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,
+ const QsaShapes& s, int32_t* ids, void* stream, int64_t active_blocks, int32_t* multi_scratch) {
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+ // several CTAs per query through the caller's scratch (decode calls: no active count, a few queries)
+ static const bool old = std::getenv("STRATA_TOPK_OLD") != nullptr;
+ if (nq > 0 && multi_scratch != nullptr && !old && active_blocks <= 0 && nq <= MT_MAXQ && s.idx_block == R &&
+ cap >= qsa_selection_width(kTopkMaxCells, s)) {
+ const int mode = topk_multi_mode();
+ if ((mode == 1 || (mode == 2 && max_blocks > (int64_t) TK_T * TK_PER)) &&
+ qsa_block_topk_multi(scores, steps, nq, max_blocks, cap, ids, stream, multi_scratch))
+ return;
+ }
+#else
+ (void) multi_scratch;
+#endif
+ qsa_block_topk(scores, steps, nq, max_blocks, cap, s, ids, stream, active_blocks);
+}
+
+void qsa_block_scores_long(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
+#if !defined(__HIPCC__) && defined(STRATA_EXPERIMENTAL_SM60)
+ if (nq > 0 && s.idx_dim == IDX_DIM && s.idx_n_head == IDX_HEADS && s.idx_block == R && scores_multi_on() &&
+ nq <= MQ && active_blocks <= 0 &&
+ qsa_block_scores_dx(pooled, dead, q_idx, steps, nq, max_blocks, scores, stream))
+ return;
+#endif
+ qsa_block_scores(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream, active_blocks);
+}
+
} // namespace strata::kernels
diff --git a/src/kernels/qsa_select_long_parity.cpp b/src/kernels/qsa_select_long_parity.cpp
new file mode 100644
index 000000000..affb41ac0
--- /dev/null
+++ b/src/kernels/qsa_select_long_parity.cpp
@@ -0,0 +1,214 @@
+// src/kernels/qsa_select_long_parity.cpp - STRATA_SELECT_LONG's decode select (PR #3820): qsa_block_scores_long
+// (block_scores_dx_kernel) and qsa_block_topk_long (the multi-CTA top-k) against qsa_block_scores and qsa_block_topk,
+// bit for bit (GPU, synthetic data, no model).
+//
+// Needs the CUDA build with STRATA_EXPERIMENTAL_SM60 (elsewhere the _long calls forward to the plain ones and there is
+// nothing to compare; the test skips with 77) and STRATA_SCORES_DX=1 STRATA_TOPK_MULTI=1 in the environment, which make
+// the new kernels run on any card (on compute capability 6.x they are the default). STRATA_TOPK_MULTI_N=<n> fixes the
+// CTAs per query (the ctest "n3" case), otherwise one per 2,048 blocks of capacity.
+//
+// scores: random pooled keys, queries and a dead key; nq 1..8 queries with their own n_bid (a decode window's
+// consecutive positions, so tail blocks of 0..3 cells), n_bid 0 / 1 / 31 / 32 / 33 / 1000 / 70001 ...; the
+// scores of every block 0..n_bid of every query are compared by memcmp (the tail block n_bid included).
+// top-k: scores that are random, all equal, 7 distinct values (many ties across CTAs, so the "first eq_budget cells
+// at the threshold" rule is exercised) and random with a boosted tail block; contexts 1,000 (width equals
+// n_kv: the identity) up to 524,288 cells; nq 1, 2, 5, 8, 16; capacity max_blocks = the largest context / 4 + 2.
+// The selected ids of every query are compared on the first `width` entries. The multi-CTA scratch is filled
+// with junk before every call (the contract says no initial value) and must have been written (the new
+// kernels ran; a forwarded call would leave it untouched).
+//
+// qsa_select_long_parity [--device N] exit 0: all cases identical; 77: no GPU or a build without the kernels
+#include "strata/kernels/qsa.hpp"
+#include "strata/kernels/qsa_select.hpp"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace k = strata::kernels;
+
+namespace {
+void ck(cudaError_t e, const char* what) {
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "qsa_select_long_parity: %s: %s\n", what, cudaGetErrorString(e));
+ std::exit(2);
+ }
+}
+template <typename T> T* dalloc(size_t n) {
+ T* p = nullptr;
+ ck(cudaMalloc((void**) &p, n * sizeof(T) + 64), "cudaMalloc");
+ ck(cudaMemset(p, 0, n * sizeof(T) + 64), "cudaMemset");
+ return p;
+}
+template <typename T> T* upload(const std::vector<T>& h) {
+ T* d = dalloc<T>(h.size());
+ ck(cudaMemcpy(d, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice), "upload");
+ return d;
+}
+
+// steps of a decode window: query i at position pos0 + i
+std::vector<int32_t> make_steps(int nq, int64_t pos0, const k::QsaShapes& s) {
+ std::vector<int32_t> st((size_t) nq * k::kStepCount, 0);
+ for (int i = 0; i < nq; ++i) {
+ int32_t* p = st.data() + (size_t) i * k::kStepCount;
+ const int64_t pos = pos0 + i;
+ p[k::kStepPos] = (int32_t) pos;
+ p[k::kStepNKv] = (int32_t) (pos + 1);
+ p[k::kStepNBid] = (int32_t) ((pos + 1) / s.idx_block);
+ p[k::kStepWidth] = (int32_t) k::qsa_selection_width(pos + 1, s);
+ }
+ return st;
+}
+
+long long g_cases = 0, g_failed = 0;
+
+void check_scores(const k::QsaShapes& s, int nq, int64_t n_bid0, std::mt19937& rng) {
+ const int64_t D = s.idx_dim, H = s.idx_n_head;
+ const int64_t max_blocks = n_bid0 + nq + 8;
+ std::normal_distribution<float> nd(0.0f, 1.0f);
+ std::vector<float> pooled((size_t) (max_blocks * D)), dead((size_t) D), q((size_t) (nq * H * D));
+ for (auto& x : pooled) x = nd(rng);
+ for (auto& x : dead) x = nd(rng);
+ for (auto& x : q) x = nd(rng);
+ // the window's last query has n_bid0 blocks; the others one position earlier each (their tails differ)
+ const std::vector<int32_t> steps = make_steps(nq, n_bid0 * s.idx_block - 1 + (int64_t) (rng() % 4u) - (nq - 1), s);
+ float *dp = upload(pooled), *dd = upload(dead), *dq = upload(q);
+ int32_t* dst = upload(steps);
+ float* a = dalloc<float>((size_t) (nq * max_blocks));
+ float* b = dalloc<float>((size_t) (nq * max_blocks));
+ ck(cudaMemset(a, 0xFF, (size_t) (nq * max_blocks) * 4), "fill a");
+ ck(cudaMemset(b, 0xFF, (size_t) (nq * max_blocks) * 4), "fill b");
+ k::qsa_block_scores(dp, dd, dq, dst, nq, max_blocks, s, a, nullptr);
+ k::qsa_block_scores_long(dp, dd, dq, dst, nq, max_blocks, s, b, nullptr, -1);
+ ck(cudaDeviceSynchronize(), "scores");
+ std::vector<float> ha((size_t) (nq * max_blocks)), hb(ha.size());
+ ck(cudaMemcpy(ha.data(), a, ha.size() * 4, cudaMemcpyDeviceToHost), "read a");
+ ck(cudaMemcpy(hb.data(), b, hb.size() * 4, cudaMemcpyDeviceToHost), "read b");
+ bool ok = true;
+ for (int i = 0; i < nq && ok; ++i) {
+ const int64_t nb = steps[(size_t) i * k::kStepCount + k::kStepNBid];
+ for (int64_t j = 0; j <= nb; ++j) { // block n_bid is the tail
+ if (std::memcmp(&ha[(size_t) (i * max_blocks + j)], &hb[(size_t) (i * max_blocks + j)], 4) != 0) {
+ std::printf("FAIL scores nq %d n_bid0 %lld: query %d block %lld: %.9g vs %.9g\n", nq, (long long) n_bid0, i,
+ (long long) j, ha[(size_t) (i * max_blocks + j)], hb[(size_t) (i * max_blocks + j)]);
+ ok = false;
+ break;
+ }
+ }
+ }
+ ++g_cases;
+ if (!ok) ++g_failed;
+ for (void* p : {(void*) dp, (void*) dd, (void*) dq, (void*) dst, (void*) a, (void*) b}) cudaFree(p);
+}
+
+void check_topk(const k::QsaShapes& s, int nq, int64_t ctx, int mode, int64_t max_blocks, std::mt19937& rng, bool& multi_ran) {
+ const int64_t cap = k::qsa_selection_width(k::kTopkMaxCells, s);
+ const std::vector<int32_t> steps = make_steps(nq, ctx - nq, s);
+ std::vector<float> sc((size_t) (nq * max_blocks), -999.0f);
+ std::uniform_real_distribution<float> ur(0.0f, 200.0f);
+ for (int i = 0; i < nq; ++i) {
+ const int32_t nb = steps[(size_t) i * k::kStepCount + k::kStepNBid];
+ const bool tail = steps[(size_t) i * k::kStepCount + k::kStepNKv] % s.idx_block != 0;
+ for (int64_t j = 0; j <= nb; ++j) {
+ float v = mode == 1 ? 3.0f : mode == 2 ? (float) (rng() % 7u) : ur(rng);
+ sc[(size_t) (i * max_blocks + j)] = v;
+ }
+ if (tail && mode != 1) sc[(size_t) (i * max_blocks + nb)] += 1e9f;
+ }
+ float* ds = upload(sc);
+ int32_t* dst = upload(steps);
+ int32_t* ia = dalloc<int32_t>((size_t) (nq * cap));
+ int32_t* ib = dalloc<int32_t>((size_t) (nq * cap));
+ const int64_t scratch_ints = k::qsa_topk_multi_scratch_ints(nq);
+ int32_t* scratch = dalloc<int32_t>((size_t) scratch_ints);
+ ck(cudaMemset(scratch, 0xFF, (size_t) scratch_ints * 4), "scratch junk");
+ ck(cudaMemset(ia, 0xFF, (size_t) (nq * cap) * 4), "fill ia");
+ ck(cudaMemset(ib, 0xFF, (size_t) (nq * cap) * 4), "fill ib");
+ k::qsa_block_topk(ds, dst, nq, max_blocks, cap, s, ia, nullptr);
+ k::qsa_block_topk_long(ds, dst, nq, max_blocks, cap, s, ib, nullptr, -1, scratch);
+ ck(cudaDeviceSynchronize(), "topk");
+ std::vector<int32_t> ha((size_t) (nq * cap)), hb(ha.size()), hs((size_t) scratch_ints);
+ ck(cudaMemcpy(ha.data(), ia, ha.size() * 4, cudaMemcpyDeviceToHost), "read ia");
+ ck(cudaMemcpy(hb.data(), ib, hb.size() * 4, cudaMemcpyDeviceToHost), "read ib");
+ ck(cudaMemcpy(hs.data(), scratch, hs.size() * 4, cudaMemcpyDeviceToHost), "read scratch");
+ // the multi kernels write the scratch (pass 0's histogram rows) when the contexts are above the identity width
+ const bool touched = std::any_of(hs.begin(), hs.end(), [](int32_t v) { return v != -1; });
+ if (touched) multi_ran = true;
+ bool ok = true;
+ for (int i = 0; i < nq && ok; ++i) {
+ const int64_t w = steps[(size_t) i * k::kStepCount + k::kStepWidth];
+ if (!std::equal(ha.begin() + i * cap, ha.begin() + i * cap + w, hb.begin() + i * cap)) {
+ size_t first = 0;
+ while (ha[(size_t) (i * cap) + first] == hb[(size_t) (i * cap) + first]) ++first;
+ std::printf("FAIL topk nq %d ctx %lld mode %d max_blocks %lld: query %d entry %zu: %d vs %d\n", nq,
+ (long long) ctx, mode, (long long) max_blocks, i, first, ha[(size_t) (i * cap) + first],
+ hb[(size_t) (i * cap) + first]);
+ ok = false;
+ }
+ }
+ ++g_cases;
+ if (!ok) ++g_failed;
+ for (void* p : {(void*) ds, (void*) dst, (void*) ia, (void*) ib, (void*) scratch}) cudaFree(p);
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ int device = 0;
+ for (int i = 1; i < argc; ++i) {
+ if (std::strcmp(argv[i], "--device") == 0 && i + 1 < argc) device = std::atoi(argv[++i]);
+ else {
+ std::fprintf(stderr, "usage: qsa_select_long_parity [--device N]\n");
+ return 2;
+ }
+ }
+ int devices = 0;
+ if (cudaGetDeviceCount(&devices) != cudaSuccess || devices == 0) {
+ std::puts("SKIP: no GPU");
+ return 77;
+ }
+ ck(cudaSetDevice(device), "cudaSetDevice");
+ if (k::qsa_topk_multi_scratch_ints(1) <= 0) {
+ std::puts("SKIP: this build has no multi-CTA select (CUDA with STRATA_EXPERIMENTAL_SM60)");
+ return 77;
+ }
+ const char* dx = std::getenv("STRATA_SCORES_DX");
+ const char* mt = std::getenv("STRATA_TOPK_MULTI");
+ if (dx == nullptr || dx[0] != '1' || mt == nullptr || mt[0] != '1') {
+ std::fprintf(stderr, "qsa_select_long_parity: set STRATA_SCORES_DX=1 and STRATA_TOPK_MULTI=1 (the kernels are "
+ "otherwise default only on compute capability 6.x)\n");
+ return 2;
+ }
+ cudaDeviceProp prop{};
+ ck(cudaGetDeviceProperties(&prop, device), "cudaGetDeviceProperties");
+ std::printf("qsa_select_long_parity on device %d (%s, %d.%d), STRATA_TOPK_MULTI_N=%s\n", device, prop.name, prop.major,
+ prop.minor, std::getenv("STRATA_TOPK_MULTI_N") ? std::getenv("STRATA_TOPK_MULTI_N") : "(per capacity)");
+
+ const k::QsaShapes s = k::qsa_real_shapes();
+ std::mt19937 rng(20261026u);
+ for (int nq : {1, 2, 3, 5, 8})
+ for (int64_t nb : {(int64_t) 8, (int64_t) 31, (int64_t) 32, (int64_t) 33, (int64_t) 1000, (int64_t) 70001, (int64_t) 131072})
+ check_scores(s, nq, nb, rng);
+
+ bool multi_ran = false;
+ const int64_t max_ctx = 524288, max_blocks = max_ctx / s.idx_block + 2;
+ for (int nq : {1, 2, 5, 8, 16})
+ for (int64_t ctx : {(int64_t) 1000, (int64_t) 2051, (int64_t) 2055, (int64_t) 8192, (int64_t) 70000, (int64_t) 135164,
+ (int64_t) 135168, (int64_t) 300003, max_ctx})
+ for (int mode = 0; mode < 3; ++mode)
+ check_topk(s, nq, ctx, mode, max_blocks, rng, multi_ran);
+ // a smaller capacity than the context asks for (max_blocks only bounds the row stride)
+ for (int nq : {1, 4}) check_topk(s, nq, 40000, 0, 40000 / s.idx_block + 2, rng, multi_ran);
+ if (!multi_ran) {
+ std::printf("FAIL: the multi-CTA top-k never wrote its scratch: the new kernels did not run\n");
+ ++g_failed;
+ }
+ std::printf("qsa_select_long_parity: %lld cases, %lld differ -> %s\n", g_cases, g_failed, g_failed == 0 ? "PASS" : "FAIL");
+ return g_failed == 0 ? 0 : 1;
+}