Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group batch (f-145-pp-batch)
Niko1221e8b1edbcb7cc
diff --cc include/strata/core/verify.hppindex a3d5fa7f9,358aacd57..f4d982512--- a/include/strata/core/verify.hpp+++ b/include/strata/core/verify.hpp@@@ -172,12 -170,14 +172,20 @@@ publicvoid set_stage(int64_t layer_begin, int64_t layer_end, const float* handoff_in, float* handoff_out) {lb_ = layer_begin; le_ = layer_end; hand_in_ = handoff_in; hand_out_ = handoff_out;}+ /// The rows the hand-off buffers of `set_stage` hold (0: kVerifyMaxT, the default). More than --batch's 8 slots on a+ /// layer split: the pipelined slot groups write their rows at hbase = group * rows-per-group, one row per slot.+ void set_handoff_rows(int rows) { handoff_rows_ = rows; }+ /// Is the zero-doorbell graph the one a batch window would run now (every expert of the stage in VRAM by the host+ /// table)? Looks at the table, as each window does. (STRATA_ADMIT_MERGE_FIRST's gate.)+ bool ar_live() { refresh_ar(); return ar_on(); }+ /// STRATA_SPLIT_OVERLAP=1 (opt-in; stages on DIFFERENT devices only): the hand-off carries a ready flag (one+ /// mapped word per hand-off buffer: host and device addresses). The writing stage raises it from the GPU when+ /// its hand-off is out (handoff_publish); the reading stage's window graph waits on it (wait_flag_ge) - so the+ /// next stage's graph is launched while this stage is still running, and the host no longer syncs between the+ /// stages. Null (the default): the old order - sync, then launch the next stage. Set before `init`.+ void set_handoff_flags(uint32_t* in_host, uint32_t* in_dev, uint32_t* out_host, uint32_t* out_dev) {+ hflag_in_h_ = in_host; hflag_in_d_ = in_dev; hflag_out_h_ = out_host; hflag_out_d_ = out_dev;+ }/// The next stage: `run` and `commit` continue into it (its pool calls get `next_user`); sampling settings/// and `final_R` are the last stage's.void set_next(Verifier* next, void* next_user) { next_ = next; next_user_ = next_user; }@@@ -420,9 -439,23 +447,24 @@@ privateunsigned* qcnt_ = nullptr; ///< S26 STRATA_QFUSE: the HC read's q8_1 group counters (n_embd / 32)unsigned long long* slot_off_d_ = nullptr; ///< E-6: the slot offsets on the deviceint64_t lb_ = 0, le_ = -1; ///< set_stage: the layers this verifier runs (-1: to the last)+ int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyMaxT)const float* hand_in_ = nullptr;float* hand_out_ = nullptr;+ uint32_t* hflag_in_h_ = nullptr; ///< set_handoff_flags: the incoming hand-off's ready word (host / device)+ uint32_t* hflag_in_d_ = nullptr;+ uint32_t* hflag_out_h_ = nullptr; ///< ... and the outgoing one's+ uint32_t* hflag_out_d_ = nullptr;+ uint32_t* hcount_d_ = nullptr; ///< handoff_publish's block counter (device memory)+ uint32_t* herr_h_ = nullptr; ///< the bounded hand-off wait gave up (mapped; the reading stage's)+ uint32_t* herr_d_ = nullptr;+ uint32_t* hdrop_h_ = nullptr; ///< STRATA_TEST_HANDOFF_DROP: the writing stage withholds its flag (mapped)+ uint32_t* hdrop_d_ = nullptr;+ bool handoff_failed(std::string& err) const; ///< the bounded hand-off wait timed out in the last window+ int prelaunched_ = 0; ///< prelaunch ran for a window of this size: run skips staging + launch+ /// The overlapped split: stage this window's inputs and launch its graph now (the graph waits on the hand-off+ /// flag); `run` then only serves the host side. Called on a helper thread while the previous stage runs.+ bool prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err);+ bool chain_overlap() const; ///< this stage hands off through a flag to a next stage that waits on itVerifier* next_ = nullptr;void* next_user_ = nullptr;bool ple_stage() const { return lb_ <= 1 && 1 < le_; } ///< holds layer 1, where the PLE block runsdiff --cc include/strata/kernels/qsa_decode_attn.hppindex 3270d8b6f,d2ecc49e0..e782035dd--- a/include/strata/kernels/qsa_decode_attn.hpp+++ b/include/strata/kernels/qsa_decode_attn.hpp@@@ -43,30 -50,9 +50,34 @@@ void qsa_decode_attn_step(const float*void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);+/// Up to this many queries of different slots in one qsa_decode_attn_slots call.+constexpr int kQsaAttnMaxSlots = 8;++/// One KV pool set per query of a qsa_decode_attn_slots call (passed by value as a kernel argument).+struct QsaAttnSlotPools {+ QsaAttnPools p[kQsaAttnMaxSlots];+};++/// qsa_decode_attn_batch for `n_q` queries that each attend over their OWN pools (`pools.p[i]`, one KV cache per batch+/// slot): one launch instead of one call per query, every block computing exactly what the per-query call computes.+/// The pools must all have one KV format. False (nothing launched) where the per-query call would run another kernel+/// (n_q > kQsaAttnMaxSlots, mixed formats, the pre-sm_75 / gfx103x or gfx906 variants): the caller loops instead.+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);+/// For src/kernels/qsa_merge_daily_parity.cpp only: qsa_decode_attn_batch's two passes apart, whatever the environment+/// says. `chunks_only` runs the chunk pass into `scratch` with `daily_chunk` 0: attn_chunk_kernel (upstream's default),+/// 1: the Daily's (attn_chunk_kernel_pre75, what STRATA_ATTN_DAILY=1 picks on CUDA). `merge_only` merges those+/// partials with `v2` 0: attn_merge_kernel, 1: attn_merge_v2_kernel (STRATA_ATTN_MERGE_V2, on with STRATA_ATTN_DAILY).+/// Width must not exceed an unaligned cap (the chunk pass keeps its upstream contract); for aligned caps the grid+/// bounds widths above cap. One input can go through every pair and the outputs be compared bit for bit.+void qsa_decode_attn_chunks_only(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, int daily_chunk,+ void* stream);+void qsa_decode_attn_merge_only(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch,+ float* attn, int64_t n_q, int v2, void* stream);+ /// qsa_decode_attn_batch for up to kAttnRowsMax queries of DIFFERENT sequences (batch slots), query z attending+ /// over rows.p[z] - one launch instead of one per query, the same per-query arithmetic (bit-identical outputs).+ void qsa_decode_attn_batch_rows(const float* q, const QsaAttnPoolsRows& rows, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);} // namespace strata::kernelsdiff --cc include/strata/prefill/prefill.hppindex 9b2893faa,11d1560da..197dc25fe--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@@ -206,12 -211,9 +217,13 @@@ privatePrefill* next_ = nullptr;Prefill* helper_ = nullptr; ///< set_stage_helperbool single_chunk_ = false; ///< a later stage: the prompt is one chunk (set by the stage before)+ int64_t stopped_at_ = -1; ///< stop_at_chunk ended the last run after this many tokens (-1: it read all)bool bind_stage_helper(int64_t T); // binds the helper's buffers for a one-chunk prompt of T tokensconst float* hand_in_ = nullptr; ///< the previous stage's rows of the chunk being read (host, pinned)+ /// STRATA_HANDOFF_PIPE: the previous stage's per-piece "landed in hand_in_" events (cudaEvent_t[hand_pieces_],+ /// on its device) when it copies the chunk in pieces without a host wait; null with the one-copy hand-off+ const void* hand_ev_ = nullptr;+ int hand_pieces_ = 0;std::string next_err_;std::future<bool> next_run_;diff --cc src/core/verify.cppindex 8c945c4e1,5bdc1093e..6c1c1e3e4--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@@ -1267,18 -1234,28 +1310,36 @@@ bool Verifier::record_window(int T, cud(const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV,n, cs);stamp(l, 4, grp);- mm(wg, z_ + (size_t) tb * ZV, (int) N, (int) ZV);+ if (!side_proj) mm(wg, z_ + (size_t) tb * ZV, (int) N, (int) ZV);stamp(l, 5, grp);+ if (side_proj && cudaStreamWaitEvent(cs, pjoin_, 0) != cudaSuccess) {+ err = "verify: the gate projection's branch could not join";+ return false;+ }+ if (sidem && cudaStreamWaitEvent(cs, mjoin_, 0) != cudaSuccess) {+ err = "verify: the GDN gates' branch could not join";+ return false;+ }}// the recurrence from the untouched state over tokens [0, te); outputs only for this group's+ static const bool gdn_groups_on = [] { // STRATA_BATCH_GDN_GROUPS=1, all slots in one launch+ const char* v = std::getenv("STRATA_BATCH_GDN_GROUPS");+ return v != nullptr && std::atoi(v) != 0;+ }();+ if (batch_rec_ && gdn_groups_on && te - tb <= 8) {+ strata::kernels::GdnGroups gg;+ for (int t = tb; t < te;) {+ const int first = t;+ while (t < te && brow_[t] == brow_[first]) ++t;+ SessionState& sx = slot_ss(first);+ gg.state[gg.n] = sx.gdn_state + (size_t) (gi - sx.gdn_ord0) * gdn_floats;+ gg.first[gg.n] = first;+ gg.ntok[gg.n] = t - first;+ ++gg.n;+ }+ strata::kernels::gdn_step_norm_groups(gg, hb, (int) C, gate, beta, z_, (const float*) wnm->data, EPS,+ y_, (int) HK, (int) HV, false, cs);+ } elseif (batch_rec_) { // each slot's recurrence over its own proposed-token groupfor (int t = tb; t < te;) {const int first = t;@@@ -1546,47 -1421,75 +1607,76 @@@if (qbr) {join(1); // the indexer query, for the block scores} else {- mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));- if (qb) {- if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)- native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,- qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,- rope_scaling(), pos_ + tb * NH, cs);- } else {- // the q/gate split as a copy kernel: the window's chain stays kernel to kernel (PDL)- copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, cs);- norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);- }- if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, cs); // <Hq, Hk> = <q, k>- bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,- N, IQ * ID, n, cs);- norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);- } else {- for (int t = tb; t < te; ++t) {- float* qc = qcur_ + t * NH * HD;- if (fuse_nr) {- native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,- (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, cs);- } else {- if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,- (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {- err = "verify: the q/gate split failed";- return false;- }- norm_rope(qc, wqn, (int) NH, (int) HD, pos_ + t * NH);- }- if (st.kv_rot) fwht256_inplace_cuda(qc, NH, cs); // <Hq, Hk> = <q, k>- }- for (int t = tb; t < te; ++t) {- float* qx = qidx_ + t * IQ * ID;- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);- norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);- }- }+ if (!side_proj && !q_chain(cs)) return false;+ if (!sidem) idx_query(cs);}stamp(l, 10, grp);+ bool streamed = false; // a streamed KV cache keeps the per-row resolve-then-attend order+ if (batch_rec_ && g_qsa_slots)+ for (int t = tb; t < te; ++t) streamed = streamed || slot_ss(t).qsa_states[qi].kv_mode == 1;+ // STRATA_BATCH_ATTN_ROWS=1: the window's rows in ONE launch per step - scores with each row's+ // own indexer keys, the top-k over all rows (row-independent), attention with each row's own pools -+ // instead of 3-4 one-row launches per row; the same per-row arithmetic. Not with KV streaming+ // (qsa_kv_resolve has work per slot then).+ static const bool attn_rows = [] {+ const char* v = std::getenv("STRATA_BATCH_ATTN_ROWS");+ return v != nullptr && std::atoi(v) != 0;+ }();+ bool rows_ok = batch_rec_ && attn_rows && te - tb <= strata::kernels::kAttnRowsMax;+ for (int t = tb; rows_ok && t < te; ++t) rows_ok = slot_ss(t).qsa_states[qi].kv_mode != 1;+ if (rows_ok) {+ const int n_rows = te - tb;+ strata::kernels::QsaScoreRows sr;+ strata::kernels::QsaAttnPoolsRows pr;+ sr.n = pr.n = n_rows;+ for (int t = tb; t < te; ++t) {+ const QsaState& sx = slot_ss(t).qsa_states[qi];+ sr.pooled[t - tb] = sx.idx_pooled;+ sr.dead[t - tb] = sx.idx_dead;+ pr.p[t - tb] = qsa_attn_pools(sx);+ }+ qsa_block_scores_rows(sr, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n_rows, max_blocks_, s,+ scores_ + (size_t) tb * max_blocks_, cs);+ qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n_rows, max_blocks_, cap_,+ s, sel_ + (size_t) tb * cap_, cs);+ qsa_decode_attn_batch_rows(qcur_ + tb * NH * HD, pr, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount,+ cap_, s, attn_scratch_ + (size_t) tb * attn_scratch_floats_,+ attn_ + tb * NH * HD, n_rows, cs);+ } else- if (batch_rec_) { // each row selects and attends over its own slot's K/V+ if (batch_rec_ && g_qsa_slots && !streamed) {+ // the rows' block scores, top-k and attention, up to 8 rows per launch. The rows only read+ // their slots' keys and K/V here and write their own scores/sel/scratch/attn rows, so running+ // each step for all rows before the next is the per-row loop's result+ for (int t0 = tb; t0 < te; t0 += kQsaAttnMaxSlots) {+ const int m = std::min(te - t0, kQsaAttnMaxSlots);+ const float* pooled[kQsaAttnMaxSlots];+ const float* dead[kQsaAttnMaxSlots];+ QsaAttnSlotPools px;+ for (int i = 0; i < m; ++i) {+ const QsaState& sx = slot_ss(t0 + i).qsa_states[qi];+ pooled[i] = sx.idx_pooled;+ dead[i] = sx.idx_dead;+ px.p[i] = qsa_attn_pools(sx);+ }+ // a launcher that would not run the per-row call's kernel launches nothing: per-row calls+ if (!qsa_block_scores_slots(pooled, dead, qidx_ + t0 * IQ * ID, step_ + t0 * kStepCount, m,+ max_blocks_, s, scores_ + (size_t) t0 * max_blocks_, cs))+ for (int t = t0; t < t0 + m; ++t)+ qsa_block_scores(pooled[t - t0], dead[t - t0], qidx_ + t * IQ * ID, step_ + t * kStepCount,+ 1, max_blocks_, s, scores_ + (size_t) t * max_blocks_, cs);+ qsa_block_topk(scores_ + (size_t) t0 * max_blocks_, step_ + t0 * kStepCount, m, max_blocks_,+ cap_, s, sel_ + (size_t) t0 * cap_, cs);+ if (!qsa_decode_attn_slots(qcur_ + t0 * NH * HD, px, sel_ + (size_t) t0 * cap_,+ step_ + t0 * kStepCount, cap_, s,+ attn_scratch_ + (size_t) t0 * attn_scratch_floats_,+ attn_ + t0 * NH * HD, m, cs))+ for (int t = t0; t < t0 + m; ++t)+ qsa_decode_attn_batch(qcur_ + t * NH * HD, px.p[t - t0], sel_ + (size_t) t * cap_,+ step_ + t * kStepCount, cap_, s,+ attn_scratch_ + (size_t) t * attn_scratch_floats_,+ attn_ + t * NH * HD, 1, cs);+ }+ } else if (batch_rec_) { // each row selects and attends over its own slot's K/Vfor (int t = tb; t < te; ++t) {const QsaState& sx = slot_ss(t).qsa_states[qi];qsa_block_scores(sx.idx_pooled, sx.idx_dead, qidx_ + t * IQ * ID, step_ + t * kStepCount, 1,diff --cc src/kernels/cuda/qsa_decode_attn.cuindex fe1ee65e5,ca8a9d4b9..062ef664e--- a/src/kernels/cuda/qsa_decode_attn.cu+++ b/src/kernels/cuda/qsa_decode_attn.cu@@@ -116,7 -116,8 +116,9 @@@ __device__ __forceinline__ int reduce12return ((lane >> 4) & 1) * 8 + ((lane >> 3) & 1) * 4 + ((lane >> 2) & 1) * 2 + ((lane >> 1) & 1);}+// The chunk kernel's body; `p` is the query's KV pools (the call's one set, or blockIdx.z's slot's set).+ // (the body is a device function so the batch window's rows kernel below can run it over each query's own pools without+ // the default kernel's signature, or the function-pointer types the pre-sm_75 / HIP selections build, changing)template <int KV_MODE, bool LANE_CELL, bool QUERY_SWIZZLE, bool REDUCE12>__device__ __forceinline__ void attn_chunk_body(const float* __restrict__ q, QsaAttnPools p,const int32_t* __restrict__ ids, const int32_t* __restrict__ step,@@@ -289,26 -290,25 +291,40 @@@ __global__ void __launch_bounds__(THREAint page_size, float scale, float* __restrict__ part_acc,float* __restrict__ part_m, float* __restrict__ part_l,int n_chunks, int cap = 0, long long scratch_stride = 0) {- attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale, part_acc,- part_m, part_l, n_chunks, cap, scratch_stride);+ attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap,+ scratch_stride);+}++// The same blocks for queries of different slots: query blockIdx.z reads its own slot's pools, everything else is+// attn_chunk_kernel's batched form (qsa_decode_attn_slots).+template <int KV_MODE, bool LANE_CELL>+__global__ void __launch_bounds__(THREADS) attn_chunk_slots_kernel(const float* __restrict__ q, QsaAttnSlotPools ps,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale,+ float* __restrict__ part_acc,+ float* __restrict__ part_m,+ float* __restrict__ part_l, int n_chunks, int cap,+ long long scratch_stride) {+ attn_chunk_body<KV_MODE, LANE_CELL, false, false>(q, ps.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);}+ // Batch windows: query blockIdx.z attends over its own slot's pools (rows.p[z]); the default kernel otherwise.+ template <int KV_MODE>+ __global__ void __launch_bounds__(THREADS) attn_chunk_rows_kernel(const float* __restrict__ q, QsaAttnPoolsRows rows,+ const int32_t* __restrict__ ids,+ const int32_t* __restrict__ step, int n_kv_heads,+ int page_size, float scale,+ float* __restrict__ part_acc,+ float* __restrict__ part_m,+ float* __restrict__ part_l, int n_chunks, int cap,+ long long scratch_stride) {+ attn_chunk_body<KV_MODE, false, false, false>(q, rows.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);+ }+-// PR #540 (sskver): the same attention with fewer shuffles and less shared-memory traffic, bit for bit the kernel// above (same operands, same order); +7% prompt speed on a V100, where this kernel reads every prompt chunk. It// uses more registers (80 vs 38 on sm_70), so on CUDA it runs on cards below sm_75 only, and exists only in the@@@ -908,64 -738,58 +924,116 @@@ void qsa_decode_attn_batch(const float}}+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {+ if (n_q <= 0) return true;+ if (n_q > kQsaAttnMaxSlots) return false;+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps) {+ std::fprintf(stderr, "qsa_decode_attn_slots: unsupported geometry or missing buffers\n");+ std::exit(1);+ }+ const int kv_mode = attn_kv_mode(pools.p[0]);+ for (int64_t z = 0; z < n_q; ++z) {+ if (!pools.p[z].page_table) {+ std::fprintf(stderr, "qsa_decode_attn_slots: missing page table\n");+ std::exit(1);+ }+ if (attn_kv_mode(pools.p[z]) != kv_mode) return false;+ }+ const bool lane_cell = lane_cell_attn();+ // only where qsa_decode_attn_batch runs attn_chunk_kernel; its other variants keep the per-query calls+#if defined(STRATA_ATTN_PRE75_BUILT)+ if (!lane_cell && pre75_attn()) return false;+#endif+#if defined(STRATA_HIP_GFX906)+ static const bool query_swizzle = [] {+ const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';+ }();+ if (query_swizzle && !lane_cell && kv_mode == 1 && query_swizzle_gfx906_device()) return false;+#endif+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);+ float* part_acc = scratch;+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;+ float* part_l = part_m + (size_t) n_chunks * s.n_head;+ const float scale = 1.0f / sqrtf((float) HD);+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);+ cudaStream_t st = (cudaStream_t) stream;+#define STRATA_ATTN_SLOTS(M, LC) attn_chunk_slots_kernel<M, LC><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)+ if (lane_cell) {+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, true);+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, true);+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, true);+ else STRATA_ATTN_SLOTS(0, true);+ } else {+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, false);+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, false);+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, false);+ else STRATA_ATTN_SLOTS(0, false);+ }+#undef STRATA_ATTN_SLOTS+ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,+ (unsigned) n_q, st);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "qsa_decode_attn_slots: %s\n", cudaGetErrorString(e));+ std::exit(1);+ }+ return true;+}++ void qsa_decode_attn_batch_rows(const float* q, const QsaAttnPoolsRows& rows, const int32_t* ids, const int32_t* steps,+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {+ if (n_q <= 0) return;+ const QsaAttnPools& pools = rows.p[0];+ if (rows.n != n_q || n_q > kAttnRowsMax || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 ||+ !scratch || !ids || !steps) {+ std::fprintf(stderr, "qsa_decode_attn_batch_rows: unsupported geometry or missing buffers\n");+ std::exit(1);+ }+ const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3+ : (pools.k_q != nullptr ? 1 : 0));+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);+ float* part_acc = scratch;+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;+ float* part_l = part_m + (size_t) n_chunks * s.n_head;+ const float scale = 1.0f / sqrtf((float) HD);+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);+ cudaStream_t st = (cudaStream_t) stream;+ // The plain kernel is what qsa_decode_attn_batch runs unless one of its variants is on (lane-cell scoring, the+ // pre-sm_75 / gfx103x kernel, the gfx906 swizzle): there each row goes through that call, one launch per row.+ static const bool lane_cell_rows = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();+ bool per_row = lane_cell_rows;+ #if defined(STRATA_ATTN_PRE75_BUILT)+ per_row = per_row || pre75_attn();+ #endif+ #if defined(STRATA_HIP_GFX906)+ per_row = true;+ #endif+ if (per_row) {+ for (int64_t r = 0; r < n_q; ++r)+ qsa_decode_attn_batch(q + (size_t) r * s.n_head * HD, rows.p[r], ids + (size_t) r * cap,+ steps + (size_t) r * kStepCount, cap, s, scratch + (size_t) r * stride,+ attn + (size_t) r * s.n_head * HD, 1, stream);+ return;+ }+ #define STRATA_ATTN_ROWS(M) attn_chunk_rows_kernel<M><<<grid, THREADS, 0, st>>>(q, rows, ids, steps, (int) s.n_head_kv, \+ (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)+ if (kv_mode == 3) STRATA_ATTN_ROWS(3);+ else if (kv_mode == 2) STRATA_ATTN_ROWS(2);+ else if (kv_mode == 1) STRATA_ATTN_ROWS(1);+ else STRATA_ATTN_ROWS(0);+ #undef STRATA_ATTN_ROWS- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,- attn, stride);++ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,++ (unsigned) n_q, st);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "qsa_decode_attn_batch_rows: %s\n", cudaGetErrorString(e));+ std::exit(1);+ }+ }+uint64_t qsa_decode_attn_scratch_floats(int64_t cap, const QsaShapes& s) {const int64_t chunks = (cap + CHUNK - 1) / CHUNK;return (uint64_t) chunks * (uint64_t) s.n_head * (HD + 2) + 64;diff --cc src/kernels/cuda/qsa_select.cuindex 39db4f65d,8d405680c..02e6c750a--- a/src/kernels/cuda/qsa_select.cu+++ b/src/kernels/cuda/qsa_select.cu@@@ -831,11 -816,12 +831,12 @@@ __global__ void __launch_bounds__(TK_T// qsa_block_scores takes it for every call without an active-block count and at most MQ queries: the captured decode// window, the uncaptured decode, and prefill's pooled16 call.constexpr int MQ = 8;-__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,- const float* __restrict__ dead,- const float* __restrict__ q_idx,- const int32_t* __restrict__ steps, int nq,- int64_t max_blocks, float* __restrict__ out,- bool early_exit, QsaScoreRows srow = {}) {+__device__ __forceinline__ void block_scores_multi_body(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq, int64_t max_blocks,- float* __restrict__ out, bool early_exit) {++ float* __restrict__ out, bool early_exit,++ const QsaScoreRows& srow) {__shared__ __align__(16) float qs[MQ * IDX_HEADS * IDX_DIM];__shared__ int64_t s_nkv[MQ], s_nbid[MQ];if (!early_exit)@@@ -882,113 -872,6 +887,114 @@@}}+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out,- bool early_exit) {- block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit);++ bool early_exit, QsaScoreRows srow = {}) {++ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);+}++// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).+struct ScoreSlotKeys {+ const float* pooled[MQ];+ const float* dead[MQ];+};+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps,+ int64_t max_blocks, float* __restrict__ out,+ bool early_exit) {+ const unsigned z = blockIdx.y;+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory+ const float* pooled = keys.pooled[0];+ const float* dead = keys.dead[0];+#pragma unroll+ for (unsigned i = 1; i < MQ; ++i)+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,- steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit);++ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,++ QsaScoreRows{});+}++// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.+template <int NQ>+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,+ const float* __restrict__ dead,+ const float* __restrict__ q_idx,+ const int32_t* __restrict__ steps, int nq,+ int64_t max_blocks, float* __restrict__ out) {+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];+ if (threadIdx.x < nq) {+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];+ }+ __syncthreads();+ int64_t top = 0;+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)+ const int lane = threadIdx.x & 31;+ float4 qr[NQ][IDX_HEADS];+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi)+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h)+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)+ : make_float4(0.f, 0.f, 0.f, 0.f);+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);+#pragma unroll+ for (int qi = 0; qi < NQ; ++qi) {+ if (qi >= nq) break;+ const int64_t n_bid = s_nbid[qi];+ if (b > n_bid) continue;+ const float4 k4 = (b == n_bid) ? kd : kp;+ float d[IDX_HEADS];+#pragma unroll+ for (int h = 0; h < IDX_HEADS; ++h) {+ const float4 q4 = qr[qi][h];+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;+ }+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head+ float r2[2];+#pragma unroll+ for (int i = 0; i < 2; ++i)+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);+ r += __shfl_xor_sync(0xffffffffu, r, 4);+ r += __shfl_xor_sync(0xffffffffu, r, 2);+ r += __shfl_xor_sync(0xffffffffu, r, 1);+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel+ const float s1 = __shfl_sync(0xffffffffu, r, 8);+ const float s2 = __shfl_sync(0xffffffffu, r, 16);+ const float s3 = __shfl_sync(0xffffffffu, r, 24);+ if (lane == 0) {+ float score = 0.0f;+ score += r > 0.0f ? r : 0.0f;+ score += s1 > 0.0f ? s1 : 0.0f;+ score += s2 > 0.0f ? s2 : 0.0f;+ score += s3 > 0.0f ? s3 : 0.0f;+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;+ out[qi * max_blocks + b] = score;+ }+ }+ }+}+#if !defined(__HIPCC__)// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to@@@ -1269,21 -1122,20 +1275,35 @@@ void qsa_block_scores(const float* poolif (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores: %s\n", cudaGetErrorString(e)); std::exit(1); }}+bool qsa_block_scores_rows(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream) {+ if (nq <= 0) return true;+ if (nq > 4 || s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R) return false;+ cudaStream_t st = (cudaStream_t) stream;+ const int n = (int) nq;+ if (n == 1) block_scores_rows_kernel<1><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);+ else if (n == 2) block_scores_rows_kernel<2><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);+ else if (n == 3) block_scores_rows_kernel<3><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);+ else block_scores_rows_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores rows: %s\n", cudaGetErrorString(e)); std::exit(1); }+ return true;+}++ void qsa_block_scores_rows(const QsaScoreRows& rows, const float* q_idx, const int32_t* steps, int64_t nq,+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream) {+ if (nq <= 0) return;+ if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > MQ || rows.n != nq) {+ std::fprintf(stderr, "qsa_block_scores_rows: unsupported geometry\n");+ std::exit(1);+ }+ static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();+ block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(nullptr, nullptr, q_idx, steps, (int) nq,+ max_blocks, scores, early, rows);+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_rows: %s\n", cudaGetErrorString(e)); std::exit(1); }+ }+bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {if (nq <= 0) return true;diff --cc src/program/generate.cppindex ed20bbd0d,f766e8da9..09bdef8e7--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@@ -8275,7 -7860,11 +8297,12 @@@ int main(int argc, char** argv)for (int st = 0; st < n_stages; ++st) {stage_ver(st).set_stage(st == 0 ? 0 : split_at[(size_t) st - 1], st + 1 < n_stages ? split_at[(size_t) st] : -1,st == 0 ? nullptr : hand[(size_t) st - 1], st + 1 < n_stages ? hand[(size_t) st] : nullptr);+ if (hrows > strata::kernels::kVerifyMaxT) stage_ver(st).set_handoff_rows(hrows);+ if (overlap)+ stage_ver(st).set_handoff_flags(st == 0 ? nullptr : hflag_h[(size_t) st - 1],+ st == 0 ? nullptr : hflag_d[(size_t) st - 1],+ st + 1 < n_stages ? hflag_h[(size_t) st] : nullptr,+ st + 1 < n_stages ? hflag_d[(size_t) st] : nullptr);split_drive.end[st] = st + 1 < n_stages ? split_at[(size_t) st] : g.n_layers;split_drive.cache_base[st] = drive.d.cache_base;split_drive.cache_slot_off[st] = drive.d.cache_slot_off;@@@ -10788,11 -10221,13 +10817,18 @@@for (int t = 0; t < GS; ++t) if (bs[(size_t) (gi * GS + t)].active) return true;return false;};+ // STRATA_TRACE: the drain before the line in hand (an admission's first phase), and when the first rows came+ // after a batch admission (the pipeline's refill after BADM) or after a drain for another line (drain + refill)+ double line_drain_ms = 0.0, adm_drain_ms = 0.0;+ const char* rows_wait = nullptr; // what the next rows are timed from (rows_wait_t)+ Clock::time_point rows_wait_t{};+ // STRATA_PIPED_CONCURRENT (read_part): while a prompt is read on a thread of its own, the slots' doorbells are+ // served on another (conc_serving); between windows that thread waits on bell_cv and a launch wakes it (a+ // timed sleep is a whole timer tick on Windows, up to 15.6 ms, and the window's first layer would wait for it)+ bool conc_serving = false;+ std::mutex bell_mu;+ std::condition_variable bell_cv;+ std::atomic<bool> bell_waiting{false};// one turn: serve every running stage, hand finished groups on, start what can startauto pump = [&](bool may_start) -> bool {++pipe_tick;