Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group batch (f-145-pp-batch)

Niko1221e8b1edbcb7cc
diff --cc include/strata/core/verify.hpp
index a3d5fa7f9,358aacd57..f4d982512
--- a/include/strata/core/verify.hpp
+++ b/include/strata/core/verify.hpp
@@@ -172,12 -170,14 +172,20 @@@ public
void set_stage(int64_t layer_begin, int64_t layer_end, const float* handoff_in, float* handoff_out) {
lb_ = layer_begin; le_ = layer_end; hand_in_ = handoff_in; hand_out_ = handoff_out;
}
+ /// The rows the hand-off buffers of `set_stage` hold (0: kVerifyMaxT, the default). More than --batch's 8 slots on a
+ /// layer split: the pipelined slot groups write their rows at hbase = group * rows-per-group, one row per slot.
+ void set_handoff_rows(int rows) { handoff_rows_ = rows; }
+ /// Is the zero-doorbell graph the one a batch window would run now (every expert of the stage in VRAM by the host
+ /// table)? Looks at the table, as each window does. (STRATA_ADMIT_MERGE_FIRST's gate.)
+ bool ar_live() { refresh_ar(); return ar_on(); }
+ /// STRATA_SPLIT_OVERLAP=1 (opt-in; stages on DIFFERENT devices only): the hand-off carries a ready flag (one
+ /// mapped word per hand-off buffer: host and device addresses). The writing stage raises it from the GPU when
+ /// its hand-off is out (handoff_publish); the reading stage's window graph waits on it (wait_flag_ge) - so the
+ /// next stage's graph is launched while this stage is still running, and the host no longer syncs between the
+ /// stages. Null (the default): the old order - sync, then launch the next stage. Set before `init`.
+ void set_handoff_flags(uint32_t* in_host, uint32_t* in_dev, uint32_t* out_host, uint32_t* out_dev) {
+ hflag_in_h_ = in_host; hflag_in_d_ = in_dev; hflag_out_h_ = out_host; hflag_out_d_ = out_dev;
+ }
/// The next stage: `run` and `commit` continue into it (its pool calls get `next_user`); sampling settings
/// and `final_R` are the last stage's.
void set_next(Verifier* next, void* next_user) { next_ = next; next_user_ = next_user; }
@@@ -420,9 -439,23 +447,24 @@@ private
unsigned* qcnt_ = nullptr; ///< S26 STRATA_QFUSE: the HC read's q8_1 group counters (n_embd / 32)
unsigned long long* slot_off_d_ = nullptr; ///< E-6: the slot offsets on the device
int64_t lb_ = 0, le_ = -1; ///< set_stage: the layers this verifier runs (-1: to the last)
+ int handoff_rows_ = 0; ///< set_handoff_rows (0: kVerifyMaxT)
const float* hand_in_ = nullptr;
float* hand_out_ = nullptr;
+ uint32_t* hflag_in_h_ = nullptr; ///< set_handoff_flags: the incoming hand-off's ready word (host / device)
+ uint32_t* hflag_in_d_ = nullptr;
+ uint32_t* hflag_out_h_ = nullptr; ///< ... and the outgoing one's
+ uint32_t* hflag_out_d_ = nullptr;
+ uint32_t* hcount_d_ = nullptr; ///< handoff_publish's block counter (device memory)
+ uint32_t* herr_h_ = nullptr; ///< the bounded hand-off wait gave up (mapped; the reading stage's)
+ uint32_t* herr_d_ = nullptr;
+ uint32_t* hdrop_h_ = nullptr; ///< STRATA_TEST_HANDOFF_DROP: the writing stage withholds its flag (mapped)
+ uint32_t* hdrop_d_ = nullptr;
+ bool handoff_failed(std::string& err) const; ///< the bounded hand-off wait timed out in the last window
+ int prelaunched_ = 0; ///< prelaunch ran for a window of this size: run skips staging + launch
+ /// The overlapped split: stage this window's inputs and launch its graph now (the graph waits on the hand-off
+ /// flag); `run` then only serves the host side. Called on a helper thread while the previous stage runs.
+ bool prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err);
+ bool chain_overlap() const; ///< this stage hands off through a flag to a next stage that waits on it
Verifier* next_ = nullptr;
void* next_user_ = nullptr;
bool ple_stage() const { return lb_ <= 1 && 1 < le_; } ///< holds layer 1, where the PLE block runs
diff --cc include/strata/kernels/qsa_decode_attn.hpp
index 3270d8b6f,d2ecc49e0..e782035dd
--- a/include/strata/kernels/qsa_decode_attn.hpp
+++ b/include/strata/kernels/qsa_decode_attn.hpp
@@@ -43,30 -50,9 +50,34 @@@ void qsa_decode_attn_step(const float*
void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);
+/// Up to this many queries of different slots in one qsa_decode_attn_slots call.
+constexpr int kQsaAttnMaxSlots = 8;
+
+/// One KV pool set per query of a qsa_decode_attn_slots call (passed by value as a kernel argument).
+struct QsaAttnSlotPools {
+ QsaAttnPools p[kQsaAttnMaxSlots];
+};
+
+/// qsa_decode_attn_batch for `n_q` queries that each attend over their OWN pools (`pools.p[i]`, one KV cache per batch
+/// slot): one launch instead of one call per query, every block computing exactly what the per-query call computes.
+/// The pools must all have one KV format. False (nothing launched) where the per-query call would run another kernel
+/// (n_q > kQsaAttnMaxSlots, mixed formats, the pre-sm_75 / gfx103x or gfx906 variants): the caller loops instead.
+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);
+/// For src/kernels/qsa_merge_daily_parity.cpp only: qsa_decode_attn_batch's two passes apart, whatever the environment
+/// says. `chunks_only` runs the chunk pass into `scratch` with `daily_chunk` 0: attn_chunk_kernel (upstream's default),
+/// 1: the Daily's (attn_chunk_kernel_pre75, what STRATA_ATTN_DAILY=1 picks on CUDA). `merge_only` merges those
+/// partials with `v2` 0: attn_merge_kernel, 1: attn_merge_v2_kernel (STRATA_ATTN_MERGE_V2, on with STRATA_ATTN_DAILY).
+/// Width must not exceed an unaligned cap (the chunk pass keeps its upstream contract); for aligned caps the grid
+/// bounds widths above cap. One input can go through every pair and the outputs be compared bit for bit.
+void qsa_decode_attn_chunks_only(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, int64_t n_q, int daily_chunk,
+ void* stream);
+void qsa_decode_attn_merge_only(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch,
+ float* attn, int64_t n_q, int v2, void* stream);
+ /// qsa_decode_attn_batch for up to kAttnRowsMax queries of DIFFERENT sequences (batch slots), query z attending
+ /// over rows.p[z] - one launch instead of one per query, the same per-query arithmetic (bit-identical outputs).
+ void qsa_decode_attn_batch_rows(const float* q, const QsaAttnPoolsRows& rows, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream);
} // namespace strata::kernels
diff --cc include/strata/prefill/prefill.hpp
index 9b2893faa,11d1560da..197dc25fe
--- a/include/strata/prefill/prefill.hpp
+++ b/include/strata/prefill/prefill.hpp
@@@ -206,12 -211,9 +217,13 @@@ private
Prefill* next_ = nullptr;
Prefill* helper_ = nullptr; ///< set_stage_helper
bool single_chunk_ = false; ///< a later stage: the prompt is one chunk (set by the stage before)
+ int64_t stopped_at_ = -1; ///< stop_at_chunk ended the last run after this many tokens (-1: it read all)
bool bind_stage_helper(int64_t T); // binds the helper's buffers for a one-chunk prompt of T tokens
const float* hand_in_ = nullptr; ///< the previous stage's rows of the chunk being read (host, pinned)
+ /// STRATA_HANDOFF_PIPE: the previous stage's per-piece "landed in hand_in_" events (cudaEvent_t[hand_pieces_],
+ /// on its device) when it copies the chunk in pieces without a host wait; null with the one-copy hand-off
+ const void* hand_ev_ = nullptr;
+ int hand_pieces_ = 0;
std::string next_err_;
std::future<bool> next_run_;
diff --cc src/core/verify.cpp
index 8c945c4e1,5bdc1093e..6c1c1e3e4
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -1267,18 -1234,28 +1310,36 @@@ bool Verifier::record_window(int T, cud
(const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV,
n, cs);
stamp(l, 4, grp);
- mm(wg, z_ + (size_t) tb * ZV, (int) N, (int) ZV);
+ if (!side_proj) mm(wg, z_ + (size_t) tb * ZV, (int) N, (int) ZV);
stamp(l, 5, grp);
+ if (side_proj && cudaStreamWaitEvent(cs, pjoin_, 0) != cudaSuccess) {
+ err = "verify: the gate projection's branch could not join";
+ return false;
+ }
+ if (sidem && cudaStreamWaitEvent(cs, mjoin_, 0) != cudaSuccess) {
+ err = "verify: the GDN gates' branch could not join";
+ return false;
+ }
}
// the recurrence from the untouched state over tokens [0, te); outputs only for this group's
+ static const bool gdn_groups_on = [] { // STRATA_BATCH_GDN_GROUPS=1, all slots in one launch
+ const char* v = std::getenv("STRATA_BATCH_GDN_GROUPS");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ if (batch_rec_ && gdn_groups_on && te - tb <= 8) {
+ strata::kernels::GdnGroups gg;
+ for (int t = tb; t < te;) {
+ const int first = t;
+ while (t < te && brow_[t] == brow_[first]) ++t;
+ SessionState& sx = slot_ss(first);
+ gg.state[gg.n] = sx.gdn_state + (size_t) (gi - sx.gdn_ord0) * gdn_floats;
+ gg.first[gg.n] = first;
+ gg.ntok[gg.n] = t - first;
+ ++gg.n;
+ }
+ strata::kernels::gdn_step_norm_groups(gg, hb, (int) C, gate, beta, z_, (const float*) wnm->data, EPS,
+ y_, (int) HK, (int) HV, false, cs);
+ } else
if (batch_rec_) { // each slot's recurrence over its own proposed-token group
for (int t = tb; t < te;) {
const int first = t;
@@@ -1546,47 -1421,75 +1607,76 @@@
if (qbr) {
join(1); // the indexer query, for the block scores
} else {
- mm(wq, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD));
- if (qb) {
- if (fuse_nr) { // the q/gate split reads q straight out of the q|gate rows (stride 2 * HD)
- native_qsa_rms_norm_rope(qfull_ + tb * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data,
- qcur_ + tb * NH * HD, (int) (n * NH), (int) HD, (int) s.n_rot, EPS,
- rope_scaling(), pos_ + tb * NH, cs);
- } else {
- // the q/gate split as a copy kernel: the window's chain stays kernel to kernel (PDL)
- copy_rows_strided(qcur_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, (int64_t) n * NH, HD, 2 * HD, cs);
- norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);
- }
- if (st.kv_rot) fwht256_inplace_cuda(qcur_ + tb * NH * HD, (int64_t) n * NH, cs); // <Hq, Hk> = <q, k>
- bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
- N, IQ * ID, n, cs);
- norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);
- } else {
- for (int t = tb; t < te; ++t) {
- float* qc = qcur_ + t * NH * HD;
- if (fuse_nr) {
- native_qsa_rms_norm_rope(qfull_ + t * NH * 2 * HD, (int) (2 * HD), (const float*) wqn->data, qc,
- (int) NH, (int) HD, (int) s.n_rot, EPS, rope_scaling(), pos_ + t * NH, cs);
- } else {
- if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
- (size_t) HD * 4, (size_t) NH, cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
- err = "verify: the q/gate split failed";
- return false;
- }
- norm_rope(qc, wqn, (int) NH, (int) HD, pos_ + t * NH);
- }
- if (st.kv_rot) fwht256_inplace_cuda(qc, NH, cs); // <Hq, Hk> = <q, k>
- }
- for (int t = tb; t < te; ++t) {
- float* qx = qidx_ + t * IQ * ID;
- bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);
- norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);
- }
- }
+ if (!side_proj && !q_chain(cs)) return false;
+ if (!sidem) idx_query(cs);
}
stamp(l, 10, grp);
+ bool streamed = false; // a streamed KV cache keeps the per-row resolve-then-attend order
+ if (batch_rec_ && g_qsa_slots)
+ for (int t = tb; t < te; ++t) streamed = streamed || slot_ss(t).qsa_states[qi].kv_mode == 1;
+ // STRATA_BATCH_ATTN_ROWS=1: the window's rows in ONE launch per step - scores with each row's
+ // own indexer keys, the top-k over all rows (row-independent), attention with each row's own pools -
+ // instead of 3-4 one-row launches per row; the same per-row arithmetic. Not with KV streaming
+ // (qsa_kv_resolve has work per slot then).
+ static const bool attn_rows = [] {
+ const char* v = std::getenv("STRATA_BATCH_ATTN_ROWS");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ bool rows_ok = batch_rec_ && attn_rows && te - tb <= strata::kernels::kAttnRowsMax;
+ for (int t = tb; rows_ok && t < te; ++t) rows_ok = slot_ss(t).qsa_states[qi].kv_mode != 1;
+ if (rows_ok) {
+ const int n_rows = te - tb;
+ strata::kernels::QsaScoreRows sr;
+ strata::kernels::QsaAttnPoolsRows pr;
+ sr.n = pr.n = n_rows;
+ for (int t = tb; t < te; ++t) {
+ const QsaState& sx = slot_ss(t).qsa_states[qi];
+ sr.pooled[t - tb] = sx.idx_pooled;
+ sr.dead[t - tb] = sx.idx_dead;
+ pr.p[t - tb] = qsa_attn_pools(sx);
+ }
+ qsa_block_scores_rows(sr, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n_rows, max_blocks_, s,
+ scores_ + (size_t) tb * max_blocks_, cs);
+ qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n_rows, max_blocks_, cap_,
+ s, sel_ + (size_t) tb * cap_, cs);
+ qsa_decode_attn_batch_rows(qcur_ + tb * NH * HD, pr, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount,
+ cap_, s, attn_scratch_ + (size_t) tb * attn_scratch_floats_,
+ attn_ + tb * NH * HD, n_rows, cs);
+ } else
- if (batch_rec_) { // each row selects and attends over its own slot's K/V
+ if (batch_rec_ && g_qsa_slots && !streamed) {
+ // the rows' block scores, top-k and attention, up to 8 rows per launch. The rows only read
+ // their slots' keys and K/V here and write their own scores/sel/scratch/attn rows, so running
+ // each step for all rows before the next is the per-row loop's result
+ for (int t0 = tb; t0 < te; t0 += kQsaAttnMaxSlots) {
+ const int m = std::min(te - t0, kQsaAttnMaxSlots);
+ const float* pooled[kQsaAttnMaxSlots];
+ const float* dead[kQsaAttnMaxSlots];
+ QsaAttnSlotPools px;
+ for (int i = 0; i < m; ++i) {
+ const QsaState& sx = slot_ss(t0 + i).qsa_states[qi];
+ pooled[i] = sx.idx_pooled;
+ dead[i] = sx.idx_dead;
+ px.p[i] = qsa_attn_pools(sx);
+ }
+ // a launcher that would not run the per-row call's kernel launches nothing: per-row calls
+ if (!qsa_block_scores_slots(pooled, dead, qidx_ + t0 * IQ * ID, step_ + t0 * kStepCount, m,
+ max_blocks_, s, scores_ + (size_t) t0 * max_blocks_, cs))
+ for (int t = t0; t < t0 + m; ++t)
+ qsa_block_scores(pooled[t - t0], dead[t - t0], qidx_ + t * IQ * ID, step_ + t * kStepCount,
+ 1, max_blocks_, s, scores_ + (size_t) t * max_blocks_, cs);
+ qsa_block_topk(scores_ + (size_t) t0 * max_blocks_, step_ + t0 * kStepCount, m, max_blocks_,
+ cap_, s, sel_ + (size_t) t0 * cap_, cs);
+ if (!qsa_decode_attn_slots(qcur_ + t0 * NH * HD, px, sel_ + (size_t) t0 * cap_,
+ step_ + t0 * kStepCount, cap_, s,
+ attn_scratch_ + (size_t) t0 * attn_scratch_floats_,
+ attn_ + t0 * NH * HD, m, cs))
+ for (int t = t0; t < t0 + m; ++t)
+ qsa_decode_attn_batch(qcur_ + t * NH * HD, px.p[t - t0], sel_ + (size_t) t * cap_,
+ step_ + t * kStepCount, cap_, s,
+ attn_scratch_ + (size_t) t * attn_scratch_floats_,
+ attn_ + t * NH * HD, 1, cs);
+ }
+ } else if (batch_rec_) { // each row selects and attends over its own slot's K/V
for (int t = tb; t < te; ++t) {
const QsaState& sx = slot_ss(t).qsa_states[qi];
qsa_block_scores(sx.idx_pooled, sx.idx_dead, qidx_ + t * IQ * ID, step_ + t * kStepCount, 1,
diff --cc src/kernels/cuda/qsa_decode_attn.cu
index fe1ee65e5,ca8a9d4b9..062ef664e
--- a/src/kernels/cuda/qsa_decode_attn.cu
+++ b/src/kernels/cuda/qsa_decode_attn.cu
@@@ -116,7 -116,8 +116,9 @@@ __device__ __forceinline__ int reduce12
return ((lane >> 4) & 1) * 8 + ((lane >> 3) & 1) * 4 + ((lane >> 2) & 1) * 2 + ((lane >> 1) & 1);
}
+// The chunk kernel's body; `p` is the query's KV pools (the call's one set, or blockIdx.z's slot's set).
+ // (the body is a device function so the batch window's rows kernel below can run it over each query's own pools without
+ // the default kernel's signature, or the function-pointer types the pre-sm_75 / HIP selections build, changing)
template <int KV_MODE, bool LANE_CELL, bool QUERY_SWIZZLE, bool REDUCE12>
__device__ __forceinline__ void attn_chunk_body(const float* __restrict__ q, QsaAttnPools p,
const int32_t* __restrict__ ids, const int32_t* __restrict__ step,
@@@ -289,26 -290,25 +291,40 @@@ __global__ void __launch_bounds__(THREA
int page_size, float scale, float* __restrict__ part_acc,
float* __restrict__ part_m, float* __restrict__ part_l,
int n_chunks, int cap = 0, long long scratch_stride = 0) {
- attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale, part_acc,
- part_m, part_l, n_chunks, cap, scratch_stride);
+ attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap,
+ scratch_stride);
+}
+
+// The same blocks for queries of different slots: query blockIdx.z reads its own slot's pools, everything else is
+// attn_chunk_kernel's batched form (qsa_decode_attn_slots).
+template <int KV_MODE, bool LANE_CELL>
+__global__ void __launch_bounds__(THREADS) attn_chunk_slots_kernel(const float* __restrict__ q, QsaAttnSlotPools ps,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale,
+ float* __restrict__ part_acc,
+ float* __restrict__ part_m,
+ float* __restrict__ part_l, int n_chunks, int cap,
+ long long scratch_stride) {
+ attn_chunk_body<KV_MODE, LANE_CELL, false, false>(q, ps.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);
}
+ // Batch windows: query blockIdx.z attends over its own slot's pools (rows.p[z]); the default kernel otherwise.
+ template <int KV_MODE>
+ __global__ void __launch_bounds__(THREADS) attn_chunk_rows_kernel(const float* __restrict__ q, QsaAttnPoolsRows rows,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale,
+ float* __restrict__ part_acc,
+ float* __restrict__ part_m,
+ float* __restrict__ part_l, int n_chunks, int cap,
+ long long scratch_stride) {
+ attn_chunk_body<KV_MODE, false, false, false>(q, rows.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);
+ }
+
-
// PR #540 (sskver): the same attention with fewer shuffles and less shared-memory traffic, bit for bit the kernel
// above (same operands, same order); +7% prompt speed on a V100, where this kernel reads every prompt chunk. It
// uses more registers (80 vs 38 on sm_70), so on CUDA it runs on cards below sm_75 only, and exists only in the
@@@ -908,64 -738,58 +924,116 @@@ void qsa_decode_attn_batch(const float
}
}
+bool qsa_decode_attn_slots(const float* q, const QsaAttnSlotPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
+ if (n_q <= 0) return true;
+ if (n_q > kQsaAttnMaxSlots) return false;
+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: unsupported geometry or missing buffers\n");
+ std::exit(1);
+ }
+ const int kv_mode = attn_kv_mode(pools.p[0]);
+ for (int64_t z = 0; z < n_q; ++z) {
+ if (!pools.p[z].page_table) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: missing page table\n");
+ std::exit(1);
+ }
+ if (attn_kv_mode(pools.p[z]) != kv_mode) return false;
+ }
+ const bool lane_cell = lane_cell_attn();
+ // only where qsa_decode_attn_batch runs attn_chunk_kernel; its other variants keep the per-query calls
+#if defined(STRATA_ATTN_PRE75_BUILT)
+ if (!lane_cell && pre75_attn()) return false;
+#endif
+#if defined(STRATA_HIP_GFX906)
+ static const bool query_swizzle = [] {
+ const char* v = std::getenv("STRATA_GFX906_ATTN_QUERY_SWIZZLE"); return v && v[0] == '1';
+ }();
+ if (query_swizzle && !lane_cell && kv_mode == 1 && query_swizzle_gfx906_device()) return false;
+#endif
+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
+ float* part_acc = scratch;
+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
+ float* part_l = part_m + (size_t) n_chunks * s.n_head;
+ const float scale = 1.0f / sqrtf((float) HD);
+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
+ cudaStream_t st = (cudaStream_t) stream;
+#define STRATA_ATTN_SLOTS(M, LC) attn_chunk_slots_kernel<M, LC><<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)
+ if (lane_cell) {
+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, true);
+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, true);
+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, true);
+ else STRATA_ATTN_SLOTS(0, true);
+ } else {
+ if (kv_mode == 3) STRATA_ATTN_SLOTS(3, false);
+ else if (kv_mode == 2) STRATA_ATTN_SLOTS(2, false);
+ else if (kv_mode == 1) STRATA_ATTN_SLOTS(1, false);
+ else STRATA_ATTN_SLOTS(0, false);
+ }
+#undef STRATA_ATTN_SLOTS
+ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,
+ (unsigned) n_q, st);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "qsa_decode_attn_slots: %s\n", cudaGetErrorString(e));
+ std::exit(1);
+ }
+ return true;
+}
+
+ void qsa_decode_attn_batch_rows(const float* q, const QsaAttnPoolsRows& rows, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
+ if (n_q <= 0) return;
+ const QsaAttnPools& pools = rows.p[0];
+ if (rows.n != n_q || n_q > kAttnRowsMax || s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 ||
+ !scratch || !ids || !steps) {
+ std::fprintf(stderr, "qsa_decode_attn_batch_rows: unsupported geometry or missing buffers\n");
+ std::exit(1);
+ }
+ const int kv_mode = pools.k_q4 != nullptr ? 2 : (pools.k_q != nullptr && pools.v_q4 != nullptr ? 3
+ : (pools.k_q != nullptr ? 1 : 0));
+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
+ float* part_acc = scratch;
+ float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
+ float* part_l = part_m + (size_t) n_chunks * s.n_head;
+ const float scale = 1.0f / sqrtf((float) HD);
+ const dim3 grid((unsigned) n_chunks, (unsigned) s.n_head_kv, (unsigned) n_q);
+ cudaStream_t st = (cudaStream_t) stream;
+ // The plain kernel is what qsa_decode_attn_batch runs unless one of its variants is on (lane-cell scoring, the
+ // pre-sm_75 / gfx103x kernel, the gfx906 swizzle): there each row goes through that call, one launch per row.
+ static const bool lane_cell_rows = [] { const char* v = std::getenv("STRATA_ATTN_LANECELL"); return v && v[0] == '1'; }();
+ bool per_row = lane_cell_rows;
+ #if defined(STRATA_ATTN_PRE75_BUILT)
+ per_row = per_row || pre75_attn();
+ #endif
+ #if defined(STRATA_HIP_GFX906)
+ per_row = true;
+ #endif
+ if (per_row) {
+ for (int64_t r = 0; r < n_q; ++r)
+ qsa_decode_attn_batch(q + (size_t) r * s.n_head * HD, rows.p[r], ids + (size_t) r * cap,
+ steps + (size_t) r * kStepCount, cap, s, scratch + (size_t) r * stride,
+ attn + (size_t) r * s.n_head * HD, 1, stream);
+ return;
+ }
+ #define STRATA_ATTN_ROWS(M) attn_chunk_rows_kernel<M><<<grid, THREADS, 0, st>>>(q, rows, ids, steps, (int) s.n_head_kv, \
+ (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride)
+ if (kv_mode == 3) STRATA_ATTN_ROWS(3);
+ else if (kv_mode == 2) STRATA_ATTN_ROWS(2);
+ else if (kv_mode == 1) STRATA_ATTN_ROWS(1);
+ else STRATA_ATTN_ROWS(0);
+ #undef STRATA_ATTN_ROWS
- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,
- attn, stride);
++ launch_merge(attn_merge_v2(), part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head,
++ (unsigned) n_q, st);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "qsa_decode_attn_batch_rows: %s\n", cudaGetErrorString(e));
+ std::exit(1);
+ }
+ }
+
uint64_t qsa_decode_attn_scratch_floats(int64_t cap, const QsaShapes& s) {
const int64_t chunks = (cap + CHUNK - 1) / CHUNK;
return (uint64_t) chunks * (uint64_t) s.n_head * (HD + 2) + 64;
diff --cc src/kernels/cuda/qsa_select.cu
index 39db4f65d,8d405680c..02e6c750a
--- a/src/kernels/cuda/qsa_select.cu
+++ b/src/kernels/cuda/qsa_select.cu
@@@ -831,11 -816,12 +831,12 @@@ __global__ void __launch_bounds__(TK_T
// qsa_block_scores takes it for every call without an active-block count and at most MQ queries: the captured decode
// window, the uncaptured decode, and prefill's pooled16 call.
constexpr int MQ = 8;
-__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
- const float* __restrict__ dead,
- const float* __restrict__ q_idx,
- const int32_t* __restrict__ steps, int nq,
- int64_t max_blocks, float* __restrict__ out,
- bool early_exit, QsaScoreRows srow = {}) {
+__device__ __forceinline__ void block_scores_multi_body(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq, int64_t max_blocks,
- float* __restrict__ out, bool early_exit) {
++ float* __restrict__ out, bool early_exit,
++ const QsaScoreRows& srow) {
__shared__ __align__(16) float qs[MQ * IDX_HEADS * IDX_DIM];
__shared__ int64_t s_nkv[MQ], s_nbid[MQ];
if (!early_exit)
@@@ -882,113 -872,6 +887,114 @@@
}
}
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out,
- bool early_exit) {
- block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit);
++ bool early_exit, QsaScoreRows srow = {}) {
++ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);
+}
+
+// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the
+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).
+struct ScoreSlotKeys {
+ const float* pooled[MQ];
+ const float* dead[MQ];
+};
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit) {
+ const unsigned z = blockIdx.y;
+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory
+ const float* pooled = keys.pooled[0];
+ const float* dead = keys.dead[0];
+#pragma unroll
+ for (unsigned i = 1; i < MQ; ++i)
+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }
+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,
- steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit);
++ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,
++ QsaScoreRows{});
+}
+
+// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue
+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a
+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory
+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two
+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices
+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products
+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered
+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the
+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the
+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.
+template <int NQ>
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out) {
+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];
+ if (threadIdx.x < nq) {
+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];
+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];
+ }
+ __syncthreads();
+ int64_t top = 0;
+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)
+ const int lane = threadIdx.x & 31;
+ float4 qr[NQ][IDX_HEADS];
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi)
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)
+ : make_float4(0.f, 0.f, 0.f, 0.f);
+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;
+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) {
+ if (qi >= nq) break;
+ const int64_t n_bid = s_nbid[qi];
+ if (b > n_bid) continue;
+ const float4 k4 = (b == n_bid) ? kd : kp;
+ float d[IDX_HEADS];
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float4 q4 = qr[qi][h];
+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ }
+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head
+ float r2[2];
+#pragma unroll
+ for (int i = 0; i < 2; ++i)
+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);
+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);
+ r += __shfl_xor_sync(0xffffffffu, r, 4);
+ r += __shfl_xor_sync(0xffffffffu, r, 2);
+ r += __shfl_xor_sync(0xffffffffu, r, 1);
+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel
+ const float s1 = __shfl_sync(0xffffffffu, r, 8);
+ const float s2 = __shfl_sync(0xffffffffu, r, 16);
+ const float s3 = __shfl_sync(0xffffffffu, r, 24);
+ if (lane == 0) {
+ float score = 0.0f;
+ score += r > 0.0f ? r : 0.0f;
+ score += s1 > 0.0f ? s1 : 0.0f;
+ score += s2 > 0.0f ? s2 : 0.0f;
+ score += s3 > 0.0f ? s3 : 0.0f;
+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;
+ out[qi * max_blocks + b] = score;
+ }
+ }
+ }
+}
+
#if !defined(__HIPCC__)
// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or
// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
@@@ -1269,21 -1122,20 +1275,35 @@@ void qsa_block_scores(const float* pool
if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores: %s\n", cudaGetErrorString(e)); std::exit(1); }
}
+bool qsa_block_scores_rows(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream) {
+ if (nq <= 0) return true;
+ if (nq > 4 || s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R) return false;
+ cudaStream_t st = (cudaStream_t) stream;
+ const int n = (int) nq;
+ if (n == 1) block_scores_rows_kernel<1><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);
+ else if (n == 2) block_scores_rows_kernel<2><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);
+ else if (n == 3) block_scores_rows_kernel<3><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);
+ else block_scores_rows_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, n, max_blocks, scores);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores rows: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return true;
+}
+
+ void qsa_block_scores_rows(const QsaScoreRows& rows, const float* q_idx, const int32_t* steps, int64_t nq,
+ int64_t max_blocks, const QsaShapes& s, float* scores, void* stream) {
+ if (nq <= 0) return;
+ if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R || nq > MQ || rows.n != nq) {
+ std::fprintf(stderr, "qsa_block_scores_rows: unsupported geometry\n");
+ std::exit(1);
+ }
+ static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();
+ block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(nullptr, nullptr, q_idx, steps, (int) nq,
+ max_blocks, scores, early, rows);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_rows: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ }
+
bool qsa_block_scores_tc(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
if (nq <= 0) return true;
diff --cc src/program/generate.cpp
index ed20bbd0d,f766e8da9..09bdef8e7
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -8275,7 -7860,11 +8297,12 @@@ int main(int argc, char** argv)
for (int st = 0; st < n_stages; ++st) {
stage_ver(st).set_stage(st == 0 ? 0 : split_at[(size_t) st - 1], st + 1 < n_stages ? split_at[(size_t) st] : -1,
st == 0 ? nullptr : hand[(size_t) st - 1], st + 1 < n_stages ? hand[(size_t) st] : nullptr);
+ if (hrows > strata::kernels::kVerifyMaxT) stage_ver(st).set_handoff_rows(hrows);
+ if (overlap)
+ stage_ver(st).set_handoff_flags(st == 0 ? nullptr : hflag_h[(size_t) st - 1],
+ st == 0 ? nullptr : hflag_d[(size_t) st - 1],
+ st + 1 < n_stages ? hflag_h[(size_t) st] : nullptr,
+ st + 1 < n_stages ? hflag_d[(size_t) st] : nullptr);
split_drive.end[st] = st + 1 < n_stages ? split_at[(size_t) st] : g.n_layers;
split_drive.cache_base[st] = drive.d.cache_base;
split_drive.cache_slot_off[st] = drive.d.cache_slot_off;
@@@ -10788,11 -10221,13 +10817,18 @@@
for (int t = 0; t < GS; ++t) if (bs[(size_t) (gi * GS + t)].active) return true;
return false;
};
+ // STRATA_TRACE: the drain before the line in hand (an admission's first phase), and when the first rows came
+ // after a batch admission (the pipeline's refill after BADM) or after a drain for another line (drain + refill)
+ double line_drain_ms = 0.0, adm_drain_ms = 0.0;
+ const char* rows_wait = nullptr; // what the next rows are timed from (rows_wait_t)
+ Clock::time_point rows_wait_t{};
+ // STRATA_PIPED_CONCURRENT (read_part): while a prompt is read on a thread of its own, the slots' doorbells are
+ // served on another (conc_serving); between windows that thread waits on bell_cv and a launch wakes it (a
+ // timed sleep is a whole timer tick on Windows, up to 15.6 ms, and the window's first layer would wait for it)
+ bool conc_serving = false;
+ std::mutex bell_mu;
+ std::condition_variable bell_cv;
+ std::atomic<bool> bell_waiting{false};
// one turn: serve every running stage, hand finished groups on, start what can start
auto pump = [&](bool may_start) -> bool {
++pipe_tick;