Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group sm75 (f-145-pp-sm75)

Niko1221df6262f270fe
diff --cc CMakeLists.txt
index 5b6854f70,ba4556c1a..d900326e3
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -432,35 -408,13 +432,41 @@@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_
target_link_libraries(qsa_topk_active_parity PRIVATE strata_kernels CUDA::cudart)
add_test(NAME qsa_topk_active_parity COMMAND qsa_topk_active_parity)
set_tests_properties(qsa_topk_active_parity PROPERTIES SKIP_RETURN_CODE 77)
+ # STRATA_QSA_SLOTS: a batch window's scores, top-k and attention as one launch over rows of different slots
+ # against one call per row, bitwise (synthetic, no model)
+ add_executable(qsa_slots_parity src/kernels/qsa_slots_parity.cpp)
+ target_include_directories(qsa_slots_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_slots_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_slots_parity COMMAND qsa_slots_parity)
+ add_test(NAME qsa_slots_parity_lanecell COMMAND qsa_slots_parity)
+ set_tests_properties(qsa_slots_parity qsa_slots_parity_lanecell PROPERTIES SKIP_RETURN_CODE 77)
+ set_tests_properties(qsa_slots_parity_lanecell PROPERTIES ENVIRONMENT "STRATA_ATTN_LANECELL=1")
+ # fork: the Daily's decode attention (STRATA_ATTN_DAILY=1: PR #540's chunk kernel on every card and the bounded
+ # merge) against upstream's kernels, bitwise; the production entries checked with the variable off and on
+ add_executable(qsa_merge_daily_parity src/kernels/qsa_merge_daily_parity.cpp)
+ target_include_directories(qsa_merge_daily_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_merge_daily_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_merge_daily_parity_default COMMAND qsa_merge_daily_parity)
+ add_test(NAME qsa_merge_daily_parity_daily COMMAND qsa_merge_daily_parity)
+ set_tests_properties(qsa_merge_daily_parity_default PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=0"
+ SKIP_RETURN_CODE 77)
+ set_tests_properties(qsa_merge_daily_parity_daily PROPERTIES ENVIRONMENT "STRATA_ATTN_DAILY=1"
+ SKIP_RETURN_CODE 77)
+ # the prompt path's block scores on the thread-per-head kernel (Turing) against the warp kernel, bitwise
+ add_executable(qsa_scores_parity src/kernels/qsa_scores_parity.cpp)
+ target_include_directories(qsa_scores_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_scores_parity PRIVATE strata_kernels CUDA::cudart)
+ add_test(NAME qsa_scores_parity COMMAND qsa_scores_parity)
+ set_tests_properties(qsa_scores_parity PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT "STRATA_SCORES_THREAD=1")
endif()
+ # QSAROWS (docs/QSAROWS.md): what the T rows of a verification window cost in a QSA layer's decode kernels at a long
+ # context (block scores, top-k, attention over each row's selection, indexer append), row by row; GPU, synthetic, no
+ # model. Not in the default build: `cmake --build <dir> --target qsa_rows_bench`.
+ if(STRATA_ENABLE_CUDA)
+ add_executable(qsa_rows_bench EXCLUDE_FROM_ALL src/kernels/qsa_rows_bench.cpp)
+ target_include_directories(qsa_rows_bench PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+ target_link_libraries(qsa_rows_bench PRIVATE strata_kernels CUDA::cudart)
+ endif()
# the QSA selection (block scores on the warp kernel and the tensor-core one, top-k): a GPU bench with a
# scorer-vs-scorer comparison and an FP64 accuracy gate (synthetic, no model); the ctest runs a 32K context under a
# 262144-cell capacity (the top-k dispatch follows the capacity) and exits non-zero on a failed gate
diff --cc docs/DETAILS.md
index 83c956e1c,9c1ad3a23..5c5724f12
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@@ -2217,26 -2046,21 +2217,45 @@@ signal, early cleanup and `--errors` er
mode is the negative control under memcheck: an eight-second watchdog releases
the original launch/service deadlock and the test fails.
+## CUDA opt-in switches measured for 0.1.45 (RTX 3060 and RTX A4000, sm_86; all off unless you set them)
+
+Ported from open pull requests and A/B'd as whole engine runs against the default (a fresh engine per side, IQ3_XXS, 5 pairs, medians;
+bit-exact means the 10-prompt + 4K/20K identity gate with the switch on equals the gate with it off). Decode after a 30K-token prompt
+unless noted; the gain is small on these two cards and larger on the Turing cards the authors measured.
+
+| Switch | What | Answers | RTX 3060 / A4000 |
+|---|---|---|---|
+| `STRATA_ATTN_DAILY=1` (#2075) | decode attention: two cells per step, a merge that skips empty chunks | bit-exact | +3.2% (5/5) / +0.7% |
+| `STRATA_QSA_ROWS=1` (#2059) | block scores of a verify window, queries in registers ([QSAROWS.md](QSAROWS.md)) | bit-exact | -0.3% / +0.3% |
+| `STRATA_EXPDEC=1` (#2058) | gate/up of the window's resident experts with the q8_1 rows in shared memory ([EXPERTDEC.md](EXPERTDEC.md)) | bit-exact | +0.5% / +1 to +5% (short context) |
+| `STRATA_SIDE_MIXER=1/2` (#1889) | the mixer's input-only work on graph branches | bit-exact | +0.5% / -0.6% (short context) |
+| `STRATA_B6_MMVQ_ROWS=1` (#1418) | exact multi-column MMVQ reusing the activation fragment across rows (K-quants, IQ4_XS) | bit-exact | +0.2..1% / +0.5..2% |
+| `STRATA_QSA_SPLIT_ATTN=1` (#1888) | decode attention split across blocks on the int8 mma path (sm_80+, int8 KV) | rounding-level (teacher-forced KL 0.0016 against a floor of 0.0012) | +2.8% / +3.5% |
+| `STRATA_HC_UPMIX=1` (#1370) | the hyper-connection up projection and `gr_mix_r` in one kernel (sm_80+) | bit-exact here (gate identical) | prompt +0.6% / +0.4% |
+| `STRATA_PROMPT_ATTN_IMMA=1` (#1367) | int8-KV prompt attention on INT8 tensor cores | rounding-level (KL at the floor) | prompt read +2.5% |
+| `STRATA_HC_REQ8=1` (#1262) | the hyper-connection projections as int8 + a scale per 32 (more expert-cache slots) | rounding-level (KL 0.0018) | decode -0.2% / +2.2% |
+| `STRATA_QUANT_GATHER=1` (#1368) | turns the once-per-token MoE input quantization back to the per-row gather | n/a | no difference measured |
+| `STRATA_PREFILL_GROUP_GATHER=0` (#1107) | turns the short-prompt grouped gather off | n/a | on is 0.6-1.3% faster |
+
+`STRATA_DEQUANT_WIDE` (the wide dequantization of Q4_K / Q5_K / Q6_K / IQ4_XS for the prompt's dense weights) is now on by default on
+sm_86 as well as sm_61: bit-exact, and the time to the first token of a 150 / 300 / 450-token prompt fell 11-15% on the RTX 3060 and 6-10% on the RTX A4000 (5 of 5 pairs each).
+`STRATA_DEQUANT_WIDE=0` turns it off.
++
+ ### RTX 20 (Turing, sm_75) kernel series (opt-in, 0.1.45)
+
+ Kernels by rafatxf (#2069-#2072, #2093, #2220-#2222), imanu86 (#2081) and xrip (#1821) for compute capability 7.5. They were measured
+ on one RTX 2080 Ti (decode +5..6 % at 4K-32K, prompts at depth up to +30 %, per the PRs) and are bitwise equal to the kernels they
+ replace, but nobody on the team has an sm_75 card to confirm them, so they are off until asked for:
+
+ `STRATA_SM75_PERF=1` turns on every bitwise Turing kernel below on a 7.5 card (no other card is ever affected). Each has its own
+ variable, which wins in both directions (`=0` off, `=1` on): `STRATA_SM75_IL` (interleaved small-batch matvec table),
+ `STRATA_SM75_EXP` (expert layout mode 4), `STRATA_GR_UP75` / `STRATA_GR_DOWN_TG` (hyper-connection read), `STRATA_SCORES_HALVE`
+ and `STRATA_TOPK_SPLIT` (decode QSA selection), `STRATA_TOPK_SAMPLE` (prompt top-k, one read of the scores), `STRATA_ATTN_V3`
+ (decode attention), `STRATA_Q2_WARP` (Q2_0 matvec), `STRATA_FUSE_WAIT=2` and `STRATA_QIL` (fewer launches per decode layer),
+ `STRATA_TC_SM75` (BF16 dequant, 16-byte stores), `STRATA_PA_V1_PREFETCH` (prompt attention v1 prefetch),
+ `STRATA_SCORES_THREAD` (#1825: the prompt path's thread-per-head block scorer, bitwise the warp kernel's scores; `=1` takes it on any
+ CUDA card, where it measured +1..3 % prompt speed on a Tesla P100 at 30-60K tokens, so it is not a default there).
+
+ Options that change answers (other summation order) are separate and explicit: `STRATA_SELECT_SIMT=1|2` (FP32 tiled block scorer,
+ KL 0.01 against the warp scorer per #2071), `STRATA_PA_V2T=1` (v2 prompt attention, #2092/#2146), `STRATA_PF_FUSED=1
+ STRATA_PF_FUSED_SM75=1` (fused int8 prompt experts for the Q2_0 pack, #2145; only on a 7.5 card, sm_80+ is unchanged), `STRATA_BF16_TC=1` with `STRATA_XN_F16` (#2091).
diff --cc src/core/verify.cpp
index 0847405e9,08d368e83..d760698ee
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -1256,30 -1198,7 +1306,30 @@@ bool Verifier::record_window(int T, cud
float* hb = h_L_ + (size_t) gi * MT * C;
float* gate = gate_L_ + (size_t) gi * MT * HV;
float* beta = beta_L_ + (size_t) gi * MT * HV;
+ if (sidem) { // the gates (a few blocks, latency-bound) beside the projections, joined below
+ if (!branch(mfork_, cs, side_, "fork")) return false;
+ gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
+ (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N,
+ (int) HV, n, side_);
+ if (cudaEventRecord(mjoin_, side_) != cudaSuccess) { err = "verify: the GDN gates' branch could not join"; return false; }
+ }
- if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs);
+ if (!q8_attn) quant_in(xm, (int) N);
+ if (side_proj) { // the gate projection (compute-bound for i-quants) beside qkv and the conv
+ // the branch reads F4's interleaved copy too (written on cs before the fork; bitwise native_mmvq's
+ // output), and qkv is recorded before the branch's launch, so its grid is dispatched first and the
+ // conv / recurrence that wait for it start sooner (z is read only after the recurrence)
+ const bool g_il = g_mmvq_il() && strata::kernels::native_mmvq_il_supported(wg->native_type, n, (int) ZV);
+ if (g_il && !il_ready) {
+ strata::kernels::native_q8_1_interleave(xq_, xil_, (int) N, n, cs);
+ il_ready = true;
+ }
+ if (!branch(pfork_, cs, side2_, "fork")) return false;
+ mm(wqkv, qkv + (size_t) tb * C, (int) N, (int) C);
+ if (g_il) strata::kernels::native_mmvq_il(wg->native_type, wg->native_data, xq_, xil_, z_ + (size_t) tb * ZV,
+ (int) N, (int) ZV, n, side2_);
+ else native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, side2_);
+ if (cudaEventRecord(pjoin_, side2_) != cudaSuccess) { err = "verify: the gate projection's branch could not join"; return false; }
+ }
if (br) { // STRATA_DF_BRANCH: a/b and z beside q/k/v and the conv; all of them read only this layer's input
fork(0);
gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
@@@ -1407,18 -1299,14 +1457,18 @@@
float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;
// the per-token GEMVs / norms / RoPEs / copies of this layer as one launch over the
// window's rows each - row-wise identical arithmetic (STRATA_DEC_BATCH=0: token by token)
- // STRATA_DF_QB_Q4 (opt-in: STRATA_DF_QB_Q4=1): a Q4_0 KV cache too - its one per-token extra, the query's rotation, is
- // row-wise and the batched path rotates (270650e); 0: Q4_0 token by token, as before
+ // STRATA_DF_QB_Q4 (default on since 0.1.45; STRATA_DF_QB_Q4=0 gives the old token-by-token path): a Q4_0 KV cache
+ // takes the batched window too - its one per-token extra, the query's rotation, is row-wise and the batched
+ // path rotates (270650e). The generated ids are identical (p620 gfx1201, A4000 sm_86, P100 sm_60, short and
+ // long prompts), and the 6-token verify graph is 1950 nodes instead of 2610 (#2403: an RX 6900 XT faulted in
+ // the first verify graph launch with --kv q4_0 + KV streaming + MTP; whether the graph size is the cause is
+ // not confirmed)
static const bool qb_q4 = [] {
const char* e = std::getenv("STRATA_DF_QB_Q4");
- return e != nullptr && std::atoi(e) != 0;
+ return e == nullptr || std::atoi(e) != 0;
}();
const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && (!st.kv_q4 || qb_q4);
- if (!q8_attn) native_quantize_q8_1(xm, xq_, (int) N, n, cs);
+ if (!q8_attn) quant_in(xm, (int) N);
// STRATA_DF_BRANCH: the query side (q and its norm/RoPE/rotation on side 0, the indexer query on side 1)
// beside the K/V side; the indexer query joins before the block scores, the query before attention
const bool qbr = br && qb;
@@@ -1965,13 -1708,8 +2014,13 @@@
copy_or_zero_from_mapped(parts_out, m_ymiss_ + (size_t) tb * K * N, (long long) n * K * N,
skip_ + grp, ring, cs);
} else {
- const bool fuse_rows = fuse_wait_on() && !prof_on_ && !remote_opt_ && dec_batch && m_bal_ == nullptr; // 143-amdk
- const bool fuse_rows = fuse_rows_on() && !prof_on_ && !remote_opt_ && dec_batch; // 143-amdk
- if (!fuse_rows) wait_flag_ge(m_flag_, ring, cs); // the CPU's share is in the mapped rows
++ const bool fuse_rows = fuse_rows_on() && !prof_on_ && !remote_opt_ && dec_batch && m_bal_ == nullptr; // 143-amdk
+ if (!fuse_rows) {
+ if (m_bal_ != nullptr) // set_pcie_balance: ... and ends as it waits for the CPU's rows
+ wait_flag_ge_stamped(m_flag_, ring, m_bal_ + ((l - lb_) * 2 + grp) * 4 + 2, nullptr, cs);
+ else
+ wait_flag_ge(m_flag_, ring, cs); // the CPU's share is in the mapped rows
+ }
stamp(l, 23, grp);
if (remote_opt_) // #578: the helper's rows come back reduced; skip them as well
remote_opt_->copy_rows(parts_out, m_ymiss_ + (size_t) tb * K * N, tb, n, p_dst, p_counts + 1, cs);
diff --cc src/kernels/cuda/fused_gr.cu
index 38b7535b6,ce2a395e2..7b7274866
--- a/src/kernels/cuda/fused_gr.cu
+++ b/src/kernels/cuda/fused_gr.cu
@@@ -1085,9 -1129,52 +1185,53 @@@ bool qfuse_hc_local()
static const bool on = [] { const char* e = std::getenv("STRATA_QFUSE_HC_LOCAL"); return e == nullptr || std::atoi(e) != 0; }();
return on;
}
+ // Turing: gr_up75_kernel for 3+ tokens (bitwise gr_up_multi_kernel; fused_gr_bench cold on an RTX 2080 Ti, up incl. a
+ // stamp: T 3 27.3 -> 24.6 us, T 4 33.2 -> 28.7, T 6 46.6 -> 36.6); STRATA_GR_UP75=0 / 1 forces it off / on
+ bool up75_on() {
+ #if defined(__HIPCC__)
+ return false;
+ #else
+ static const int env = [] { const char* v = std::getenv("STRATA_GR_UP75"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();
+ if (env >= 0) return env != 0;
+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) return false;
+ if (!per_dev[dev]) {
+ int major = 0, minor = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+ per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1
+ }
+ return per_dev[dev] == 2;
+ #endif
+ }
+ // sm_75: the staged down kernel's tokens per block (gridDim.y = ct / tg), 0 = one block row for all of them.
+ // STRATA_GR_DOWN_TG=n forces n where it divides the launch (0 = off).
+ int down_tg(int ct) {
+ #if defined(__HIPCC__)
+ (void) ct;
+ return 0;
+ #else
+ static const int env = [] { const char* v = std::getenv("STRATA_GR_DOWN_TG"); return v ? std::atoi(v) : -1; }();
+ if (ct < 2) return 0;
+ if (env >= 0) return env > 0 && env <= 3 && ct % env == 0 ? env : 0;
+ // fused_gr_bench cold on an RTX 2080 Ti (down incl. a stamp): T 4 29.1 -> 27.3 us, T 6 38.5 -> 32.2 with pairs;
+ // one token per block is slower at every T (the weights are read from L2 once per group)
+ static int per_dev[64]; // 0 unknown, 1 no, 2 sm_75
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) return 0;
+ if (!per_dev[dev]) {
+ int major = 0, minor = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+ per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1
+ }
+ return per_dev[dev] == 2 && (ct == 4 || ct == 6) ? 2 : 0;
+ #endif
+ }
void launch_multi(const GrMulti& m, int variant, cudaStream_t st, unsigned long long* stamp_buf, int stamp_i0) {
const int n_tok = m.T;
+ const bool q8w = m.a[0].s_down != nullptr; // STRATA_HC_REQ8: w_down / w_up hold int8 codes
#if STRATA_GR_FAST_BUILD
// gfx906: the latency-hidden norm/up (STRATA_GR_FAST=0: off) - the same sums in the same order as the kernels
// they replace (gr_parity checks the multi-token read against single-token calls bitwise)
@@@ -1173,16 -1261,16 +1325,26 @@@
return;
}
#endif
- if (n_tok >= 3 && up75_on()) { // T 1-2: gr_up_multi_kernel is as fast or faster (fused_gr_bench, cold)
+ if (q8w) { // STRATA_HC_REQ8: the up kernel on int8 rows
+ switch (no_multi_gr ? kFusedGrMaxT : n_tok) {
+ case 1: gr_up_multi_kernel<1, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 2: gr_up_multi_kernel<2, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 3: gr_up_multi_kernel<3, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 4: gr_up_multi_kernel<4, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 5: gr_up_multi_kernel<5, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 6: gr_up_multi_kernel<6, true, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ default: gr_up_multi_kernel<kFusedGrMaxT, false, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ }
++ } else if (n_tok >= 3 && up75_on()) { // T 1-2: gr_up_multi_kernel is as fast or faster (fused_gr_bench, cold)
+ switch (no_multi_gr ? kFusedGrMaxT : n_tok) {
+ case 1: gr_up75_kernel<1, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 2: gr_up75_kernel<2, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 3: gr_up75_kernel<3, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 4: gr_up75_kernel<4, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 5: gr_up75_kernel<5, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ case 6: gr_up75_kernel<6, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ default: gr_up75_kernel<kFusedGrMaxT, false><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
+ }
} else
switch (no_multi_gr ? kFusedGrMaxT : n_tok) {
case 1: gr_up_multi_kernel<1, true><<<UPM_BLOCKS, THREADS, 0, st>>>(m); break;
diff --cc src/kernels/cuda/qsa_decode_attn.cu
index 062ef664e,71b5bc358..854b479ee
--- a/src/kernels/cuda/qsa_decode_attn.cu
+++ b/src/kernels/cuda/qsa_decode_attn.cu
@@@ -284,47 -282,131 +285,172 @@@ __device__ __forceinline__ void attn_ch
for (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];
}
+template <int KV_MODE, bool LANE_CELL = false, bool QUERY_SWIZZLE = false, bool REDUCE12 = false>
+__global__ void __launch_bounds__(THREADS) attn_chunk_kernel(const float* __restrict__ q, QsaAttnPools p,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale, float* __restrict__ part_acc,
+ float* __restrict__ part_m, float* __restrict__ part_l,
+ int n_chunks, int cap = 0, long long scratch_stride = 0) {
+ attn_chunk_body<KV_MODE, LANE_CELL, QUERY_SWIZZLE, REDUCE12>(q, p, ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap,
+ scratch_stride);
+}
+
+// The same blocks for queries of different slots: query blockIdx.z reads its own slot's pools, everything else is
+// attn_chunk_kernel's batched form (qsa_decode_attn_slots).
+template <int KV_MODE, bool LANE_CELL>
+__global__ void __launch_bounds__(THREADS) attn_chunk_slots_kernel(const float* __restrict__ q, QsaAttnSlotPools ps,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale,
+ float* __restrict__ part_acc,
+ float* __restrict__ part_m,
+ float* __restrict__ part_l, int n_chunks, int cap,
+ long long scratch_stride) {
+ attn_chunk_body<KV_MODE, LANE_CELL, false, false>(q, ps.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);
+}
+
+// Batch windows: query blockIdx.z attends over its own slot's pools (rows.p[z]); the default kernel otherwise.
+template <int KV_MODE>
+__global__ void __launch_bounds__(THREADS) attn_chunk_rows_kernel(const float* __restrict__ q, QsaAttnPoolsRows rows,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale,
+ float* __restrict__ part_acc,
+ float* __restrict__ part_m,
+ float* __restrict__ part_l, int n_chunks, int cap,
+ long long scratch_stride) {
+ attn_chunk_body<KV_MODE, false, false, false>(q, rows.p[blockIdx.z], ids, step, n_kv_heads, page_size, scale,
+ part_acc, part_m, part_l, n_chunks, cap, scratch_stride);
+}
+
+ #if !defined(__HIPCC__)
+ // Turing (INT8 pools): attn_chunk_kernel<1> bit for bit, rearranged for the SM's shared-memory pipe, which the
+ // default kernel saturates (its time grew with the queries even with the SMs half empty). Three changes, none to an
+ // operand or the order of an operation:
+ // - the lane's slice of the 12 query heads (8 dimensions each) is read from shared memory once, into registers,
+ // instead of two 16-byte reads per head per cell (with a two-way bank conflict at lane * 8);
+ // - the 12 heads' lane sums are reduced with reduce12_gfx906 (16 shuffles, each sum warp_sum's pairs) instead of
+ // 12 warp_sums (60 shuffles);
+ // - the probabilities are stored cell-major, so the value loop reads a cell's 12 as three 16-byte broadcasts.
+ // RTX 2080 Ti, 2051 selected cells: 1 / 2 / 3 / 4 queries 51.6 / 78.5 / 110 / 146 -> 34.5 / 41.0 / 73.0 / 84.9 us
+ // (with the merge); 600 cells 45.9 / 74.6 / 105 / 137 -> 28.6 / 32.8 / 33.5 / 33.3 us. STRATA_ATTN_V3=0 keeps the
+ // default kernel; =1 takes this one on any CUDA card.
+ __global__ void __launch_bounds__(THREADS) attn_chunk_q8_v3_kernel(const float* __restrict__ q, QsaAttnPools p,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ step, int n_kv_heads,
+ int page_size, float scale, float* __restrict__ part_acc,
+ float* __restrict__ part_m, float* __restrict__ part_l,
+ int n_chunks, int cap = 0, long long scratch_stride = 0) {
+ q += (size_t) blockIdx.z * (size_t) (n_kv_heads * G) * HD;
+ ids += (size_t) blockIdx.z * (size_t) cap;
+ step += (size_t) blockIdx.z * kStepCount;
+ part_acc += (size_t) blockIdx.z * (size_t) scratch_stride;
+ part_m += (size_t) blockIdx.z * (size_t) scratch_stride;
+ part_l += (size_t) blockIdx.z * (size_t) scratch_stride;
+ __shared__ __align__(16) float sq[G][HD];
+ __shared__ __align__(16) float spt[CHUNK][G]; // cell-major: one cell's 12 heads are 3 float4
+ __shared__ long long srow[CHUNK];
+ const int n_ids = __ldg(step + kStepWidth);
+ const int chunk = blockIdx.x, kvh = blockIdx.y;
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ const int c0 = chunk * CHUNK;
+ const int n_here = min(CHUNK, n_ids - c0);
+ const int slot = kvh * n_chunks + chunk;
+ if (n_here <= 0) {
+ if (t < G) { part_m[slot * G + t] = -FLT_MAX; part_l[slot * G + t] = 0.0f; }
+ return;
+ }
+ // stored with the gfx906 swizzle (the two float4 halves of lanes with bit 2 swapped): fewer conflicts on the one read
+ for (int i = t; i < G * HD; i += THREADS) {
+ const int d = i % HD;
+ sq[i / HD][d ^ ((d >> 3) & 4)] = q[(size_t) (kvh * G) * HD + i];
+ }
+ if (t < CHUNK) {
+ long long r = -1;
+ if (t < n_here) {
+ const int cell = ids[c0 + t];
+ const long long page = (long long) p.page_table[cell / page_size];
+ if (page >= 0) r = (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+ }
+ srow[t] = r;
+ }
+ __syncthreads();
+ float qr[G][8];
+ #pragma unroll
+ for (int h = 0; h < G; ++h) {
+ const int qflip = lane & 4;
+ const float4 qa = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + qflip]);
+ const float4 qb = *reinterpret_cast<const float4*>(&sq[h][lane * 8 + (qflip ^ 4)]);
+ qr[h][0] = qa.x; qr[h][1] = qa.y; qr[h][2] = qa.z; qr[h][3] = qa.w;
+ qr[h][4] = qb.x; qr[h][5] = qb.y; qr[h][6] = qb.z; qr[h][7] = qb.w;
+ }
+ for (int c = warp; c < CHUNK; c += WARPS) {
+ if (c >= n_here || srow[c] < 0) {
+ if (lane < G) spt[c][lane] = -FLT_MAX;
+ continue;
+ }
+ float k8[8];
+ load8<1>(p, false, srow[c], lane * 8, k8);
+ float part[16];
+ #pragma unroll
+ for (int h = 0; h < G; ++h)
+ part[h] = k8[0] * qr[h][0] + k8[1] * qr[h][1] + k8[2] * qr[h][2] + k8[3] * qr[h][3] +
+ k8[4] * qr[h][4] + k8[5] * qr[h][5] + k8[6] * qr[h][6] + k8[7] * qr[h][7];
+ #pragma unroll
+ for (int h = G; h < 16; ++h) part[h] = 0.0f;
+ const float score = reduce12_gfx906(part, lane);
+ const int head = reduce12_head_of_lane(lane);
+ if ((lane & 1) == 0 && head < G) spt[c][head] = score * scale;
+ }
+ __syncthreads();
+ for (int h = warp; h < G; h += WARPS) {
+ const float a = spt[lane][h], b = spt[lane + 32][h];
+ const float m = warp_max(fmaxf(a, b));
+ const float ea = (lane < n_here && srow[lane] >= 0) ? __expf(a - m) : 0.0f;
+ const float eb = (lane + 32 < n_here && srow[lane + 32] >= 0) ? __expf(b - m) : 0.0f;
+ spt[lane][h] = ea;
+ spt[lane + 32][h] = eb;
+ const float l = warp_sum(ea + eb);
+ if (lane == 0) { part_m[slot * G + h] = m; part_l[slot * G + h] = l; }
+ }
+ __syncthreads();
+ float acc[G];
+ #pragma unroll
+ for (int h = 0; h < G; ++h) acc[h] = 0.0f;
+ for (int c = 0; c < n_here; ++c) {
+ if (srow[c] < 0) continue; // masked above, weight 0
+ const float sc = __half2float(__ushort_as_half(p.v_scale[srow[c] * (HD / KV_Q8_GROUP) + t / KV_Q8_GROUP]));
+ const float v = (float) p.v_q[srow[c] * HD + t] * sc;
+ const float4 p0 = *reinterpret_cast<const float4*>(&spt[c][0]);
+ const float4 p1 = *reinterpret_cast<const float4*>(&spt[c][4]);
+ const float4 p2 = *reinterpret_cast<const float4*>(&spt[c][8]);
+ const float pr[G] = {p0.x, p0.y, p0.z, p0.w, p1.x, p1.y, p1.z, p1.w, p2.x, p2.y, p2.z, p2.w};
+ #pragma unroll
+ for (int h = 0; h < G; ++h) acc[h] = fmaf(pr[h], v, acc[h]);
+ }
+ #pragma unroll
+ for (int h = 0; h < G; ++h) part_acc[((size_t) slot * G + h) * HD + t] = acc[h];
+ }
+ bool attn_v3_on() {
+ static int on[16] = {}; // by device ordinal: 0 unknown, 1 yes, -1 no
+ static const int env = [] { const char* v = std::getenv("STRATA_ATTN_V3"); return v ? (v[0] == '0' ? 0 : 1) : -1; }();
+ if (env >= 0) return env == 1;
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 16) return false;
+ if (on[dev] == 0) {
+ int maj = 0, mnr = 0;
+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&
+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;
+ if (!ok) cudaGetLastError();
+ on[dev] = ok && strata::sm75_default(maj, mnr) ? 1 : -1; // Turing only, with STRATA_SM75_PERF=1
+ }
+ return on[dev] > 0;
+ }
+ #endif
+
// PR #540 (sskver): the same attention with fewer shuffles and less shared-memory traffic, bit for bit the kernel
// above (same operands, same order); +7% prompt speed on a V100, where this kernel reads every prompt chunk. It
// uses more registers (80 vs 38 on sm_70), so on CUDA it runs on cards below sm_75 only, and exists only in the
@@@ -874,49 -814,27 +1000,55 @@@ void launch_chunks_batch(const float* q
(int) s.n_head_kv, (int) s.page_size, scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
}
} else
+ #endif
+ #if !defined(__HIPCC__)
- if (kv_mode == 1 && attn_v3_on())
++ if (kernel < 0 && kv_mode == 1 && attn_v3_on()) // (the parity harness names its kernel: never the environment)
+ attn_chunk_q8_v3_kernel<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
+ scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+ else
#endif
if (kv_mode == 3)
- STRATA_ATTN_CHUNK(3)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+ STRATA_ATTN_CHUNK_SEL(3, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,
+ (int) s.page_size, scale, part_acc, part_m, part_l,
+ n_chunks, (int) cap, stride);
else if (kv_mode == 2)
- STRATA_ATTN_CHUNK(2)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+ STRATA_ATTN_CHUNK_SEL(2, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,
+ (int) s.page_size, scale, part_acc, part_m, part_l,
+ n_chunks, (int) cap, stride);
else if (kv_mode == 1)
- STRATA_ATTN_CHUNK(1)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
+ STRATA_ATTN_CHUNK_SEL(1, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,
+ (int) s.page_size, scale, part_acc, part_m, part_l,
+ n_chunks, (int) cap, stride);
else
- STRATA_ATTN_CHUNK(0)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv, (int) s.page_size,
- scale, part_acc, part_m, part_l, n_chunks, (int) cap, stride);
- attn_merge_kernel<<<dim3((unsigned) s.n_head, (unsigned) n_q), HD, 0, st>>>(part_acc, part_m, part_l, n_chunks,
- attn, stride);
+ STRATA_ATTN_CHUNK_SEL(0, pre)<<<grid, THREADS, 0, st>>>(q, pools, ids, steps, (int) s.n_head_kv,
+ (int) s.page_size, scale, part_acc, part_m, part_l,
+ n_chunks, (int) cap, stride);
+}
+
+// qsa_decode_attn_batch's merge pass over the partials launch_chunks_batch wrote; `v2` names the merge kernel
+void launch_merge_batch(const int32_t* steps, int64_t cap, const QsaShapes& s, const float* scratch, float* attn,
+ int64_t n_q, int v2, cudaStream_t st) {
+ const int n_chunks = (int) ((cap + CHUNK - 1) / CHUNK);
+ const long long stride = (long long) qsa_decode_attn_scratch_floats(cap, s);
+ const float* part_acc = scratch;
+ const float* part_m = scratch + (size_t) n_chunks * s.n_head * HD;
+ const float* part_l = part_m + (size_t) n_chunks * s.n_head;
+ launch_merge(v2, part_acc, part_m, part_l, n_chunks, steps, attn, stride, (unsigned) s.n_head, (unsigned) n_q, st);
+}
+
+} // namespace
+
+void qsa_decode_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+ int64_t cap, const QsaShapes& s, float* scratch, float* attn, int64_t n_q, void* stream) {
+ if (n_q <= 0) return;
+ if (s.head_dim != HD || s.n_head != (int64_t) G * s.n_head_kv || cap <= 0 || !scratch || !ids || !steps ||
+ !pools.page_table || n_q > 65535) {
+ std::fprintf(stderr, "qsa_decode_attn_batch: unsupported geometry or missing buffers\n");
+ std::exit(1);
+ }
+ cudaStream_t st = (cudaStream_t) stream;
+ launch_chunks_batch(q, pools, ids, steps, cap, s, scratch, n_q, st, -1);
+ launch_merge_batch(steps, cap, s, scratch, attn, n_q, attn_merge_v2(), st);
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) {
std::fprintf(stderr, "qsa_decode_attn_batch: %s\n", cudaGetErrorString(e));
diff --cc src/kernels/cuda/qsa_prompt_attn.cu
index 06ed48c90,54f40e9aa..b2b5480ce
--- a/src/kernels/cuda/qsa_prompt_attn.cu
+++ b/src/kernels/cuda/qsa_prompt_attn.cu
@@@ -679,250 -793,7 +794,251 @@@ __global__ void __launch_bounds__(THREA
}
}
+ #undef PA2_PART
+__global__ void __launch_bounds__(THREADS) decode_attn_i8_split_kernel(const float* __restrict__ q, QsaAttnPools p,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ steps, int n_kv_heads,
+ int page_size, float scale_log2,
+ float* __restrict__ part, int cap) {
+ extern __shared__ __align__(16) unsigned char smem_raw[];
+ Smem2& S = *reinterpret_cast<Smem2*>(smem_raw);
+ const int qi = blockIdx.x, kvh = blockIdx.y, sp = blockIdx.z, nsp = gridDim.z;
+ const int n_head = n_kv_heads * G;
+ q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+ // partials of (query, split): acc [n_head][HD] unnormalized, then m / l [n_head][2]
+ float* pacc = part + (size_t) (qi * nsp + sp) * n_head * (HD + 2) + (size_t) kvh * G * HD;
+ float* pml = part + (size_t) (qi * nsp + sp) * n_head * (HD + 2) + (size_t) n_head * HD + (size_t) kvh * G * 2;
+ ids += (size_t) qi * cap;
+ const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);
+ // this split's chunks: [ci0, ci1) of the query's ceil(n / CH2)
+ const int n_all = (n + CH2 - 1) / CH2, per = (n_all + nsp - 1) / nsp;
+ const int ci0 = sp * per, ci1 = min(n_all, ci0 + per);
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ const int gid = lane >> 2, tig = lane & 3;
+ const int dim0 = warp * 64;
+
+ // q: the power-of-two prescale over all 12 heads (as v1), then this warp's 64 dims as hi/lo A fragments
+ float qm = 0.0f;
+ for (int i = t; i < G * HD; i += THREADS) qm = fmaxf(qm, fabsf(q[i]));
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) qm = fmaxf(qm, __shfl_xor_sync(0xffffffffu, qm, o));
+ if (lane == 0) S.qmax[warp] = qm;
+ if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; }
+ __syncthreads();
+ qm = fmaxf(fmaxf(S.qmax[0], S.qmax[1]), fmaxf(S.qmax[2], S.qmax[3]));
+ int qe = 0;
+ if (qm > 0.0f) frexpf(qm, &qe);
+ const float qup = ldexpf(1.0f, 14 - qe), qdown = ldexpf(scale_log2, qe - 14);
+ uint32_t qh[4][4], ql[4][4];
+#pragma unroll
+ for (int kk = 0; kk < 4; ++kk) {
+#pragma unroll
+ for (int r = 0; r < 4; ++r) {
+ const int row = gid + (r & 1) * 8, col = dim0 + kk * 16 + 2 * tig + (r >> 1) * 8;
+ float2 x = make_float2(0.f, 0.f);
+ if (row < G) x = *reinterpret_cast<const float2*>(q + (size_t) row * HD + col);
+ x.x *= qup;
+ x.y *= qup;
+ const __half2 hi = __floats2half2_rn(x.x, x.y);
+ const float2 hf = __half22float2(hi);
+ const __half2 lo = __floats2half2_rn(x.x - hf.x, x.y - hf.y);
+ qh[kk][r] = *reinterpret_cast<const uint32_t*>(&hi);
+ ql[kk][r] = *reinterpret_cast<const uint32_t*>(&lo);
+ }
+ }
+
+ if (ci0 >= ci1) { // an empty split: m = -inf, l = 0, acc = 0
+ for (int i = t; i < G * HD; i += THREADS) pacc[(size_t) (i / HD) * HD + i % HD] = 0.0f;
+ if (t < G) { pml[2 * t] = -INFINITY; pml[2 * t + 1] = 0.0f; }
+ return;
+ }
+ // the chunk pipeline: cells two chunks ahead, their pool rows one chunk ahead, the data (cp.async) one ahead
+ auto cell_of = [&](int c) -> int { return c < n ? __ldg(ids + c) : -1; };
+ auto row_of = [&](int cell) -> long long {
+ if (cell < 0) return -1;
+ const long long page = (long long) __ldg(p.page_table + cell / page_size);
+ return (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+ };
+ auto issue = [&](long long r, int st, float& ksr, float& vsr) {
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int idx = lane + 32 * j, cell = idx >> 2, pc = idx & 3;
+ const long long rr = __shfl_sync(0xffffffffu, r, cell);
+ const bool ok = rr >= 0;
+ const size_t off = ok ? (size_t) rr * HD + dim0 + pc * 16 : 0;
+ cp_async16(&S.kv[st][warp][0][0][0] + swz(cell, pc * 16), p.k_q + off, ok);
+ cp_async16(&S.kv[st][warp][1][0][0] + swz(cell, pc * 16), p.v_q + off, ok);
+ }
+ ksr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.k_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;
+ vsr = r >= 0 ? __half2float(__ushort_as_half(__ldg(p.v_scale + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;
+ };
+ float ksn, vsn;
+ issue(row_of(cell_of(ci0 * CH2 + lane)), 0, ksn, vsn);
+ cp_async_commit();
+ S.sc[0][warp][0][lane] = ksn;
+ S.sc[0][warp][1][lane] = vsn;
+ long long r_next = row_of(cell_of((ci0 + 1) * CH2 + lane));
+ int cell_next2 = cell_of((ci0 + 2) * CH2 + lane);
+
+ float acc[8][4];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;
+
+ for (int ci = ci0; ci < ci1; ++ci) {
+ const int st = (ci - ci0) & 1, c0 = ci * CH2;
+ const bool more = ci + 1 < ci1;
+ if (more) issue(r_next, st ^ 1, ksn, vsn);
+ cp_async_commit();
+ r_next = row_of(cell_next2);
+ cell_next2 = cell_of((ci + 3) * CH2 + lane);
+ cp_async_wait1();
+ __syncwarp();
+ const int8_t* K = &S.kv[st][warp][0][0][0];
+ const int8_t* V = &S.kv[st][warp][1][0][0];
+ // q.k over this warp's 64 dims, times the cell's K scale for this group
+#pragma unroll
+ for (int nt = 0; nt < CH2 / 8; ++nt) {
+ float tg[4] = {0.f, 0.f, 0.f, 0.f};
+ const int cell = nt * 8 + gid;
+#pragma unroll
+ for (int kk = 0; kk < 4; ++kk) {
+ uint32_t b[2];
+ b[0] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig)));
+ b[1] = i8x2_to_h2(*reinterpret_cast<const uint16_t*>(K + swz(cell, kk * 16 + 2 * tig + 8)));
+ mma16816(tg, qh[kk], b);
+ mma16816(tg, ql[kk], b);
+ }
+ const int c = nt * 8 + 2 * tig;
+ const float s0 = S.sc[st][warp][0][c], s1 = S.sc[st][warp][0][c + 1];
+ S.part[warp][gid][c] = tg[0] * s0;
+ S.part[warp][gid][c + 1] = tg[1] * s1;
+ S.part[warp][gid + 8][c] = tg[2] * s0;
+ S.part[warp][gid + 8][c + 1] = tg[3] * s1;
+ }
+ __syncthreads();
+ // online softmax over the four groups' sum (fixed order): row t/8, 4 cells per thread
+ {
+ const int r = t >> 3, sub = t & 7;
+ float x[4], mx = -INFINITY;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int c = sub * 4 + j;
+ x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * qdown
+ : -INFINITY;
+ mx = fmaxf(mx, x[j]);
+ }
+#pragma unroll
+ for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));
+ const float m_old = S.mrow[r];
+ const float m_new = fmaxf(m_old, mx);
+ float sum = 0.0f;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
+ S.p[r][sub * 4 + j] = e;
+ sum += e;
+ }
+#pragma unroll
+ for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+ __syncwarp();
+ if (sub == 0) {
+ const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
+ S.alpha[r] = a;
+ S.lsum[r] = fmaf(S.lsum[r], a, sum);
+ S.mrow[r] = m_new;
+ }
+ }
+ __syncthreads();
+ // p.v over this warp's 64 dims (as v1)
+ {
+ float vmax = S.sc[st][warp][1][lane];
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) vmax = fmaxf(vmax, __shfl_xor_sync(0xffffffffu, vmax, o));
+ const float vup = vmax > 0.0f ? 16384.0f / vmax : 0.0f, vdown = vmax * (1.0f / 16384.0f);
+ float tmp[8][4];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) tmp[j][0] = tmp[j][1] = tmp[j][2] = tmp[j][3] = 0.0f;
+#pragma unroll
+ for (int ks = 0; ks < CH2 / 16; ++ks) {
+ const int cA = ks * 16 + 2 * tig, cB = cA + 8;
+ const float w0 = S.sc[st][warp][1][cA] * vup, w1 = S.sc[st][warp][1][cA + 1] * vup,
+ w2 = S.sc[st][warp][1][cB] * vup, w3 = S.sc[st][warp][1][cB + 1] * vup;
+ const float p00 = S.p[gid][cA] * w0, p01 = S.p[gid][cA + 1] * w1;
+ const float p10 = S.p[gid + 8][cA] * w0, p11 = S.p[gid + 8][cA + 1] * w1;
+ const float p02 = S.p[gid][cB] * w2, p03 = S.p[gid][cB + 1] * w3;
+ const float p12 = S.p[gid + 8][cB] * w2, p13 = S.p[gid + 8][cB + 1] * w3;
+ uint32_t ah[4], al[4];
+ ah[0] = pack_h2(p00, p01);
+ ah[1] = pack_h2(p10, p11);
+ ah[2] = pack_h2(p02, p03);
+ ah[3] = pack_h2(p12, p13);
+ {
+ const __half2* h = reinterpret_cast<const __half2*>(ah);
+ float2 f;
+ f = __half22float2(h[0]); al[0] = pack_h2(p00 - f.x, p01 - f.y);
+ f = __half22float2(h[1]); al[1] = pack_h2(p10 - f.x, p11 - f.y);
+ f = __half22float2(h[2]); al[2] = pack_h2(p02 - f.x, p03 - f.y);
+ f = __half22float2(h[3]); al[3] = pack_h2(p12 - f.x, p13 - f.y);
+ }
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const int d = j * 8 + gid;
+ const uint32_t x0 = (uint8_t) V[swz(cA, d)] | ((uint32_t) (uint8_t) V[swz(cA + 1, d)] << 8);
+ const uint32_t x1 = (uint8_t) V[swz(cB, d)] | ((uint32_t) (uint8_t) V[swz(cB + 1, d)] << 8);
+ uint32_t b[2];
+ b[0] = i8x2_to_h2(x0);
+ b[1] = i8x2_to_h2(x1);
+ mma16816(tmp[j], ah, b);
+ mma16816(tmp[j], al, b);
+ }
+ }
+ const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ acc[j][0] = fmaf(acc[j][0], a0, tmp[j][0] * vdown);
+ acc[j][1] = fmaf(acc[j][1], a0, tmp[j][1] * vdown);
+ acc[j][2] = fmaf(acc[j][2], a1, tmp[j][2] * vdown);
+ acc[j][3] = fmaf(acc[j][3], a1, tmp[j][3] * vdown);
+ }
+ }
+ if (more) {
+ S.sc[st ^ 1][warp][0][lane] = ksn;
+ S.sc[st ^ 1][warp][1][lane] = vsn;
+ }
+ __syncwarp();
+ }
+ __syncthreads();
+ // the split's unnormalized sums and each head row's maximum and sum (the merge rescales and divides)
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const int d = dim0 + j * 8 + 2 * tig;
+ *reinterpret_cast<float2*>(pacc + (size_t) gid * HD + d) = make_float2(acc[j][0], acc[j][1]);
+ if (gid + 8 < G)
+ *reinterpret_cast<float2*>(pacc + (size_t) (gid + 8) * HD + d) = make_float2(acc[j][2], acc[j][3]);
+ }
+ if (t < G) { pml[2 * t] = S.mrow[t]; pml[2 * t + 1] = S.lsum[t]; }
+}
+
+// the splits of each (query, head) merged with the log-sum-exp rescale (base 2, as the kernel's online softmax), in
+// split order; a block per (query, head), a thread per dimension
+__global__ void __launch_bounds__(HD) decode_attn_merge_kernel(const float* __restrict__ part, int n_head, int nsp,
+ float* __restrict__ attn) {
+ const int qi = blockIdx.x, h = blockIdx.y, d = threadIdx.x;
+ const size_t qs = (size_t) qi * nsp * n_head * (HD + 2);
+ float M = -INFINITY;
+ for (int s = 0; s < nsp; ++s) M = fmaxf(M, part[qs + (size_t) s * n_head * (HD + 2) + (size_t) n_head * HD + 2 * h]);
+ float L = 0.0f, o = 0.0f;
+ for (int s = 0; s < nsp; ++s) {
+ const float* ps = part + qs + (size_t) s * n_head * (HD + 2);
+ const float m = ps[(size_t) n_head * HD + 2 * h], l = ps[(size_t) n_head * HD + 2 * h + 1];
+ const float w = m == -INFINITY ? 0.0f : exp2f(m - M);
+ L = fmaf(l, w, L);
+ o = fmaf(ps[(size_t) h * HD + d], w, o);
+ }
+ attn[((size_t) qi * n_head + h) * HD + d] = L > 0.0f ? o * (1.0f / L) : 0.0f;
+}
+
+
bool launch_i8(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
@@@ -953,356 -830,7 +1075,356 @@@
}
return true;
}
+
+// ---- v3 (int8 KV, opt-in STRATA_PROMPT_ATTN_IMMA=1): v2's block, pipeline and online softmax, with both products on
+// INT8 tensor cores (mma m16n8k32 s8, int32 accumulation). K and V enter as the int8 codes they are - no int8 -> fp16
+// conversion - and the other operand as THREE 8-bit parts: q per (head, 64-dim group) and p * v_scale per (head, chunk)
+// scaled to 127, then X = rint(x * 65536) split exactly into h (signed), l (signed) and m (unsigned) bytes - 24 bits,
+// as fine as FP32 itself and finer than the FP16 hi + lo split v2 uses (~22 bits). Each part's int32 sum is exact; the three meet in
+// FP32 as h + (l * 256 + m) / 65536 (m unsigned: a u8 x s8 MMA). INT8 MMA runs ~3.1x FP16's rate on sm_120 (732 vs 233 measured), so three INT8
+// MMAs cost about one FP16 MMA where v2 spends two. p.v takes each thread's 8 consecutive dims across the 8 n-tiles
+// (n-tile j, column n = dim n * 8 + j), so a thread reads 8 bytes of a V row per cell and byte-transposes 4 cells.
+__device__ __forceinline__ void imma16832(int* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_PA_SM80
+ __trap();
+#else
+ asm volatile("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+ "{%0,%1,%2,%3};\n"
+ : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3])
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+// the low part's MMA: unsigned A (m in 0..255), signed B
+__device__ __forceinline__ void imma16832_us(int* c, const uint32_t* a, const uint32_t* b) {
+#if !STRATA_PA_SM80
+ __trap();
+#else
+ asm volatile("mma.sync.aligned.m16n8k32.row.col.s32.u8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+ "{%0,%1,%2,%3};\n"
+ : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3])
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
+#endif
+}
+// x (|x| <= 127) as X = rint(x * 65536) = h * 65536 + l * 256 + m exactly: h, l signed bytes, m an unsigned one
+__device__ __forceinline__ void split3(float x, int& h, int& l, int& m) {
+ const int X = __float2int_rn(x * 65536.0f);
+ h = (X + 32768) >> 16;
+ const int r = X - (h << 16); // -32768 .. 32767
+ l = r >> 8; // -128 .. 127
+ m = r & 255; // 0 .. 255
+}
+__device__ __forceinline__ uint32_t pack4(int a, int b, int c, int d) {
+ return (uint32_t) (a & 0xff) | ((uint32_t) (b & 0xff) << 8) | ((uint32_t) (c & 0xff) << 16) | ((uint32_t) (d & 0xff) << 24);
+}
+__device__ __forceinline__ float join3(int h, int l, int m) { // the parts' exact int32 sums -> h + (l*256 + m)/65536
+ return (float) h + (float) ((l << 8) + m) * (1.0f / 65536.0f);
+}
+// four words of four bytes each (word i = cell i) -> four words of the same byte position across the cells
+__device__ __forceinline__ void tr4x4(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t w3, uint32_t* o) {
+ const uint32_t t0 = __byte_perm(w0, w1, 0x5140), t1 = __byte_perm(w0, w1, 0x7362);
+ const uint32_t t2 = __byte_perm(w2, w3, 0x5140), t3 = __byte_perm(w2, w3, 0x7362);
+ o[0] = __byte_perm(t0, t2, 0x5410);
+ o[1] = __byte_perm(t0, t2, 0x7632);
+ o[2] = __byte_perm(t1, t3, 0x5410);
+ o[3] = __byte_perm(t1, t3, 0x7632);
+}
+
+// v3's shared memory: ONE stage of K and one of V per warp (a warp gathers and reads only its own 64-dim slices, so
+// reusing a buffer needs only that warp's own order): K(c + 1) is gathered while the block runs softmax(c) and
+// p.v(c), V(c + 1) while it runs q.k(c + 1) and softmax(c + 1); and only the 12 real head rows of the partial sums
+// and p. 25.5 KB instead of v2's 45.6 KB: three blocks an SM instead of two (one block an SM ran 1.54x slower than
+// two - this kernel waits on latency, not on its MMAs).
+struct Smem3 {
+ int8_t kv[4][2][CH2][64]; // warp, K/V, cell, 64 dims in 16-byte pieces XOR-swizzled by the cell
+ float sc[4][2][CH2]; // warp, K/V scale of the cell for the warp's group
+ float part[4][G][CH2 + 1]; // q.k per dim group (the 12 head rows)
+ float p[G][CH2 + 1];
+ float alpha[16];
+ float lsum[16];
+ float mrow[16];
+};
+
+__global__ void __launch_bounds__(THREADS, 3) prompt_attn_i8v3_kernel(const float* __restrict__ q, QsaAttnPools p,
+ const int32_t* __restrict__ ids,
+ const int32_t* __restrict__ steps, int n_kv_heads,
+ int page_size, float scale_log2,
+ float* __restrict__ attn, int cap) {
+ extern __shared__ __align__(16) unsigned char smem_raw[];
+ Smem3& S = *reinterpret_cast<Smem3*>(smem_raw);
+ const int qi = blockIdx.x, kvh = blockIdx.y;
+ const int n_head = n_kv_heads * G;
+ q += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+ attn += (size_t) qi * n_head * HD + (size_t) kvh * G * HD;
+ ids += (size_t) qi * cap;
+ const int n = __ldg(steps + (size_t) qi * kStepCount + kStepWidth);
+ const int t = threadIdx.x, lane = t & 31, warp = t >> 5;
+ const int gid = lane >> 2, tig = lane & 3;
+ const int dim0 = warp * 64;
+ if (t < 16) { S.mrow[t] = -INFINITY; S.lsum[t] = 0.0f; S.alpha[t] = 0.0f; }
+
+ // q: rows gid / gid + 8 (heads; 12..15 are pad), this warp's 64 dims as the A fragments of two k-steps of 32,
+ // three 8-bit parts with a scale per (row, 64-dim group)
+ uint32_t qa[3][2][4];
+ float qinv[2];
+ {
+ float x[2][2][2][4]; // [row half][k-step][k half][4 dims]
+ float mx[2] = {0.0f, 0.0f};
+#pragma unroll
+ for (int rh = 0; rh < 2; ++rh) {
+ const int row = gid + rh * 8;
+#pragma unroll
+ for (int ks = 0; ks < 2; ++ks)
+#pragma unroll
+ for (int kh = 0; kh < 2; ++kh) {
+ float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
+ if (row < G) v = *reinterpret_cast<const float4*>(q + (size_t) row * HD + dim0 + ks * 32 + kh * 16 + tig * 4);
+ x[rh][ks][kh][0] = v.x; x[rh][ks][kh][1] = v.y; x[rh][ks][kh][2] = v.z; x[rh][ks][kh][3] = v.w;
+ mx[rh] = fmaxf(mx[rh], fmaxf(fmaxf(fabsf(v.x), fabsf(v.y)), fmaxf(fabsf(v.z), fabsf(v.w))));
+ }
+#pragma unroll
+ for (int o = 1; o < 4; o <<= 1) mx[rh] = fmaxf(mx[rh], __shfl_xor_sync(0xffffffffu, mx[rh], o));
+ }
+ float up[2];
+#pragma unroll
+ for (int rh = 0; rh < 2; ++rh) {
+ up[rh] = mx[rh] > 0.0f ? 127.0f / mx[rh] : 0.0f;
+ qinv[rh] = mx[rh] * (1.0f / 127.0f);
+ }
+#pragma unroll
+ for (int ks = 0; ks < 2; ++ks)
+#pragma unroll
+ for (int r = 0; r < 4; ++r) { // a0: row gid, k 0..15 half; a1: row gid + 8; a2: row gid, k 16..31; a3
+ const int rh = r & 1, kh = r >> 1;
+ int hh[4], ll[4], mm[4];
+#pragma unroll
+ for (int e = 0; e < 4; ++e) split3(x[rh][ks][kh][e] * up[rh], hh[e], ll[e], mm[e]);
+ qa[0][ks][r] = pack4(hh[0], hh[1], hh[2], hh[3]);
+ qa[1][ks][r] = pack4(ll[0], ll[1], ll[2], ll[3]);
+ qa[2][ks][r] = pack4(mm[0], mm[1], mm[2], mm[3]);
+ }
+ }
+
+ const int n_chunks = (n + CH2 - 1) / CH2;
+ auto cell_of = [&](int c) -> int { return c < n ? __ldg(ids + c) : -1; };
+ auto row_of = [&](int cell) -> long long {
+ if (cell < 0) return -1;
+ const long long page = (long long) __ldg(p.page_table + cell / page_size);
+ return (page * n_kv_heads + kvh) * page_size + (cell % page_size);
+ };
+ // this warp's 64-dim slice of K (kv = 0) or V (1) of the chunk whose rows the lanes hold, and the cells' scales
+ auto issue = [&](long long r, int kv) {
+ const int8_t* pool = kv == 0 ? p.k_q : p.v_q;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int idx = lane + 32 * j, cell = idx >> 2, pc = idx & 3;
+ const long long rr = __shfl_sync(0xffffffffu, r, cell);
+ const bool ok = rr >= 0;
+ const size_t off = ok ? (size_t) rr * HD + dim0 + pc * 16 : 0;
+ cp_async16(&S.kv[warp][kv][0][0] + swz(cell, pc * 16), pool + off, ok);
+ }
+ const uint16_t* sc = kv == 0 ? p.k_scale : p.v_scale;
+ S.sc[warp][kv][lane] = r >= 0 ? __half2float(__ushort_as_half(__ldg(sc + r * (HD / KV_Q8_GROUP) + warp))) : 0.0f;
+ };
+ long long r_nxt = row_of(cell_of(lane)); // chunk 0
+ issue(r_nxt, 0);
+ cp_async_commit();
+ issue(r_nxt, 1);
+ cp_async_commit();
+ r_nxt = row_of(cell_of(CH2 + lane)); // chunk 1
+ int cell_nn = cell_of(2 * CH2 + lane); // chunk 2
+ __syncthreads();
+
+ float acc[8][4];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.0f;
+ const int8_t* K = &S.kv[warp][0][0][0];
+ const int8_t* V = &S.kv[warp][1][0][0];
+
+ for (int ci = 0; ci < n_chunks; ++ci) {
+ const int c0 = ci * CH2;
+ const bool more = ci + 1 < n_chunks;
+ cp_async_wait1(); // K(ci) is in (pending: V(ci))
+ __syncwarp();
+ // q.k over this warp's 64 dims: n-tile nt holds cells nt * 8 .. + 7 (column n = cell nt * 8 + n)
+#pragma unroll
+ for (int nt = 0; nt < CH2 / 8; ++nt) {
+ int ch[4] = {0, 0, 0, 0}, cl[4] = {0, 0, 0, 0}, cm[4] = {0, 0, 0, 0};
+ const int cell = nt * 8 + gid;
+#pragma unroll
+ for (int ks = 0; ks < 2; ++ks) {
+ uint32_t b[2];
+ b[0] = *reinterpret_cast<const uint32_t*>(K + swz(cell, ks * 32 + tig * 4));
+ b[1] = *reinterpret_cast<const uint32_t*>(K + swz(cell, ks * 32 + 16 + tig * 4));
+ imma16832(ch, qa[0][ks], b);
+ imma16832(cl, qa[1][ks], b);
+ imma16832_us(cm, qa[2][ks], b);
+ }
+ const int c = nt * 8 + 2 * tig;
+ const float s0 = S.sc[warp][0][c], s1 = S.sc[warp][0][c + 1];
+ S.part[warp][gid][c] = join3(ch[0], cl[0], cm[0]) * qinv[0] * s0;
+ S.part[warp][gid][c + 1] = join3(ch[1], cl[1], cm[1]) * qinv[0] * s1;
+ if (gid + 8 < G) {
+ S.part[warp][gid + 8][c] = join3(ch[2], cl[2], cm[2]) * qinv[1] * s0;
+ S.part[warp][gid + 8][c + 1] = join3(ch[3], cl[3], cm[3]) * qinv[1] * s1;
+ }
+ }
+ __syncwarp(); // every lane has read K(ci) and its scales
+ if (more) issue(r_nxt, 0);
+ cp_async_commit();
+ __syncthreads();
+ // online softmax over the four groups' sum (fixed order): rows 0..11, 8 threads a row (warp 3 idles)
+ if (t < G * 8) {
+ const int r = t >> 3, sub = t & 7;
+ float x[4], mx = -INFINITY;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const int c = sub * 4 + j;
+ x[j] = c0 + c < n ? (((S.part[0][r][c] + S.part[1][r][c]) + S.part[2][r][c]) + S.part[3][r][c]) * scale_log2
+ : -INFINITY;
+ mx = fmaxf(mx, x[j]);
+ }
+#pragma unroll
+ for (int o = 1; o < 8; o <<= 1) mx = fmaxf(mx, __shfl_xor_sync(0xffffffffu, mx, o));
+ const float m_old = S.mrow[r];
+ const float m_new = fmaxf(m_old, mx);
+ float sum = 0.0f;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ const float e = x[j] == -INFINITY ? 0.0f : exp2f(x[j] - m_new);
+ S.p[r][sub * 4 + j] = e;
+ sum += e;
+ }
+#pragma unroll
+ for (int o = 1; o < 8; o <<= 1) sum += __shfl_xor_sync(0xffffffffu, sum, o);
+ __syncwarp();
+ if (sub == 0) {
+ const float a = m_old == -INFINITY ? 0.0f : exp2f(m_old - m_new);
+ S.alpha[r] = a;
+ S.lsum[r] = fmaf(S.lsum[r], a, sum);
+ S.mrow[r] = m_new;
+ }
+ }
+ __syncthreads();
+ cp_async_wait1(); // V(ci) is in (pending: K(ci + 1))
+ __syncwarp();
+ // p.v over this warp's 64 dims: A = p * v_scale (rows gid / gid + 8, k = the chunk's 32 cells) in three parts
+ // scaled per row; B = V (k = cells, n-tile j column n = dim n * 8 + j)
+ {
+ uint32_t pa[3][4];
+ float pinv[2];
+ { // a scale per row: a chunk whose p are all small still gets its 24 bits (one per warp and chunk lost
+ // ~3x of the accuracy against FP64)
+ float a[4][4], mx[2] = {0.0f, 0.0f};
+#pragma unroll
+ for (int r = 0; r < 4; ++r) { // a0: row gid, cells tig * 4 ..; a1: row gid + 8; a2 / a3: cells 16 + ...
+ const int row = gid + (r & 1) * 8, cb = (r >> 1) * 16 + tig * 4;
+#pragma unroll
+ for (int e = 0; e < 4; ++e) {
+ a[r][e] = row < G ? S.p[row][cb + e] * S.sc[warp][1][cb + e] : 0.0f;
+ mx[r & 1] = fmaxf(mx[r & 1], a[r][e]);
+ }
+ }
+ float up[2];
+#pragma unroll
+ for (int rh = 0; rh < 2; ++rh) {
+#pragma unroll
+ for (int o = 1; o < 4; o <<= 1) mx[rh] = fmaxf(mx[rh], __shfl_xor_sync(0xffffffffu, mx[rh], o));
+ up[rh] = mx[rh] > 0.0f ? 127.0f / mx[rh] : 0.0f;
+ pinv[rh] = mx[rh] * (1.0f / 127.0f);
+ }
+#pragma unroll
+ for (int r = 0; r < 4; ++r) {
+ int hh[4], ll[4], mm[4];
+#pragma unroll
+ for (int e = 0; e < 4; ++e) split3(a[r][e] * up[r & 1], hh[e], ll[e], mm[e]);
+ pa[0][r] = pack4(hh[0], hh[1], hh[2], hh[3]);
+ pa[1][r] = pack4(ll[0], ll[1], ll[2], ll[3]);
+ pa[2][r] = pack4(mm[0], mm[1], mm[2], mm[3]);
+ }
+ }
+ // B: this thread's 8 dims (gid * 8 .. + 7) of cells tig * 4 .. + 3 (b0) and 16 + tig * 4 .. + 3 (b1)
+ uint32_t bv[2][8]; // [k half][dim j]
+#pragma unroll
+ for (int kh = 0; kh < 2; ++kh) {
+ uint32_t lo[4], hi[4];
+#pragma unroll
+ for (int e = 0; e < 4; ++e) {
+ const uint2 w = *reinterpret_cast<const uint2*>(V + swz(kh * 16 + tig * 4 + e, gid * 8));
+ lo[e] = w.x;
+ hi[e] = w.y;
+ }
+ tr4x4(lo[0], lo[1], lo[2], lo[3], &bv[kh][0]);
+ tr4x4(hi[0], hi[1], hi[2], hi[3], &bv[kh][4]);
+ }
+ __syncwarp(); // every lane has read V(ci) and its scales
+ if (more) issue(r_nxt, 1);
+ cp_async_commit();
+ const float a0 = S.alpha[gid], a1 = S.alpha[gid + 8];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ int ch[4] = {0, 0, 0, 0}, cl[4] = {0, 0, 0, 0}, cm[4] = {0, 0, 0, 0};
+ const uint32_t b[2] = {bv[0][j], bv[1][j]};
+ imma16832(ch, pa[0], b);
+ imma16832(cl, pa[1], b);
+ imma16832_us(cm, pa[2], b);
+ acc[j][0] = fmaf(acc[j][0], a0, join3(ch[0], cl[0], cm[0]) * pinv[0]);
+ acc[j][1] = fmaf(acc[j][1], a0, join3(ch[1], cl[1], cm[1]) * pinv[0]);
+ acc[j][2] = fmaf(acc[j][2], a1, join3(ch[2], cl[2], cm[2]) * pinv[1]);
+ acc[j][3] = fmaf(acc[j][3], a1, join3(ch[3], cl[3], cm[3]) * pinv[1]);
+ }
+ }
+ r_nxt = row_of(cell_nn);
+ cell_nn = cell_of((ci + 3) * CH2 + lane);
+ }
+ __syncthreads();
+ const float l0 = S.lsum[gid], l1 = gid + 8 < G ? S.lsum[gid + 8] : 0.0f;
+ const float i0 = l0 > 0.0f ? 1.0f / l0 : 0.0f, i1 = l1 > 0.0f ? 1.0f / l1 : 0.0f;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) { // column 2 * tig is dim 16 * tig + j, column 2 * tig + 1 dim 16 * tig + 8 + j
+ const int d0 = dim0 + 16 * tig + j, d1 = d0 + 8;
+ attn[(size_t) gid * HD + d0] = acc[j][0] * i0;
+ attn[(size_t) gid * HD + d1] = acc[j][1] * i0;
+ if (gid + 8 < G) {
+ attn[(size_t) (gid + 8) * HD + d0] = acc[j][2] * i1;
+ attn[(size_t) (gid + 8) * HD + d1] = acc[j][3] * i1;
+ }
+ }
+}
+
+bool launch_i8v3(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
+ const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
+ static bool attr[64] = {};
+ int dev = 0;
+ cudaGetDevice(&dev);
+ const int bytes = (int) sizeof(Smem3);
+ if (dev < 0 || dev >= 64) return false;
+ if (!attr[dev]) {
+ if (cudaFuncSetAttribute(prompt_attn_i8v3_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes) !=
+ cudaSuccess) {
+ cudaGetLastError();
+ return false;
+ }
+ // the shared-memory side of the L1 split as large as it goes, so three blocks fit an SM
+ cudaFuncSetAttribute(prompt_attn_i8v3_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100);
+ cudaGetLastError();
+ attr[dev] = true;
+ }
+ const float scale_log2 = 1.4426950408889634f / sqrtf((float) HD);
+ for (int64_t q0 = 0; q0 < n_q; q0 += 65535) {
+ const int64_t nb = n_q - q0 < 65535 ? n_q - q0 : 65535;
+ prompt_attn_i8v3_kernel<<<dim3((unsigned) nb, (unsigned) s.n_head_kv), THREADS, bytes, st>>>(
+ q + q0 * s.n_head * HD, pools, ids + q0 * cap, steps + q0 * kStepCount, (int) s.n_head_kv,
+ (int) s.page_size, scale_log2, attn + q0 * s.n_head * HD, (int) cap);
+ }
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "qsa_prompt_attn_batch (v3): %s\n", cudaGetErrorString(e));
+ std::exit(1);
+ }
+ return true;
+}
+
- template <int KV_MODE>
+ template <int KV_MODE, bool PF = false>
bool launch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps, int64_t cap,
const QsaShapes& s, float* attn, int64_t n_q, cudaStream_t st) {
static bool attr[64] = {}; // per device, as above
@@@ -2193,13 -1644,16 +2321,22 @@@ bool qsa_prompt_attn_batch(const float
// exist before sm_80.
static const bool v1 = std::getenv("STRATA_PROMPT_ATTN_V1") != nullptr;
if (volta) return launch70<1>(q, pools, ids, steps, cap, s, attn, n_q, st);
- if (v1 || turing) return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);
+ // Turing: v2 with register staging instead of cp.async, opt-in (STRATA_PA_V2T=1) until measured
+ static const bool v2t = [] { const char* e = std::getenv("STRATA_PA_V2T"); return e != nullptr && e[0] == '1'; }();
+ if (v1 || (turing && !v2t)) {
+ // #1821 (xrip): v1 with the next chunk's rows prefetched in registers and K/V sharing one buffer (two blocks an
+ // SM), the same bits. STRATA_PA_V1_PREFETCH=0 / 1 forces it off / on; unset: a Turing card with
+ // STRATA_SM75_PERF=1.
+ static const int pf_env = [] { const char* e = std::getenv("STRATA_PA_V1_PREFETCH"); return e ? (std::atoi(e) != 0 ? 1 : 0) : -1; }();
+ if (pf_env == 1 || (pf_env < 0 && t75 && strata::sm75_series())) return launch<1, true>(q, pools, ids, steps, cap, s, attn, n_q, st);
+ return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);
+ }
+ // STRATA_PROMPT_ATTN_IMMA=1 (opt-in, sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel)
+ if (g_pa_imma < 0) {
+ const char* v = std::getenv("STRATA_PROMPT_ATTN_IMMA");
+ g_pa_imma = v != nullptr && v[0] == '1' ? 1 : 0;
+ }
- if (g_pa_imma == 1) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);
++ if (g_pa_imma == 1 && !turing) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);
return launch_i8(q, pools, ids, steps, cap, s, attn, n_q, st);
}
if (!pools.k_pool || !pools.v_pool) return false;
diff --cc src/kernels/cuda/qsa_select.cu
index 203c15809,2651982d4..e956dc8b9
--- a/src/kernels/cuda/qsa_select.cu
+++ b/src/kernels/cuda/qsa_select.cu
@@@ -579,23 -556,102 +580,119 @@@ __global__ void __launch_bounds__(256)
}
}
+// The FP32 tiled scores and the tail block; false when the grid does not fit. `rdna2_tiles`: gfx103x's tiling.
+static bool block_scores_simt(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps,
+ int64_t nq, int64_t max_blocks, int64_t reach, float* scores, cudaStream_t stream,
+ bool rdna2_tiles) {
+ const int64_t qt = 32, nb = rdna2_tiles ? 128 : 64;
+ const dim3 grid((unsigned) ((reach + nb - 1) / nb), (unsigned) ((nq + qt - 1) / qt));
+ if (grid.y > 65535) return false;
+ if (rdna2_tiles)
+ block_scores_simt_kernel<32, 128, 16, 8><<<grid, 256, 0, stream>>>(pooled, q_idx, steps, nq, max_blocks, reach, scores);
+ else
+ block_scores_simt_kernel<32, 64, 32, 4><<<grid, 256, 0, stream>>>(pooled, q_idx, steps, nq, max_blocks, reach, scores);
+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, stream>>>(dead, q_idx, steps, max_blocks, scores);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return true;
+}
+
+ // block_scores_simt_kernel with larger tiles (QT queries x NB blocks, RI x CJ products per thread) and the next K
+ // chunk's loads in registers while the current one is used. Every (row, block) dot is the same fmaf chain over
+ // k = 0..127 from 0, and the relu sum over the heads in the same order: the same bits as block_scores_simt_kernel.
+ // STRATA_SELECT_SIMT=2.
+ constexpr int S2_KC = 32;
+ template <int QT, int NB>
+ __global__ void __launch_bounds__(256, 1) block_scores_simt2_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int64_t nq,
+ int64_t max_blocks, int64_t reach,
+ float* __restrict__ out) {
+ constexpr int ROWS = QT * IDX_HEADS, RI = ROWS / 16, CJ = NB / 16, QS = ROWS + 4, KS = NB + 4;
+ constexpr int QL = ROWS * S2_KC / 4 / 256, KL = NB * S2_KC / 4 / 256; // float4 loads per thread
+ static_assert(RI % 4 == 0 && CJ % 4 == 0 && QL >= 1 && KL >= 1, "tile");
+ __shared__ __align__(16) float Qs[S2_KC][QS];
+ __shared__ __align__(16) float Ks[S2_KC][KS];
+ const int64_t q0 = (int64_t) blockIdx.y * QT, b0 = (int64_t) blockIdx.x * NB;
+ const int64_t qlast = (q0 + QT < nq ? q0 + QT : nq) - 1;
+ if (b0 >= steps[qlast * kStepCount + kStepNBid]) return; // n_bid rises with the position
+ const int tid = threadIdx.x, tx = tid & 15, ty = tid >> 4;
+ float acc[RI][CJ];
+ #pragma unroll
+ for (int i = 0; i < RI; ++i)
+ #pragma unroll
+ for (int j = 0; j < CJ; ++j) acc[i][j] = 0.0f;
+ float4 qv[QL], kv[KL];
+ auto load = [&](int kc) {
+ #pragma unroll
+ for (int i = 0; i < QL; ++i) {
+ const int idx = tid + i * 256, r = idx >> 3, k4 = idx & 7;
+ qv[i] = make_float4(0.f, 0.f, 0.f, 0.f);
+ if (q0 + (r >> 2) < nq)
+ qv[i] = *reinterpret_cast<const float4*>(q_idx + (q0 * IDX_HEADS + r) * IDX_DIM + kc + k4 * 4);
+ }
+ #pragma unroll
+ for (int i = 0; i < KL; ++i) {
+ const int idx = tid + i * 256, c = idx >> 3, k4 = idx & 7;
+ kv[i] = make_float4(0.f, 0.f, 0.f, 0.f);
+ if (b0 + c < reach) kv[i] = *reinterpret_cast<const float4*>(pooled + (b0 + c) * IDX_DIM + kc + k4 * 4);
+ }
+ };
+ load(0);
+ #pragma unroll 1
+ for (int kc = 0; kc < IDX_DIM; kc += S2_KC) {
+ #pragma unroll
+ for (int i = 0; i < QL; ++i) {
+ const int idx = tid + i * 256, r = idx >> 3, k4 = idx & 7;
+ Qs[k4 * 4 + 0][r] = qv[i].x; Qs[k4 * 4 + 1][r] = qv[i].y; Qs[k4 * 4 + 2][r] = qv[i].z; Qs[k4 * 4 + 3][r] = qv[i].w;
+ }
+ #pragma unroll
+ for (int i = 0; i < KL; ++i) {
+ const int idx = tid + i * 256, c = idx >> 3, k4 = idx & 7;
+ Ks[k4 * 4 + 0][c] = kv[i].x; Ks[k4 * 4 + 1][c] = kv[i].y; Ks[k4 * 4 + 2][c] = kv[i].z; Ks[k4 * 4 + 3][c] = kv[i].w;
+ }
+ __syncthreads();
+ if (kc + S2_KC < IDX_DIM) load(kc + S2_KC); // in flight during this chunk's products
+ #pragma unroll 4
+ for (int k = 0; k < S2_KC; ++k) {
+ float qr[RI], kr[CJ];
+ #pragma unroll
+ for (int m = 0; m < RI / 4; ++m) {
+ const float4 a = *reinterpret_cast<const float4*>(&Qs[k][ty * RI + 4 * m]);
+ qr[4 * m] = a.x; qr[4 * m + 1] = a.y; qr[4 * m + 2] = a.z; qr[4 * m + 3] = a.w;
+ }
+ #pragma unroll
+ for (int g = 0; g < CJ / 4; ++g) {
+ const float4 b = *reinterpret_cast<const float4*>(&Ks[k][g * 64 + tx * 4]);
+ kr[4 * g] = b.x; kr[4 * g + 1] = b.y; kr[4 * g + 2] = b.z; kr[4 * g + 3] = b.w;
+ }
+ #pragma unroll
+ for (int i = 0; i < RI; ++i)
+ #pragma unroll
+ for (int j = 0; j < CJ; ++j) acc[i][j] = fmaf(qr[i], kr[j], acc[i][j]);
+ }
+ __syncthreads();
+ }
+ #pragma unroll
+ for (int qi = 0; qi < RI / 4; ++qi) {
+ const int64_t q = q0 + ty * (RI / 4) + qi;
+ if (q >= nq) continue;
+ const int64_t n_bid = steps[q * kStepCount + kStepNBid];
+ #pragma unroll
+ for (int j = 0; j < CJ; ++j) {
+ const int64_t b = b0 + (j >> 2) * 64 + tx * 4 + (j & 3);
+ if (b >= n_bid || b >= max_blocks) continue;
+ float sc = 0.0f;
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float d = acc[qi * 4 + h][j];
+ sc += d > 0.0f ? d : 0.0f;
+ }
+ out[q * max_blocks + b] = sc;
+ }
+ }
+ }
+
// ---- the same top-k with each query's keys read once: 1,024 threads hold up to TK_PER consecutive blocks' keys in
// registers (contexts up to 4 * 1024 * TK_PER cells), per-warp histograms, block-wide scans. The selection rule is
// block_topk_kernel's (radix threshold, ties to the lowest index, cells ascending): identical ids.
@@@ -752,8 -794,9 +849,11 @@@ __global__ void __launch_bounds__(TK_T
// neighbours at a time. The cells are emitted ascending in the order (warp, row, lane): warp w holds the `per` rows
// of 32 consecutive blocks from block w * 32 * per. block_topk_kernel's selection rule (radix threshold, ties to the
// lowest index): identical ids.
+// campaign3: also compiled on HIP (gfx11: plain shared/atomic code, no sm_90 features; the dispatch still needs
+// reach > fit, i.e. the deep contexts where the ref radix was the only option on AMD).
+ constexpr int TS_MAXQ = 4096;
+ __device__ int g_tk_fail[TS_MAXQ]; // block_topk_sample_kernel: 1 = the query is block_topk_wide_kernel<true>'s
+ template <bool ONLY_FAILED = false>
__global__ void __launch_bounds__(TK_T) block_topk_wide_kernel(const float* __restrict__ scores,
const int32_t* __restrict__ steps, int64_t max_blocks,
int64_t cap, int32_t* __restrict__ ids) {
@@@ -928,214 -969,91 +1029,299 @@@ __device__ __forceinline__ void block_s
}
}
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_multi_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit, QsaScoreRows srow = {}) {
+ block_scores_multi_body(pooled, dead, q_idx, steps, nq, max_blocks, out, early_exit, srow);
+}
+
+// Queries of different slots, each with its own indexer keys: blockIdx.y is the query, and its CTAs run exactly the
+// single-query call of block_scores_multi_kernel (qsa_block_scores_slots).
+struct ScoreSlotKeys {
+ const float* pooled[MQ];
+ const float* dead[MQ];
+};
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_slots_kernel(ScoreSlotKeys keys,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps,
+ int64_t max_blocks, float* __restrict__ out,
+ bool early_exit) {
+ const unsigned z = blockIdx.y;
+ // the slot's pointers picked by unrolled selects: indexing the argument array by z would copy it to local memory
+ const float* pooled = keys.pooled[0];
+ const float* dead = keys.dead[0];
+#pragma unroll
+ for (unsigned i = 1; i < MQ; ++i)
+ if (z == i) { pooled = keys.pooled[i]; dead = keys.dead[i]; }
+ block_scores_multi_body(pooled, dead, q_idx + (size_t) z * IDX_HEADS * IDX_DIM,
+ steps + (size_t) z * kStepCount, 1, max_blocks, out + (size_t) z * max_blocks, early_exit,
+ QsaScoreRows{});
+}
+
+// QSAROWS (docs/QSAROWS.md, STRATA_QSA_ROWS=1): block_scores_multi_kernel's work for NQ <= 4 queries with less issue
+// pressure per (block, query). nsys of a 131k decode window (RTX 2080 Ti + RTX 3060): the multi kernel costs ~20 us more a
+// call for every row of the window on both cards - per (block, query) it reads the four query slices from shared memory
+// (4 x LDS.128) and reduces each head's lane products with its own five shuffles (20 a block and query), and those two
+// pipes bound it, not the key reads (the keys are read once for all the queries, as there). Here (1) the lane's slices
+// of the NQ queries (4 heads x 4 floats each) live in registers for the whole kernel, (2) the four heads' lane products
+// are reduce-scattered with warp_sum's pairing (xor 16, 8, then 4, 2, 1: six shuffles for the four heads) and gathered
+// to lane 0 in head order (three more). Every sum adds the same two operands as block_scores_kernel's warp_sum at the
+// same level (float add commutes), the dot expression is the same text in the same file, the relu-and-add runs over the
+// heads in the same order: the same score bits. The tail block and the +1e9 are as in the multi kernel.
+template <int NQ>
+__global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_rows_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out) {
+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];
+ if (threadIdx.x < nq) {
+ s_nkv[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNKv];
+ s_nbid[threadIdx.x] = steps[threadIdx.x * kStepCount + kStepNBid];
+ }
+ __syncthreads();
+ int64_t top = 0;
+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // the multi kernel's early exit (CTA-uniform)
+ const int lane = threadIdx.x & 31;
+ float4 qr[NQ][IDX_HEADS];
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi)
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ qr[qi][h] = qi < nq ? *reinterpret_cast<const float4*>(q_idx + qi * IDX_HEADS * IDX_DIM + h * IDX_DIM + lane * 4)
+ : make_float4(0.f, 0.f, 0.f, 0.f);
+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+ const bool hi16 = (lane & 16) != 0, hi8 = (lane & 8) != 0;
+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+#pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) {
+ if (qi >= nq) break;
+ const int64_t n_bid = s_nbid[qi];
+ if (b > n_bid) continue;
+ const float4 k4 = (b == n_bid) ? kd : kp;
+ float d[IDX_HEADS];
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float4 q4 = qr[qi][h];
+ d[h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ }
+ // reduce-scatter: after xor 16 the lane keeps heads {0,1} (bit 4 clear) or {2,3}; after xor 8 one head
+ float r2[2];
+#pragma unroll
+ for (int i = 0; i < 2; ++i)
+ r2[i] = (hi16 ? d[i + 2] : d[i]) + __shfl_xor_sync(0xffffffffu, hi16 ? d[i] : d[i + 2], 16);
+ float r = (hi8 ? r2[1] : r2[0]) + __shfl_xor_sync(0xffffffffu, hi8 ? r2[0] : r2[1], 8);
+ r += __shfl_xor_sync(0xffffffffu, r, 4);
+ r += __shfl_xor_sync(0xffffffffu, r, 2);
+ r += __shfl_xor_sync(0xffffffffu, r, 1);
+ // lane 8h holds head h's full sum: gather to lane 0 and add the relus in head order, as the warp kernel
+ const float s1 = __shfl_sync(0xffffffffu, r, 8);
+ const float s2 = __shfl_sync(0xffffffffu, r, 16);
+ const float s3 = __shfl_sync(0xffffffffu, r, 24);
+ if (lane == 0) {
+ float score = 0.0f;
+ score += r > 0.0f ? r : 0.0f;
+ score += s1 > 0.0f ? s1 : 0.0f;
+ score += s2 > 0.0f ? s2 : 0.0f;
+ score += s3 > 0.0f ? s3 : 0.0f;
+ if (b == n_bid && s_nkv[qi] % R != 0) score += 1e9f;
+ out[qi * max_blocks + b] = score;
+ }
+ }
+ }
+}
+
+#if defined(__HIPCC__)
+// Optional gfx1100 port of PR1825 (14bf3215): same FP32 expression and reduction tree.
+// Decode and the original top-k stay unchanged. No additional persistent weight/state allocation.
+constexpr int TS_Q = 64; // queries per CTA (8 per wave, four lanes per query)
+constexpr int TS_KB = 32; // key blocks per shared-memory tile
+constexpr int TS_BR = 512; // key blocks per CTA
+
+__device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {
+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression
+}
+
+__global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int64_t nq,
+ int64_t max_blocks, int64_t reach,
+ float* __restrict__ out) {
+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];
+ __shared__ float so[TS_Q][TS_KB + 1];
+ __shared__ int64_t s_nbid[TS_Q];
+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's
+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet
+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer
+ // shared-memory loads per FMA. The MIO-pipe motivation was measured by PR1825 on NVIDIA.
+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};
+ const int t = threadIdx.x, lane = t & 31;
+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter
+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;
+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;
+ float4 qv[IDX_HEADS][8];
+ {
+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+#pragma unroll
+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];
+ }
+ __syncthreads();
+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here
+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;
+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;
+ int64_t b_hi = b_lo + TS_BR;
+ if (b_hi > reach) b_hi = reach;
+ if (b_hi > top) b_hi = top;
+ if (b_hi > max_blocks) b_hi = max_blocks;
+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {
+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);
+ __syncthreads(); // the previous tile's reads of ks and so are done
+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)
+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];
+ __syncthreads();
+ for (int j = 0; j < nb; ++j) {
+ float4 kk[8];
+#pragma unroll
+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];
+ float e[IDX_HEADS];
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) +
+ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) +
+ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) +
+ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));
+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3
+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)
+ }
+ if (j4 == 0) {
+ float score = 0.0f;
+#pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;
+ so[ql][j] = score;
+ }
+ }
+ __syncthreads();
+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {
+ const int q = i / nb, j = i % nb;
+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];
+ }
+ }
+}
+
+// The exact scorer runs on gfx1100 (the PR's own card) when STRATA_SCORES_THREAD=1, and on gfx12 (RDNA4) unless STRATA_SCORES_THREAD=0:
+// there it is bitwise equal to the warp scorer on an R9700 (qsa_scores_gfx1100_parity --full, 35 cases; exact_pp residual dumps at 70K and
+// 131K tokens) and 3.6-4.4x faster in the scorer stage, +8% (131K) to +15% (230K) on a prompt read. Other wave32 parts have no measurement.
+// Returns 0 (not offered), 1 (gfx1100: only on request), 2 (gfx12: the default).
+int scores_thread_arch() {
+ int device = -1;
+ if (cudaGetDevice(&device) != cudaSuccess) { cudaGetLastError(); return 0; }
+ static thread_local int cached_device = -1;
+ static thread_local int arch = 0;
+ if (device != cached_device) {
+ cudaDeviceProp properties{};
+ arch = 0;
+ if (cudaGetDeviceProperties(&properties, device) == cudaSuccess)
+ arch = gfx_arch_is_gfx12(properties.gcnArchName) ? 2 : gfx_arch_is(properties.gcnArchName, "gfx1100") ? 1 : 0;
+ cached_device = device;
+ cudaGetLastError();
+ }
+ return arch;
+}
+#endif
+
+ #if !defined(__HIPCC__)
+ // ---- prompt calls on cards without the tensor-core scorer (Turing): block_scores_kernel's scores, BITWISE, with a
+ // thread per (query, head) instead of a warp per (query, block). There a warp spends 20 shuffles and five 512-byte
+ // loads per score; here a lane holds its query head (128 floats) in registers, every lane of the warp reads the same
+ // key from shared memory (a broadcast), and the 4-value partials are added in the tree lane 0 of the xor-shuffle sum
+ // computes: partial g covers values 4g..4g+3, level o adds g and g + o. Float addition is commutative, so the same
+ // pairs in the same tree give the same bits. The heads are then summed 0..3 from 0.0f as there. Blocks < n_bid only:
+ // the tail block n_bid (the `dead` key, +1e9) is block_scores_tail_kernel's.
+ constexpr int TS_Q = 64; // queries per CTA (8 per warp, a lane per query head)
+ constexpr int TS_KB = 32; // key blocks per shared-memory tile
+ constexpr int TS_BR = 512; // key blocks per CTA
+
+ __device__ __forceinline__ float ts_part(const float4 k4, const float4 q4) {
+ return k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w; // block_scores_kernel's per-lane expression
+ }
+
+ __global__ void __launch_bounds__(256, 1) block_scores_thread_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int64_t nq,
+ int64_t max_blocks, int64_t reach,
+ float* __restrict__ out) {
+ __shared__ __align__(16) float4 ks[TS_KB][IDX_DIM / 4];
+ __shared__ float so[TS_Q][TS_KB + 1];
+ __shared__ int64_t s_nbid[TS_Q];
+ // Four lanes per query, lane j (0..3) one quarter of the tree for all four heads: the partials c[j] of lane 0's
+ // xor-shuffle sum combine groups j, j+16, j+8, j+24, j+4, j+20, j+12, j+28, and only then do the quarters meet
+ // (c0 + c2, c1 + c3, then their sum) - two shuffles here. A key load now feeds four heads: four times fewer
+ // shared-memory loads per FMA (the kernel was bound by the MIO pipe, ncu: mio_throttle first).
+ constexpr int QG[8] = {0, 16, 8, 24, 4, 20, 12, 28};
+ const int t = threadIdx.x, lane = t & 31;
+ const int ql = (t >> 5) * 8 + (lane >> 2), j4 = lane & 3; // this lane's query (in the CTA) and quarter
+ const int64_t q0 = (int64_t) blockIdx.y * TS_Q, qi = q0 + ql;
+ if (t < TS_Q) s_nbid[t] = q0 + t < nq ? steps[(q0 + t) * kStepCount + kStepNBid] : -1;
+ float4 qv[IDX_HEADS][8];
+ {
+ const float4* src = reinterpret_cast<const float4*>(q_idx + (qi < nq ? qi : 0) * IDX_HEADS * IDX_DIM);
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h)
+ #pragma unroll
+ for (int m = 0; m < 8; ++m) qv[h][m] = src[h * (IDX_DIM / 4) + j4 + QG[m]];
+ }
+ __syncthreads();
+ int64_t top = -1; // the largest n_bid of the CTA's queries: blocks at or past it are never written here
+ for (int i = 0; i < TS_Q; ++i) top = s_nbid[i] > top ? s_nbid[i] : top;
+ const int64_t b_lo = (int64_t) blockIdx.x * TS_BR;
+ int64_t b_hi = b_lo + TS_BR;
+ if (b_hi > reach) b_hi = reach;
+ if (b_hi > top) b_hi = top;
+ if (b_hi > max_blocks) b_hi = max_blocks;
+ for (int64_t b0 = b_lo; b0 < b_hi; b0 += TS_KB) {
+ const int nb = (int) (b_hi - b0 < TS_KB ? b_hi - b0 : TS_KB);
+ __syncthreads(); // the previous tile's reads of ks and so are done
+ for (int i = t; i < nb * (IDX_DIM / 4); i += blockDim.x)
+ ks[i / (IDX_DIM / 4)][i % (IDX_DIM / 4)] = reinterpret_cast<const float4*>(pooled + (b0 + i / (IDX_DIM / 4)) * IDX_DIM)[i % (IDX_DIM / 4)];
+ __syncthreads();
+ for (int j = 0; j < nb; ++j) {
+ float4 kk[8];
+ #pragma unroll
+ for (int m = 0; m < 8; ++m) kk[m] = ks[j][j4 + QG[m]];
+ float e[IDX_HEADS];
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ const float c = ((ts_part(kk[0], qv[h][0]) + ts_part(kk[1], qv[h][1])) +
+ (ts_part(kk[2], qv[h][2]) + ts_part(kk[3], qv[h][3]))) +
+ ((ts_part(kk[4], qv[h][4]) + ts_part(kk[5], qv[h][5])) +
+ (ts_part(kk[6], qv[h][6]) + ts_part(kk[7], qv[h][7])));
+ const float d = c + __shfl_xor_sync(0xffffffffu, c, 2); // lane 0: c0 + c2, lane 1: c1 + c3
+ e[h] = d + __shfl_xor_sync(0xffffffffu, d, 1); // lane 0: (c0 + c2) + (c1 + c3)
+ }
+ if (j4 == 0) {
+ float score = 0.0f;
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) score += e[h] > 0.0f ? e[h] : 0.0f;
+ so[ql][j] = score;
+ }
+ }
+ __syncthreads();
+ for (int i = t; i < TS_Q * nb; i += blockDim.x) {
+ const int q = i / nb, j = i % nb;
+ if (b0 + j < s_nbid[q]) out[(q0 + q) * max_blocks + b0 + j] = so[q][j];
+ }
+ }
+ }
+ #endif
+
#if !defined(__HIPCC__)
// ---- the decode top-k on a thread-block cluster (sm_90+; S19). One CTA per query (block_topk_reg_kernel, or
// block_topk_kernel above 33,792 blocks: a --max-context over ~135K) makes four radix passes and two scans over up to
@@@ -1358,38 -1276,138 +1544,168 @@@ __global__ void __launch_bounds__(CL_T
#endif
}
#endif // !__HIPCC__
+bool scores_multi_on() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();
+ return on;
+}
+// STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits
+bool scores_early_exit() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();
+ return on;
+}
} // namespace
+bool qsa_block_scores_slots(const float* const* pooled, const float* const* dead, const float* q_idx,
+ const int32_t* steps, int64_t nq, int64_t max_blocks, const QsaShapes& s, float* scores,
+ void* stream) {
+ if (nq <= 0) return true;
+ if (s.idx_dim != IDX_DIM || s.idx_n_head != IDX_HEADS || s.idx_block != R) {
+ std::fprintf(stderr, "qsa_block_scores_slots: unsupported indexer geometry\n");
+ std::exit(1);
+ }
+ if (nq > MQ || !scores_multi_on()) return false; // the single-query call would not run the multi kernel
+ ScoreSlotKeys keys{};
+ for (int64_t z = 0; z < nq; ++z) {
+ keys.pooled[z] = pooled[z];
+ keys.dead[z] = dead[z];
+ }
+ block_scores_slots_kernel<<<dim3(256, (unsigned) nq), SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(
+ keys, q_idx, steps, max_blocks, scores, scores_early_exit());
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores_slots: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return true;
+}
+
+ // block_scores_multi_kernel with the warp sums as one recursive-halving reduction. A (query, head) dot is a lane's
+ // 4-term product summed over the 32 lanes by an xor butterfly (offsets 16, 8, 4, 2, 1): 5 shuffles per value, 20 per
+ // (block, query) - on Turing the decode scorer's time grew with the window's queries (420K cells: 117 / 265 / 509 us
+ // for 1 / 3 / 6 queries), the shuffles, not the keys. Here the V = 4 x queries values (padded to a power of 2) are
+ // reduced together: at offset o each lane keeps half of its values and adds its partner's copy of them - the pair
+ // (lane l, lane l ^ o) the butterfly adds for that value at that offset, so every value's additions are the
+ // butterfly's, operand for operand (float addition commutes): the same bits, with V - 1 shuffles plus the plain
+ // butterfly's remaining stages instead of 5 V. Lane l ends with value (l >> (5 - log2 V)); the query's 4 heads are
+ // then gathered by its head-0 lane and summed in block_scores_multi_kernel's order. STRATA_SCORES_HALVE=0: the old kernel.
+ template <int V>
+ __global__ void __launch_bounds__(SCORE_WARPS * 32) block_scores_halve_kernel(const float* __restrict__ pooled,
+ const float* __restrict__ dead,
+ const float* __restrict__ q_idx,
+ const int32_t* __restrict__ steps, int nq,
+ int64_t max_blocks, float* __restrict__ out) {
+ constexpr int NQ = V / IDX_HEADS; // query slots (nq <= NQ)
+ constexpr int LV = V == 4 ? 2 : V == 8 ? 3 : V == 16 ? 4 : 5, SH = 5 - LV;
+ static_assert(V == 4 || V == 8 || V == 16 || V == 32, "V");
+ __shared__ __align__(16) float qs[NQ * IDX_HEADS * IDX_DIM];
+ __shared__ int64_t s_nkv[NQ], s_nbid[NQ];
+ if (threadIdx.x < NQ) {
+ s_nkv[threadIdx.x] = threadIdx.x < nq ? steps[threadIdx.x * kStepCount + kStepNKv] : 0;
+ s_nbid[threadIdx.x] = threadIdx.x < nq ? steps[threadIdx.x * kStepCount + kStepNBid] : -1;
+ }
+ __syncthreads();
+ int64_t top = 0;
+ for (int q = 0; q < nq; ++q) top = s_nbid[q] > top ? s_nbid[q] : top;
+ if ((int64_t) blockIdx.x * SCORE_WARPS > top) return; // CTA-uniform (as block_scores_multi_kernel's early exit)
+ for (int i = threadIdx.x; i < NQ * IDX_HEADS * IDX_DIM; i += blockDim.x)
+ qs[i] = i < nq * IDX_HEADS * IDX_DIM ? q_idx[i] : 0.0f;
+ __syncthreads();
+ const int lane = threadIdx.x & 31;
+ const int64_t wstride = (int64_t) gridDim.x * SCORE_WARPS;
+ const float4 kd = *reinterpret_cast<const float4*>(dead + lane * 4);
+ // up to 4 queries: the lane's query values in registers (read once, not once per block from shared memory: the
+ // shared-memory reads were the kernel's limit, ncu mio_throttle)
+ constexpr bool QREG = V <= 16;
+ float4 qr[QREG ? V : 1];
+ if constexpr (QREG) {
+ #pragma unroll
+ for (int i = 0; i < V; ++i) qr[i] = *reinterpret_cast<const float4*>(qs + (i >> 2) * IDX_HEADS * IDX_DIM + (i & 3) * IDX_DIM + lane * 4);
+ }
+ int s_nbid_r[NQ]; // n_bid < 2^31
+ #pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) s_nbid_r[qi] = (int) s_nbid[qi];
+ for (int64_t b = (int64_t) blockIdx.x * SCORE_WARPS + (threadIdx.x >> 5); b <= top && b < max_blocks; b += wstride) {
+ const float4 kp = *reinterpret_cast<const float4*>(pooled + b * IDX_DIM + lane * 4);
+ float v[V];
+ #pragma unroll
+ for (int qi = 0; qi < NQ; ++qi) {
+ const float4 k4 = (b == s_nbid_r[qi]) ? kd : kp;
+ const float* q = qs + qi * IDX_HEADS * IDX_DIM + lane * 4;
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) {
+ float4 q4;
+ if constexpr (QREG) q4 = qr[qi * IDX_HEADS + h];
+ else q4 = *reinterpret_cast<const float4*>(q + h * IDX_DIM);
+ v[qi * IDX_HEADS + h] = k4.x * q4.x + k4.y * q4.y + k4.z * q4.z + k4.w * q4.w;
+ }
+ }
+ // recursive halving over offsets 16, 8, ... while more than one value is left, then the plain butterfly
+ int cnt = V;
+ #pragma unroll
+ for (int o = 16; o > 0; o >>= 1) {
+ if (cnt > 1) {
+ const int half = cnt >> 1;
+ const bool hi = (lane & o) != 0;
+ #pragma unroll
+ for (int i = 0; i < V / 2; ++i) {
+ if (i < half) {
+ const float send = hi ? v[i] : v[i + half];
+ const float keep = hi ? v[i + half] : v[i];
+ const float recv = __shfl_xor_sync(0xffffffffu, send, o);
+ v[i] = keep + recv;
+ }
+ }
+ cnt = half;
+ } else {
+ v[0] += __shfl_xor_sync(0xffffffffu, v[0], o);
+ }
+ }
+ // lane l holds value j = l >> SH = qi * 4 + h; the head-0 lane of each query gathers heads 1-3
+ const int j = lane >> SH, qi = j >> 2;
+ const float d1 = __shfl_sync(0xffffffffu, v[0], (lane + (1 << SH)) & 31);
+ const float d2 = __shfl_sync(0xffffffffu, v[0], (lane + (2 << SH)) & 31);
+ const float d3 = __shfl_sync(0xffffffffu, v[0], (lane + (3 << SH)) & 31);
+ if ((j & 3) == 0 && (lane & ((1 << SH) - 1)) == 0 && qi < nq && b <= s_nbid_r[qi & (NQ - 1)]) {
+ const float d[4] = {v[0], d1, d2, d3};
+ float score = 0.0f;
+ #pragma unroll
+ for (int h = 0; h < IDX_HEADS; ++h) score += d[h] > 0.0f ? d[h] : 0.0f;
+ if (b == s_nbid_r[qi & (NQ - 1)] && s_nkv[qi] % R != 0) score += 1e9f;
+ out[qi * max_blocks + b] = score;
+ }
+ }
+ }
+
+ // #1825: the prompt path's thread-per-head scorer (bitwise the warp kernel's scores). Default: Turing with
+ // STRATA_SM75_PERF=1 (the P100 gain measured here is +1..3% at 30-60K tokens, under the 3% bar for a new default);
+ // STRATA_SCORES_THREAD=1 turns it on, =0 off, on any CUDA card
+ bool scores_thread_on() {
+ static const int env = [] { const char* v = std::getenv("STRATA_SCORES_THREAD"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();
+ if (env >= 0) return env != 0;
+ static int ok[64] = {}; // 0 unknown, 1 yes, 2 no
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+ if (ok[dev] == 0) {
+ int major = 0, minor = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+ ok[dev] = strata::sm75_default(major, minor) ? 1 : 2; // Turing, STRATA_SM75_PERF=1
+ }
+ return ok[dev] == 1;
+ }
+
+ bool scores_halve_on() {
+ static const int env = [] { const char* v = std::getenv("STRATA_SCORES_HALVE"); return v ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();
+ if (env >= 0) return env != 0;
+ static int ok[64] = {}; // 0 unknown, 1 yes, 2 no
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }
+ if (ok[dev] == 0) {
+ int major = 0, minor = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);
+ ok[dev] = strata::sm75_default(major, minor) ? 1 : 2; // Turing, STRATA_SM75_PERF=1
+ }
+ return ok[dev] == 1;
+ }
+
void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
int64_t max_blocks, const QsaShapes& s, float* scores, void* stream, int64_t active_blocks) {
if (nq <= 0) return;
@@@ -1398,45 -1416,48 +1714,76 @@@
std::exit(1);
}
// a block past a query's n_bid returns at once: the grid need only reach the batch's largest n_bid (C-1)
- static const bool multi = [] { const char* v = std::getenv("STRATA_SCORES_MULTI"); return v == nullptr || std::atoi(v) != 0; }();
- if (multi && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16
+ if (scores_multi_on() && nq <= MQ && active_blocks <= 0) { // no active count: decode (captured or not) and prefill's pooled16
+ // QSAROWS: STRATA_QSA_ROWS=1 (exactly) takes block_scores_rows_kernel for calls of up to 4 queries - the same bits
+ static const bool rows_env = [] { const char* v = std::getenv("STRATA_QSA_ROWS"); return v != nullptr && v[0] == '1' && v[1] == '\0'; }();
+ if (rows_env && qsa_block_scores_rows(pooled, dead, q_idx, steps, nq, max_blocks, s, scores, stream)) return;
+ if (scores_halve_on()) { // Turing: block_scores_halve_kernel, the same bits
+ const cudaStream_t st = (cudaStream_t) stream;
+ if (nq <= 1) block_scores_halve_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);
+ else if (nq <= 2) block_scores_halve_kernel<8><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);
+ else if (nq <= 4) block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, (int) nq, max_blocks, scores);
+ else { // 5-8 queries: two calls of up to 4 (the register form; each query's scores are its own)
+ block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q_idx, steps, 4, max_blocks, scores);
+ const int rest = (int) nq - 4;
+ const float* q2 = q_idx + 4 * IDX_HEADS * IDX_DIM;
+ const int32_t* st2 = steps + 4 * kStepCount;
+ float* sc2 = scores + 4 * max_blocks;
+ if (rest <= 1) block_scores_halve_kernel<4><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);
+ else if (rest <= 2) block_scores_halve_kernel<8><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);
+ else block_scores_halve_kernel<16><<<256, SCORE_WARPS * 32, 0, st>>>(pooled, dead, q2, st2, rest, max_blocks, sc2);
+ }
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores halve: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return;
+ }
- // STRATA_QSA_EARLY_EXIT=0 keeps the old staging (every CTA loads the queries first); the scores are the same bits
- static const bool early = [] { const char* v = std::getenv("STRATA_QSA_EARLY_EXIT"); return v == nullptr || std::atoi(v) != 0; }();
block_scores_multi_kernel<<<256, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, (int) nq,
- max_blocks, scores, early);
+ max_blocks, scores, scores_early_exit());
const cudaError_t e = cudaGetLastError();
if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores multi: %s\n", cudaGetErrorString(e)); std::exit(1); }
return;
}
const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
-#if !defined(__HIPCC__)
+#if defined(__HIPCC__)
+ // STRATA_SCORES_THREAD: 1 = on (gfx1100, gfx12), 0 = off, unset = on for gfx12 only (gfx1100: off, nothing measured here)
+ static const int thread_env = [] {
+ const char* value = std::getenv("STRATA_SCORES_THREAD");
+ return value == nullptr ? -1 : (std::atoi(value) == 1 ? 1 : 0);
+ }();
+ // The bound is the ACTUAL live extent. A reserved 256K context never enables this by itself.
+ if (thread_env != 0 && active_blocks >= 16385 && active_blocks <= max_blocks && nq >= 64 &&
+ (scores_thread_arch() == 2 || (thread_env == 1 && scores_thread_arch() == 1))) {
+ static thread_local bool logged = false;
+ if (!logged && (thread_env == -1 || std::getenv("STRATA_SCORES_THREAD_TRACE"))) {
+ std::fprintf(stderr, "qsa exact thread: the prompt's block scorer runs as the four-lanes-per-query kernel (nq=%lld active_blocks=%lld "
+ "capacity=%lld; the same scores bit for bit; STRATA_SCORES_THREAD=0 turns it off)\n",
+ (long long)nq, (long long)active_blocks, (long long)max_blocks);
+ logged = true;
+ }
+ const dim3 tgrid((unsigned)((reach + TS_BR - 1) / TS_BR), (unsigned)((nq + TS_Q - 1) / TS_Q));
+ block_scores_thread_kernel<<<tgrid, 256, 0, (cudaStream_t)stream>>>(pooled, q_idx, steps, nq,
+ max_blocks, reach, scores);
+ block_scores_tail_kernel<<<(unsigned)nq, 32, 0, (cudaStream_t)stream>>>(dead, q_idx, steps,
+ max_blocks, scores);
+ const cudaError_t error = cudaGetLastError();
+ if (error != cudaSuccess) {
+ std::fprintf(stderr, "qsa exact thread: %s\n", cudaGetErrorString(error));
+ std::exit(1);
+ }
+ return;
+ }
++#else
+ // prompt calls (an active-block count): the thread-per-head kernel, the same bits (opt-in: scores_thread_on); else the
+ // warp kernel below
+ if (active_blocks > 0 && scores_thread_on()) {
+ const dim3 tgrid((unsigned) ((reach + TS_BR - 1) / TS_BR), (unsigned) ((nq + TS_Q - 1) / TS_Q));
+ block_scores_thread_kernel<<<tgrid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks, reach,
+ scores);
+ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks, scores);
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores thread: %s\n", cudaGetErrorString(e)); std::exit(1); }
+ return;
+ }
#endif
const dim3 grid((unsigned) ((reach + SCORE_WARPS - 1) / SCORE_WARPS), (unsigned) nq);
block_scores_kernel<<<grid, SCORE_WARPS * 32, 0, (cudaStream_t) stream>>>(pooled, dead, q_idx, steps, max_blocks,
@@@ -1535,8 -1513,31 +1883,28 @@@ bool qsa_block_scores_tc(const float* p
}();
if (!simt) return false;
const int64_t reach = active_blocks > 0 && active_blocks < max_blocks ? active_blocks : max_blocks;
+ if (simt == 2) {
+ // STRATA_SIMT2_TILE=2: 64 x 64 tiles (measured slower: 400K 3.00 vs 2.82 ms)
+ static const int tile = [] { const char* v = std::getenv("STRATA_SIMT2_TILE"); return v ? std::atoi(v) : 1; }();
+ if (tile != 2) {
+ const dim3 grid((unsigned) ((reach + 127) / 128), (unsigned) ((nq + 31) / 32));
+ if (grid.y > 65535) return false;
+ block_scores_simt2_kernel<32, 128><<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq,
+ max_blocks, reach, scores);
+ } else {
+ const dim3 grid((unsigned) ((reach + 63) / 64), (unsigned) ((nq + 63) / 64));
+ if (grid.y > 65535) return false;
+ block_scores_simt2_kernel<64, 64><<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq,
+ max_blocks, reach, scores);
+ }
- } else {
- const dim3 grid((unsigned) ((reach + SM_NB - 1) / SM_NB), (unsigned) ((nq + SM_QT - 1) / SM_QT));
- if (grid.y > 65535) return false;
- block_scores_simt_kernel<<<grid, 256, 0, (cudaStream_t) stream>>>(pooled, q_idx, steps, nq, max_blocks,
- reach, scores);
++ block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks,
++ scores);
++ const cudaError_t e = cudaGetLastError();
++ if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }
++ return true;
+ }
- block_scores_tail_kernel<<<(unsigned) nq, 32, 0, (cudaStream_t) stream>>>(dead, q_idx, steps, max_blocks,
- scores);
- const cudaError_t e = cudaGetLastError();
- if (e != cudaSuccess) { std::fprintf(stderr, "qsa_block_scores (simt): %s\n", cudaGetErrorString(e)); std::exit(1); }
- return true;
+ return block_scores_simt(pooled, dead, q_idx, steps, nq, max_blocks, reach, scores, (cudaStream_t) stream,
+ false);
}
}
static bool attr[64] = {}; // the shared-memory opt-in is per device (a layer split runs it on several)
@@@ -1724,9 -2193,15 +2567,15 @@@ void qsa_block_topk(const float* scores
const char* v = std::getenv("STRATA_TOPK_STREAM");
return v == nullptr || v[0] != '0';
}();
- if (reach > fit || (turing_wide && counted && reach > 22528)) // past the register kernel's reach, or Turing's band
- block_topk_wide_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
- else
+ if (reach > fit || (turing_wide && counted && reach > 22528)) { // past the register kernel's reach, or Turing's band
++#if !defined(__HIPCC__)
+ if (counted && nq <= TS_MAXQ && reach > 28672 && topk_sample_on()) { // one read of the scores (below ~115K cells the wide kernel is as fast)
+ block_topk_sample_kernel<<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
+ block_topk_wide_kernel<true><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
- } else {
- block_topk_wide_kernel<false><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
- }
- } else
++ } else
+ #endif
++ block_topk_wide_kernel<false><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
++ } else
if (reach <= (int64_t) TK_T * TK_PER)
block_topk_reg_kernel<TK_PER><<<(unsigned) nq, TK_T, 0, (cudaStream_t) stream>>>(scores, steps, max_blocks, cap, ids);
else
diff --cc src/kernels/qsa_prompt_attn_parity.cpp
index 706fa13b1,5de9f38fa..0a7b8b097
--- a/src/kernels/qsa_prompt_attn_parity.cpp
+++ b/src/kernels/qsa_prompt_attn_parity.cpp
@@@ -298,36 -285,18 +298,42 @@@ int run(int fmt, int64_t ctx, int64_t n
cudaEventRecord(e1);
ck(cudaEventSynchronize(e1), "time");
cudaEventElapsedTime(&ms_new, e0, e1);
- const bool ok1 = err_new <= std::max(4.0 * err_old, 1e-6 * ref_scale);
+ bool ok_im = true;
+ if (fmt == 1) { // the INT8-MMA kernel: the same bounds as v2's, against FP64 and against v2
+ double d_iv = 0;
+ for (size_t i = 0; i < o.size(); ++i) d_iv = std::max(d_iv, (double) std::fabs(im[i] - nw[i]));
+ float ms_im = 0, ms_v2 = 0;
+ for (int round = 0; round < 2; ++round) { // alternating, v2 then imma, twice; the second round counts
+ cudaEventRecord(e0);
+ for (int r = 0; r < reps; ++r) new_run();
+ cudaEventRecord(e1);
+ ck(cudaEventSynchronize(e1), "time");
+ cudaEventElapsedTime(&ms_v2, e0, e1);
+ cudaEventRecord(e0);
+ for (int r = 0; r < reps; ++r) imma_run();
+ cudaEventRecord(e1);
+ ck(cudaEventSynchronize(e1), "time");
+ cudaEventElapsedTime(&ms_im, e0, e1);
+ }
+ ok_im = err_im <= std::max(4.0 * err_old, 1e-6 * ref_scale) && d_iv <= 1e-4 * scale;
+ std::printf("%s int8 IMMA ctx %lld, %lld queries: vs FP64 %.3g (v2 %.3g, FP32 %.3g); IMMA vs v2 %.3g (%.2g of scale); "
+ "v2 %.3f -> IMMA %.3f ms per chunk (%.2fx)\n", ok_im ? "PASS" : "FAIL", (long long) ctx, (long long) nq,
+ err_im, err_new, err_old, d_iv, d_iv / scale, ms_v2 / reps, ms_im / reps, ms_v2 / ms_im);
+ }
+ k::qsa_prompt_attn_set_imma(-1);
+ cudaFree(d_im);
+ const bool ok1 = err_new <= std::max(4.0 * err_old, 1e-6 * ref_scale) && ok_im;
const bool ok2 = diff <= 1e-4 * scale;
+ uint64_t sum = 1469598103934665603ull; // the new kernel's output bits: two builds of it compare by this
+ for (float f : nw) {
+ uint32_t b;
+ std::memcpy(&b, &f, 4);
+ sum = (sum ^ b) * 1099511628211ull;
+ }
std::printf("%s %s ctx %lld, %lld queries: vs FP64 old %.3g new %.3g (output scale %.3g); new vs old %.3g (%.2g of "
- "scale); %.3f -> %.3f ms per chunk (%.2fx)\n",
+ "scale); %.3f -> %.3f ms per chunk (%.2fx); new output checksum %016llx\n",
ok1 && ok2 ? "PASS" : "FAIL", fmt == 2 ? "q4_0" : fmt == 1 ? "int8" : "fp16", (long long) ctx, (long long) nq, err_old,
- err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new);
+ err_new, ref_scale, diff, diff / scale, ms_old / reps, ms_new / reps, ms_old / ms_new, (unsigned long long) sum);
cudaFree((void*) d_ids); cudaFree((void*) d_steps); cudaFree((void*) d_q); cudaFree(d_old); cudaFree(d_new);
cudaFree(scratch);
cudaFree((void*) pl.k_q); cudaFree((void*) pl.v_q); cudaFree((void*) pl.k_scale); cudaFree((void*) pl.v_scale);
diff --cc src/prefill/gemm.cu
index 50ca2b142,9ad9eb105..538e17eb3
--- a/src/prefill/gemm.cu
+++ b/src/prefill/gemm.cu
@@@ -816,99 -621,19 +816,110 @@@ bool Gemm::bf16_hcd_exact(const uint16_
#endif
}
+#if defined(__HIPCC__)
+// ---- RDNA2 (gfx103x): the prompt path's GEMMs as FP32 SGEMMs ------------------------------------------------------
+// The GEMMs here take 16-bit inputs (FP16 or BF16) and write FP32 (rocBLAS's "HS" / "BS" types). The rocBLAS the engine
+// ships has tuned gfx1030 kernels for FP16 -> FP16 ("HH"), int8 and FP32 -> FP32 ("SS") only: every 16-bit-input,
+// FP32-output GEMM runs a "fallback" kernel at ~5 TFLOPS. Measured on an RX 6800 (ROCm 10.2.0a20260930) with the
+// engine's exact calls (opA = T, opB = N, fp32 compute): at the Coder's prompt shapes the FP16 GEMMs of one fresh
+// 12K-token prompt take 14.1 s, the same GEMMs as SGEMM on widened copies 4.7 s including the widening (e.g.
+// N 10240 x T 8192 x K 2560: 86.7 -> 27.7 ms); the BF16 hyper-connection GEMMs 10.9 -> 7.5 ms (N 320, K 10240).
+// Widening FP16 / BF16 to FP32 is exact and SGEMM accumulates in FP32 like the native call, so the products are the
+// same up to summation order. The weight is widened once per call, the activations in slices of up to 128 MiB of
+// FP32 (an 8,192-token chunk at K = 2560 is one slice). Small N (< 64: routers, gates) keeps the native call:
+// there the widening costs more than the GEMM gains. If the FP32 buffers cannot be allocated, the native call runs.
+// OPT-IN here: STRATA_RDNA2_SGEMM=1 turns it on for gfx103x devices (other cards keep the native call, also in a
+// mixed layer split); =all turns it on for every AMD card (A/B arms, tests); unset or 0 is the native path. The
+// author ran it as the gfx103x default (RX 6800 343 -> 534 tok/s); it is not bit-identical to the native call.
+namespace {
+__global__ void rdna2_bf16_to_f32_kernel(const uint16_t* __restrict__ in, float* __restrict__ out, int64_t n) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) out[i] = __uint_as_float((uint32_t) in[i] << 16);
+}
+__global__ void rdna2_f16_to_f32_kernel(const __half* __restrict__ in, float* __restrict__ out, int64_t n) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) out[i] = __half2float(in[i]);
+}
+void rdna2_widen(const uint16_t* in, float* out, int64_t n, bool bf16, cudaStream_t st) {
+ if (n <= 0) return;
+ const unsigned blocks = (unsigned) ((n + 255) / 256);
+ if (bf16) rdna2_bf16_to_f32_kernel<<<blocks, 256, 0, st>>>(in, out, n);
+ else rdna2_f16_to_f32_kernel<<<blocks, 256, 0, st>>>(reinterpret_cast<const __half*>(in), out, n);
+}
+// whether this GEMM runs as an SGEMM (see above)
+bool rdna2_sgemm_on(int64_t N) {
+ static const int mode = [] { // 0 off (unset), 1 gfx103x devices, 2 every AMD device
+ const char* v = std::getenv("STRATA_RDNA2_SGEMM");
+ if (v == nullptr || v[0] == '\0') return 0;
+ if (std::strcmp(v, "all") == 0) return 2;
+ return std::atoi(v) > 0 ? 1 : 0;
+ }();
+ if (mode == 0 || N < 64) return false;
+ if (mode == 2) return true;
+ static std::atomic<int> arch[64] = {}; // per device: 0 not known, 1 gfx103x, 2 another architecture
+ int dev = 0;
+ if (hipGetDevice(&dev) != hipSuccess || dev < 0 || dev >= 64) { (void) hipGetLastError(); return false; }
+ int v = arch[dev].load(std::memory_order_relaxed);
+ if (v == 0) {
+ hipDeviceProp_t prop{};
+ if (hipGetDeviceProperties(&prop, dev) != hipSuccess) { (void) hipGetLastError(); return false; }
+ v = std::strncmp(prop.gcnArchName, "gfx103", 6) == 0 ? 1 : 2;
+ arch[dev].store(v, std::memory_order_relaxed);
+ }
+ return v == 1;
+}
+// `have`, `want`: FP32 elements (the buffers are the instance's 2-byte tc_ buffers, sized in 2-byte elements)
+bool rdna2_grow(uint16_t*& p, int64_t& have_u16, int64_t want_f32) {
+ const int64_t want = 2 * want_f32;
+ if (have_u16 >= want) return true;
+ if (p) cudaFree(p);
+ p = nullptr;
+ have_u16 = 0;
+ if (cudaMalloc((void**) &p, (size_t) want * 2) != cudaSuccess) { (void) cudaGetLastError(); p = nullptr; return false; }
+ have_u16 = want;
+ return true;
+}
+constexpr int64_t kRdna2SliceF32 = 32ll << 20; // 128 MiB of FP32 activations per slice
+} // namespace
+
+// Y^T[N, T] (+)= W[N, K] . X^T[K, T] as SGEMMs on FP32 copies; false: not this card / shape, or no buffers (the caller
+// runs its native GEMM, nothing was written)
+bool Gemm::rdna2_sgemm(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy,
+ float beta, bool bf16) {
+ if (K <= 0 || !rdna2_sgemm_on(N)) return false;
+ const int64_t rows = std::max<int64_t>(1, std::min<int64_t>(T, kRdna2SliceF32 / K));
+ // growing frees a buffer an earlier GEMM on this stream may still read (as the sm_60 FP32 route does)
+ if (tc_w_elems_ < 2 * N * K || tc_x_elems_ < 2 * rows * K) (void) hipStreamSynchronize((hipStream_t) stream_);
+ if (!rdna2_grow(tc_w_, tc_w_elems_, N * K) || !rdna2_grow(tc_x_, tc_x_elems_, rows * K)) return false;
+ float* const wf = reinterpret_cast<float*>(tc_w_);
+ float* const xf = reinterpret_cast<float*>(tc_x_);
+ const float alpha = 1.0f;
+ rdna2_widen(W, wf, N * K, bf16, (cudaStream_t) stream_);
+ for (int64_t t0 = 0; t0 < T; t0 += rows) {
+ const int64_t n = std::min<int64_t>(rows, T - t0);
+ rdna2_widen(X + t0 * K, xf, n * K, bf16, (cudaStream_t) stream_);
+ // every slice is a disjoint block of Y, so each gets the caller's beta
+ ck(hipblasSgemm((hipblasHandle_t) handle_, HIPBLAS_OP_T, HIPBLAS_OP_N, (int) N, (int) n, (int) K, &alpha, wf,
+ (int) K, xf, (int) K, &beta, Y + t0 * ldy, (int) ldy),
+ "hipblasSgemm (RDNA2)");
+ }
+ return true;
+}
+#endif
+
+ bool Gemm::bf16_converts_x() const {
+ #if defined(__HIPCC__) || defined(STRATA_EXPERIMENTAL_SM60)
+ return false;
+ #else
+ return bf16_path() == 1;
+ #endif
+ }
+ __global__ void f16_to_bf16_kernel(const __half* __restrict__ in, uint16_t* __restrict__ out, int64_t n) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) out[i] = (uint16_t) (__float_as_uint(__half2float(in[i])) >> 16); // exact: the value was a BF16
+ }
void Gemm::bf16(const uint16_t* X, const uint16_t* W, float* Y, int64_t T, int64_t N, int64_t K, int64_t ldy,
- float beta, int64_t ldx) {
+ float beta, int64_t ldx, bool x_f16) {
if (T <= 0 || N <= 0) return;
if (ldy <= 0) ldy = N;
if (ldx <= K) ldx = 0;
diff --cc src/prefill/kernels.cu
index 9b79e4010,50178462a..29c9176cb
--- a/src/prefill/kernels.cu
+++ b/src/prefill/kernels.cu
@@@ -42,30 -42,18 +42,40 @@@ __device__ __forceinline__ uint16_t bf(
__device__ __forceinline__ uint16_t bf_lo(float f, uint16_t hi) { return bf(f - __uint_as_float((uint32_t) hi << 16)); }
__device__ __forceinline__ float sigm(float x) { return 1.0f / (1.0f + __expf(-x)); }
__device__ __forceinline__ uint16_t hf(float f) { return __half_as_ushort(__float2half_rn(f)); }
++}
// A SwiGLU product for an FP16 GEMM: saturated, so a token with a massive activation cannot turn into inf and then
// NaN in the down projection (decode's q8_1 has room to ~8e6; FP16 ends at 65504). A NaN stays NaN (fminf/fmaxf
// would make it -65504 and hide where it came from); finite values below 65504 round exactly as before.
+// STRATA_F16_RANGE=1 (set_f16_range): what reaches FP16's range on this device - the largest |value| the activation
+// images converted (hf_sat is their one funnel) and how many were beyond 65504 or NaN; W and Y are counted in gemm.cu.
+// Off, the cost is one flag read per value. The max is kept as float bits (positive floats order as unsigned ints),
+// and a thread takes the atomic only when it beats the value it read, so the atomics stop once the max has settled.
+#if defined(__HIPCC__) // HIP only: the CUDA kernels stay exactly as they were (no flag read in hf_sat)
+__device__ int g_f16_range_on = 0;
+__device__ unsigned g_f16_range_max = 0;
+__device__ unsigned long long g_f16_range_over = 0;
+__device__ __forceinline__ void f16_range_note(float f) {
+ const float a = fabsf(f);
+ const unsigned bits = __float_as_uint(a);
+ if (bits > g_f16_range_max) atomicMax(&g_f16_range_max, bits);
+ if (a > 65504.0f || isnan(f)) atomicAdd(&g_f16_range_over, 1ull);
+}
+#endif
+__device__ __forceinline__ uint16_t hf_sat(float f) {
+#if defined(__HIPCC__)
+ if (g_f16_range_on) f16_range_note(f);
+#endif
+ return hf(isnan(f) ? f : fminf(fmaxf(f, -65504.0f), 65504.0f));
+}
++
+ // Turing's FP16 image of the hyper-connection's xn16 (gemm.cu's bf16_to_f16_kernel applied to a BF16 value): the
+ // writers store what the BF16 tensor-core path would convert, so the GEMM reads it without a conversion pass
+ __device__ __forceinline__ uint16_t bf_to_hf_conv(uint16_t b) {
+ float f = __uint_as_float((uint32_t) b << 16);
+ if (f > 65504.0f && !isinf(f)) f = 65504.0f;
+ else if (f < -65504.0f && !isinf(f)) f = -65504.0f;
+ return __half_as_ushort(__float2half_rn(f));
+ }
-__device__ __forceinline__ uint16_t hf_sat(float f) { return hf(isnan(f) ? f : fminf(fmaxf(f, -65504.0f), 65504.0f)); }
// The prompt path's 16-bit activation image for the BF16-weight GEMMs: BF16, or FP16 where the GEMM library is fast
// only in FP16 (prompt_f16() in gemm.cu: rocBLAS on gfx103x). Set once per device before the first prompt.
#if defined(__HIPCC__) // HIP only (#835): the CUDA kernels stay exactly as they were, they never read the flag
diff --cc src/prefill/moe_fused_iq.cu
index 637f0ec3d,ce51c838e..bc870c892
--- a/src/prefill/moe_fused_iq.cu
+++ b/src/prefill/moe_fused_iq.cu
@@@ -21,12 -21,8 +21,13 @@@
// (SwiGLU, H to int8 per 32 features; down into the per-slot rows) are moe_fused.cu's.
#include "strata/prefill/moe_fused_iq.hpp"
#include "strata/kernels/gfx_arch.hpp"
+ #include "strata/core/emulate.hpp"
+#if defined(__HIPCC__)
+#include "moe_f16_common.cuh"
+#include "moe_f16_kq.hpp"
+#endif
+
#include <cuda_fp16.h>
#include <cuda_runtime.h>
@@@ -1224,13 -1169,11 +1320,16 @@@ bool kq_on()
}();
return on;
}
+// STRATA_PF_F16=1 (gfx11): the formats native_f16_kernel takes (the K-quants are moe_f16_kq.cu's)
+bool f16_gu(int t) {
+ return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS;
+}
+bool f16_d(int t) { return t == T_Q2_0 || t == T_IQ4_NL || t == T_Q8_0; }
bool gu_covered(int t) {
return t == T_IQ2_XXS || t == T_IQ2_XS || t == T_IQ2_S || t == T_IQ3_XXS || t == T_IQ3_S || t == T_IQ4_XS
+ #if !defined(__HIPCC__)
+ || (t == T_Q2_0 && dev_info().t75) // the canonical Q2_0 pack's gate/up: Turing opt-in only (sm_80+ keeps MMQ for it)
+ #endif
#if defined(__HIPCC__)
|| ((t == T_Q4_K || t == T_Q5_K) && kq_on()) // the gfx11 kernels only
#endif
@@@ -1243,17 -1186,10 +1342,20 @@@ bool d_covered(int t)
#endif
;
}
+// d_covered on this device: STRATA_PF_F16=1 also takes Q8_0 down on gfx11 without STRATA_PF_FUSED_KQ - the F16 kernel
+// has no minimum term, and its output already differs from the int8 kernels' (UD-IQ4_XS: the five Q8_0-down layers
+// would otherwise leave the fused path for MMQ).
+bool d_covered_on(int t, const DevInfo& d) {
+#if defined(__HIPCC__)
+ if (t == T_Q8_0 && !d.w12 && f16::requested()) return true;
+#endif
+ (void) d;
+ return d_covered(t);
+}
+ #if defined(__HIPCC__)
+ int host_astages() { return ASTAGES; } // (#2145 defines it for CUDA above; HIP keeps the one depth)
+ #endif
template <int T, bool GU>
void launch(int ww, unsigned grid, const Batch& b, const NativeGeom& g, const Tables& tb, const void* act,
const int32_t* src, void* out, float* dm, cudaStream_t s) {
@@@ -1300,9 -1236,9 +1402,10 @@@ bool native_supported(int gu_type, int
#endif
if (env < 0 && gu_type != T_IQ3_S && gu_type != T_IQ3_XXS) return false; // default: only these two
const DevInfo& d = dev_info();
+ if (!d.ok || !d_covered_on(d_type, d)) return false;
+ if (d.t75 && (gu_type != T_Q2_0 || d_type != T_Q2_0)) return false;
- if (env < 0) return d.ok && d.w12;
- return d.ok && (requested() || d.w12);
+ if (env < 0) return d.w12;
+ return requested() || d.w12;
}
void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream) {
diff --cc src/prefill/prefill.cpp
index 2357ea5e4,fe7a37dbe..17428925e
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -1384,8 -1224,10 +1385,12 @@@ bool Prefill::init(const core::WeightTa
m.f16_io = prompt_f16();
m.gemm.set_f16_io(m.f16_io);
set_act_f16(m.f16_io);
+ static const bool f16_range = std::getenv("STRATA_F16_RANGE") != nullptr; // what reaches FP16's range
+ if (f16_range) { set_f16_range(true); gemm_set_f16_range(true); }
+ // the hyper-connection's xn16 straight as the FP16 the tensor-core GEMM reads (its two readers then skip the
+ // conversion pass; the same values): STRATA_XN_F16=0 keeps the BF16 image
+ static const bool xn_env = [] { const char* e = std::getenv("STRATA_XN_F16"); return e == nullptr || e[0] != '0'; }();
+ m.xn_f16 = xn_env && !m.f16_io && m.gemm.bf16_converts_x();
}
if (!carve(T, &o)) {
size_t fb = 0, tb = 0;
@@@ -2251,26 -1992,10 +2256,26 @@@ bool native_proj(Gemm& gm, const core::
return true;
}
bool bf16_proj(Gemm& gm, const core::WeightRef* w, const uint16_t* X, float* Y, int64_t T, const std::string& name,
- std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, int64_t ldx = 0, bool x_f16 = false) {
+ std::string& err, int64_t ldy = 0, const uint16_t* X_lo = nullptr, int64_t ldx = 0,
- void* stream = nullptr) {
++ void* stream = nullptr, bool x_f16 = false) {
if (w->kind != core::WeightKind::Bf16InF32 || !w->data) { err = "prefill: " + name + " is not a resident BF16 tensor"; return false; }
- gm.bf16(X, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx, x_f16);
- if (X_lo) gm.bf16(X_lo, (const uint16_t*) w->data, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f);
+ const uint16_t* wb = (const uint16_t*) w->data;
+ if (w->q8_scale != nullptr) {
+ // STRATA_HC_REQ8: the GEMM takes bf16, so the int8 codes are expanded into one reused scratch on this stream
+ static uint16_t* scratch = nullptr;
+ static int64_t cap = 0;
+ if (w->elements > cap) {
+ if (scratch) cudaFree(scratch);
+ if (cudaMalloc((void**) &scratch, (size_t) w->elements * 2) != cudaSuccess) {
+ scratch = nullptr; cap = 0; err = "prefill: the STRATA_HC_REQ8 scratch"; return false;
+ }
+ cap = w->elements;
+ }
+ strata::kernels::hc_q8_to_bf16(w->data, w->q8_scale, scratch, (long long) w->elements, stream);
+ wb = scratch;
+ }
- gm.bf16(X, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx);
++ gm.bf16(X, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 0.0f, ldx, x_f16);
+ if (X_lo) gm.bf16(X_lo, wb, Y, T, w->ne1 > 0 ? w->ne1 : 1, w->ne0, ldy, 1.0f);
return true;
}
@@@ -2964,10 -2641,10 +2970,10 @@@ bool Prefill::run_impl(const int64_t* t
}
hcd = true;
}
- if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs)) return false;
- if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, xf16)) return false;
++ if (!hcd && !hdown && !bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs, xf16)) return false;
gr_silu(m.lo, m.lo16, T, m.cs, m.lo16_lo);
bool upmixed = false;
- if (hc_upmix() && T >= pf_switch_min_t() && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo &&
+ if (wu->q8_scale == nullptr && hc_upmix() && T >= std::max(pf_switch_min_t(), hc_upmix_min_t()) && !gr_unfused() && !m.lo16_lo && !m.mixed_bf_lo &&
wu->kind == core::WeightKind::Bf16InF32 && wu->data && wu->ne0 == LR && wu->ne1 == D) {
static int checks = [] { const char* e = std::getenv("STRATA_HC_UPMIX_CHECK"); return e ? std::atoi(e) : 0; }();
if (checks > 0) { // the default pair first, kept for the comparison
@@@ -2993,8 -2670,8 +2999,8 @@@
m.mixed, m.mixed_bf, m.mixed_h, T, m.cs);
}
}
- if (!upmixed && !bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo)) return false;
- if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, xf16)) return false;
+ if (!upmixed && !bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo, 0, m.cs)) return false;
- if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs)) return false;
++ if (!hcd && !bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo, ldx != D ? ldx : 0, m.cs, xf16)) return false;
if (upmixed) {
} else if (gr_unfused()) {
gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, m.cs, m.mixed_h, m.mixed_bf_lo);