Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Prefill fused IQ experts: opt-in 256-thread kernels STRATA_PF_FUSED_NT=256 and whole-layer launches STRATA_PF_BATCH=N (#3779, #3712)

linxuhaobdcc34dfa80f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9bbd87165..31d3664b6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2076,6 +2076,10 @@ if(STRATA_NATIVE_EXPERTS)
# layers on sm_80+ (77 below sm_80 and without a card)
add_test(NAME prefill_fused_iq_test_q2n COMMAND prefill_fused_iq_test "--only=Q2_0 / Q2_0" --unset-pf-fused)
set_tests_properties(prefill_fused_iq_test_q2n PROPERTIES ENVIRONMENT "STRATA_PF_FUSED_Q2N=1" SKIP_RETURN_CODE 77 TIMEOUT 600)
+ # the opt-in STRATA_PF_FUSED_NT=256 kernels (256 threads, two blocks a multiprocessor, two-pass weight loads) against the
+ # same references; the batches of the test reach them through the Batch overload of experts_native (77 below sm_80)
+ add_test(NAME prefill_fused_iq_test_nt256 COMMAND prefill_fused_iq_test)
+ set_tests_properties(prefill_fused_iq_test_nt256 PROPERTIES ENVIRONMENT "STRATA_PF_FUSED_NT=256" SKIP_RETURN_CODE 77 TIMEOUT 600)
# the MoE input quantized per token and scattered to its rows (mmq::quantize_scatter) against the per-row
# gather: the same q8_1 bytes for every MMQ layout (--bench: both at 2K, 8K and 32K tokens)
add_executable(prefill_mmq_scatter_test tests/cuda/prefill_mmq_scatter_test.cpp)
diff --git a/include/strata/prefill/moe_fused.hpp b/include/strata/prefill/moe_fused.hpp
index eef9e2dc9..47df71445 100644
--- a/include/strata/prefill/moe_fused.hpp
+++ b/include/strata/prefill/moe_fused.hpp
@@ -24,6 +24,14 @@ namespace strata::prefill::fused {
constexpr int kTileRows = 64;
/// Experts per launch: their blob pointers travel in the launch's parameters (1 KB).
constexpr int kMaxBatch = 128;
+/// The room of WideBatch::blob (below): a whole layer of 512 experts, which STRATA_PF_BATCH=512 (opt-in) launches at
+/// once on the native kernels' NT = 256 instances when all of them are resident (4 KB of kernel parameters: CUDA 12.1+,
+/// Volta and newer). HIP, gfx906 and the Pascal / Volta build (STRATA_EXPERIMENTAL_SM60): kMaxBatch.
+#if defined(__HIPCC__) || defined(STRATA_USE_HIP) || defined(STRATA_HIP_GFX906) || defined(STRATA_EXPERIMENTAL_SM60)
+constexpr int kBatchRoom = kMaxBatch;
+#else
+constexpr int kBatchRoom = 512;
+#endif
/// This build has the kernels (a CUDA build; HIP and builds without MMQ do not).
bool built();
@@ -57,6 +65,12 @@ struct Batch {
int e0 = 0, e1 = 0;
const uint8_t* blob[kMaxBatch] = {};
};
+/// Batch with room for kBatchRoom experts (e1 - e0 <= kBatchRoom): only the opt-in launches of STRATA_PF_BATCH above
+/// kMaxBatch on the native NT = 256 kernels (moe_fused_iq.hpp, native_wide()) take it; every other launch takes Batch.
+struct WideBatch {
+ int e0 = 0, e1 = 0;
+ const uint8_t* blob[kBatchRoom] = {};
+};
/// The batch's products: gate/up from `xa` (quantize_act of the layer's input, [tokens][2560]) through the rows
/// `src` of `group`, SwiGLU, H as int8 into `ha` (act_bytes(n, 640)), down into `dm` [n][2560] FP32 at the rows of
/// `group`. `n`: the layer's routed rows (tokens * k), which bounds the launch grid.
diff --git a/include/strata/prefill/moe_fused_iq.hpp b/include/strata/prefill/moe_fused_iq.hpp
index 7e63df9ba..a5ebd54f0 100644
--- a/include/strata/prefill/moe_fused_iq.hpp
+++ b/include/strata/prefill/moe_fused_iq.hpp
@@ -38,5 +38,11 @@ void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, v
/// through Alloc's 256-byte takes from a 256-aligned cache slot or cudaMalloc).
void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
const int32_t* src, void* ha, float* dm, void* stream);
+/// The NT = 256 kernels (STRATA_PF_FUSED_NT=256) are set up on the current device: experts_native takes a WideBatch
+/// (STRATA_PF_BATCH up to kBatchRoom experts a launch).
+bool native_wide();
+/// experts_native() of a WideBatch, on the NT = 256 kernels; only when native_wide().
+void experts_native(const WideBatch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+ const int32_t* src, void* ha, float* dm, void* stream);
} // namespace strata::prefill::fused
diff --git a/src/prefill/moe_fused_iq.cu b/src/prefill/moe_fused_iq.cu
index d239972f8..de40847c4 100644
--- a/src/prefill/moe_fused_iq.cu
+++ b/src/prefill/moe_fused_iq.cu
@@ -44,6 +44,7 @@
#include <cstdlib>
#include <cstring>
#include <mutex>
+#include <type_traits>
namespace strata::prefill::fused {
namespace {
@@ -63,7 +64,7 @@ constexpr float MAGICF = 12582912.0f;
// per launch by the rows per expert (pick_ww)
constexpr int THREADS = 512;
constexpr int WLD = 80; // bytes per decoded weight row of a stage: 64 int8 + 16 (no bank conflicts)
-__host__ __device__ constexpr int tile_rows(int ww) { return 256 / ww; }
+__host__ __device__ constexpr int tile_rows(int ww, int nt = THREADS) { return nt / 32 / ww * 16; }
__host__ __device__ constexpr int weight_rows(int ww) { return 64 * ww; }
constexpr int ASTAGES = 4; // cp.async depth of the activations (64 values of K a stage)
// Turing: 2 (its copies are plain loads, so depth buys nothing, and 4 does not fit its 64 KB of shared memory)
@@ -107,9 +108,9 @@ __host__ __device__ constexpr int grid_bytes(int t) {
return t == T_IQ2_XXS ? 256 * 8 : t == T_IQ2_XS ? 512 * 8 : t == T_IQ2_S ? 1024 * 8 : t == T_IQ3_XXS ? 256 * 4
: t == T_IQ3_S ? 512 * 4 : 0;
}
-__host__ __device__ constexpr size_t smem_bytes(int t, int ww, int astages = ASTAGES) {
- return (size_t) 2 * weight_rows(ww) * (WLD + 16) + (size_t) astages * tile_rows(ww) * AB +
- (size_t) tile_rows(ww) * 4 + grid_bytes(t);
+__host__ __device__ constexpr size_t smem_bytes(int t, int ww, int astages = ASTAGES, int nt = THREADS) {
+ return (size_t) 2 * weight_rows(ww) * (WLD + 16) + (size_t) astages * tile_rows(ww, nt) * AB +
+ (size_t) tile_rows(ww, nt) * 4 + grid_bytes(t);
}
// moe_fused.cu's grouping tables (the same layout: fused::group writes them)
@@ -340,6 +341,58 @@ template <int T, int NW> __device__ __forceinline__ void load_unit(const uint8_t
}
}
+// load_unit's bytes in two passes (native_kernel's opt-in NT = 256 instances only, CUDA): `issue` loads the 16-bit and
+// 8-bit pieces into registers and `take` assembles the words from them a stage later, so no instruction waits on the
+// loads in the stage that issues them (in one pass the shifts and ORs right after the loads stalled every stage on a
+// memory round trip). The same bytes in the same words as load_unit.
+constexpr int NP = 12; // pieces: at most IQ4_XS's 10 halves and a byte
+struct LdIssue {
+ uint32_t* pc; int k;
+ __device__ __forceinline__ uint32_t h(const uint8_t* p) { pc[k++] = *(const uint16_t*) p; return 0; }
+ __device__ __forceinline__ uint32_t b(const uint8_t* p) { pc[k++] = *p; return 0; }
+};
+struct LdTake {
+ const uint32_t* pc; int k;
+ __device__ __forceinline__ uint32_t h(const uint8_t*) { return pc[k++]; }
+ __device__ __forceinline__ uint32_t b(const uint8_t*) { return pc[k++]; }
+};
+template <class L> __device__ __forceinline__ uint32_t lw(L& l, const uint8_t* p) {
+ const uint32_t lo = l.h(p);
+ const uint32_t hi = l.h(p + 2);
+ return lo | (hi << 16);
+}
+template <int T, class L, int NW> __device__ __forceinline__ void load_unit_l(L& l, const uint8_t* bp, int ib, uint32_t (&w)[NW]) {
+ static_assert(NW >= raw_words(T), "raw words");
+ if constexpr (T == T_IQ2_XXS || T == T_Q2_0) {
+ w[0] = lw(l, bp + 2 + 8 * ib); w[1] = lw(l, bp + 6 + 8 * ib); w[2] = l.h(bp);
+ } else if constexpr (T == T_IQ2_XS) {
+ w[0] = lw(l, bp + 2 + 8 * ib); w[1] = lw(l, bp + 6 + 8 * ib);
+ const uint32_t d = l.h(bp); const uint32_t s = l.b(bp + 66 + ib);
+ w[2] = d | (s << 16);
+ } else if constexpr (T == T_IQ2_S) {
+ w[0] = lw(l, bp + 2 + 4 * ib); w[1] = lw(l, bp + 34 + 4 * ib);
+ const uint32_t d = l.h(bp); const uint32_t qh = l.b(bp + 66 + ib); const uint32_t sc = l.b(bp + 74 + ib);
+ w[2] = d | (qh << 16) | (sc << 24);
+ } else if constexpr (T == T_IQ3_XXS) {
+ w[0] = lw(l, bp + 2 + 8 * ib); w[1] = lw(l, bp + 6 + 8 * ib); w[2] = lw(l, bp + 66 + 4 * ib); w[3] = l.h(bp);
+ } else if constexpr (T == T_IQ3_S) {
+ w[0] = lw(l, bp + 2 + 8 * ib); w[1] = lw(l, bp + 6 + 8 * ib); w[2] = lw(l, bp + 74 + 4 * ib);
+ const uint32_t d = l.h(bp); const uint32_t qh = l.b(bp + 66 + ib); const uint32_t sc = l.b(bp + 106 + ib / 2);
+ w[3] = d | (qh << 16) | (((sc >> (4 * (ib & 1))) & 15) << 24);
+ } else if constexpr (T == T_IQ4_XS) {
+#pragma unroll
+ for (int k = 0; k < 4; ++k) w[k] = lw(l, bp + 8 + 16 * ib + 4 * k);
+ const uint32_t lo = l.b(bp + 4 + ib / 2); const uint32_t hi = l.h(bp + 2); const uint32_t d = l.h(bp);
+ const uint32_t ls = ((lo >> (4 * (ib & 1))) & 15) | (((hi >> (2 * ib)) & 3) << 4);
+ w[4] = d | (ls << 16);
+ } else { // IQ4_NL
+ static_assert(T == T_IQ4_NL, "the CUDA native formats");
+#pragma unroll
+ for (int k = 0; k < 4; ++k) w[k] = lw(l, bp + 2 + 4 * k);
+ w[4] = l.h(bp);
+ }
+}
+
// The sub-block as 32 int8 (q[0..7], natural order) and its scales (s0: values 0-15, s1: 16-31).
template <int T, int NW>
__device__ __forceinline__ void convert(const uint32_t (&w)[NW], const uint8_t* grid, const uint32_t (&kv)[4],
@@ -475,13 +528,20 @@ template <int T> __device__ __forceinline__ const void* grid_src() {
// tokens' activations, SwiGLU, H to int8 per 32 features into `out`. !GU: down rows (format WT) against H, FP32 into
// `dm`. Warp (wf, wt): weight rows 64 wf.. as four m16 tiles (GU: a tile = 8 features, gate rows as mma rows 0-7 and
// up rows as 8-15, so a lane holds gate and up of one feature), routed rows 16 wt.. as two n8.
-template <int WT, bool GU, int WW>
-__global__ void __launch_bounds__(THREADS, 1)
-native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_t* __restrict__ act,
+// NT (STRATA_PF_FUSED_NT=256, sm_80+ opt-in): 256 threads, two blocks a multiprocessor - one block's products run while
+// the other waits at a barrier. WW 2 only (its 2 WR = 256 decode units are the block's threads); a work item is 128
+// weight rows x one 64-row tile. Every output is the same thread's sum in the same order: the same bits. The NT = 256
+// instances take a WideBatch (STRATA_PF_BATCH up to kBatchRoom experts a launch), the default ones Batch.
+template <int NT> using BatchOf = std::conditional_t<NT == THREADS, Batch, WideBatch>;
+template <int WT, bool GU, int WW, int NT = THREADS>
+__global__ void __launch_bounds__(NT, NT == THREADS ? 1 : 2)
+native_kernel(const BatchOf<NT> b, const NativeGeom geo, const Tables tb, const uint8_t* __restrict__ act,
const int32_t* __restrict__ src, uint8_t* __restrict__ out, float* __restrict__ dm) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
- constexpr int TR = tile_rows(WW), WR = weight_rows(WW);
+ constexpr int TR = tile_rows(WW, NT), WR = weight_rows(WW);
+ static_assert(2 * WR <= NT, "every decode unit is a thread of the block");
constexpr int WT_BYTES = WR * WLD, WS_FLOATS = WR * 4, ACT_STAGE = TR * AB;
+ constexpr bool TWO_PASS = NT != THREADS; // the two-pass loads (load_unit_l) only in the opt-in NT = 256 kernels
constexpr int NS = (GU ? GU_ROWS_K : D_ROWS_K) / 64; // 64-value stages along K
constexpr int NFB = (GU ? 1280 : 2560) / WR; // weight-row blocks per tile
constexpr int ACT_LD = NS * AB; // bytes per activation row: 3200 (a token), 800 (a row's H)
@@ -498,7 +558,7 @@ native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_
const int wf = warp % WW, nb0 = 16 * (warp / WW);
if constexpr (grid_bytes(WT) > 0) {
const uint32_t* gs = (const uint32_t*) grid_src<WT>();
- for (int i = tid; i < grid_bytes(WT) / 4; i += THREADS) ((uint32_t*) sgrid)[i] = gs[i];
+ for (int i = tid; i < grid_bytes(WT) / 4; i += NT) ((uint32_t*) sgrid)[i] = gs[i];
}
uint32_t kv[4] = {0, 0, 0, 0};
if constexpr (WT == T_IQ4_XS || WT == T_IQ4_NL) {
@@ -555,7 +615,7 @@ native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_
// of a partial one are never written (no reduction mixes columns)
auto load_act = [&](int s) {
uint8_t* st = stages + (s % ASTAGES_DEV) * ACT_STAGE;
- for (int c = tid; c < TR * 5; c += THREADS) {
+ for (int c = tid; c < TR * 5; c += NT) {
const int r = c / 5, q = c % 5;
if (r < nrows) cp16(st + r * AB + q * 16, act + (size_t) srow[r] * ACT_LD + s * AB + q * 16);
}
@@ -576,7 +636,11 @@ native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_
cp_commit();
}
uint32_t raw[5] = {0, 0, 0, 0, 0};
+ uint32_t pc[NP]; // stage s + 2's pieces in flight (load_unit_l)
+ auto issue = [&](int s) { LdIssue l{pc, 0}; uint32_t w[5]; load_unit_l<WT>(l, unit(s), sub(s), w); };
+ auto take = [&](int s) { LdTake l{pc, 0}; load_unit_l<WT>(l, unit(s), sub(s), raw); };
if (dec) load_unit<WT>(unit(0), sub(0), raw);
+ if (dec && TWO_PASS && 2 < NS) issue(2);
put(raw, 0);
if (dec) load_unit<WT>(unit(1), sub(1), raw);
// a warp whose rows are all past the tile's end only takes part in the loads
@@ -672,7 +736,14 @@ native_kernel(const Batch b, const NativeGeom geo, const Tables tb, const uint8_
// 5.2 / 9.5 / 17.5 ms -> 6.0 / 10.2 / 18.3.)
if (s + 1 < NS) {
put(raw, (s + 1) & 1);
- if (dec && s + 2 < NS) load_unit<WT>(unit(s + 2), sub(s + 2), raw);
+ if (dec && s + 2 < NS) {
+ if constexpr (TWO_PASS) {
+ take(s + 2);
+ if (s + 3 < NS) issue(s + 3);
+ } else {
+ load_unit<WT>(unit(s + 2), sub(s + 2), raw);
+ }
+ }
}
}
if (!on0) continue;
@@ -1191,6 +1262,7 @@ struct DevInfo {
bool q2n = false; // sm_80+ with STRATA_PF_FUSED_Q2N=1: the native Q2_0 pack's Q2_0 gate/up is set up as well (opt-in)
bool s120 = false; // compute capability 12.0 (sm120_series.hpp): the default for the formats measured there
int sms = 0, occ = 1;
+ int occ_nt = 0; // blocks per multiprocessor of the NT = 256 kernels (0: not set up)
};
std::mutex g_mu;
DevInfo g_dev[32];
@@ -1208,20 +1280,27 @@ int host_astages() {
}
return per[dev];
}
-template <int T, bool GU, int WW> bool setup_ww(int& occ) {
+template <int T, bool GU, int WW, int NT = THREADS> bool setup_ww(int& occ) {
cudaFuncAttributes fa{};
- if (cudaFuncGetAttributes(&fa, native_kernel<T, GU, WW>) != cudaSuccess || fa.ptxVersion < 75) return false;
- if (cudaFuncSetAttribute(native_kernel<T, GU, WW>, cudaFuncAttributeMaxDynamicSharedMemorySize,
- (int) smem_bytes(T, WW, host_astages())) != cudaSuccess)
+ if (cudaFuncGetAttributes(&fa, native_kernel<T, GU, WW, NT>) != cudaSuccess || fa.ptxVersion < 75) return false;
+ const size_t sm = smem_bytes(T, WW, host_astages(), NT);
+ if (cudaFuncSetAttribute(native_kernel<T, GU, WW, NT>, cudaFuncAttributeMaxDynamicSharedMemorySize, (int) sm) != cudaSuccess)
return false;
int o = 0;
- if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&o, native_kernel<T, GU, WW>, THREADS, smem_bytes(T, WW, host_astages())) !=
- cudaSuccess || o < 1)
+ if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&o, native_kernel<T, GU, WW, NT>, NT, sm) != cudaSuccess || o < 1)
return false;
occ = std::min(occ, o);
return true;
}
template <int T, bool GU> bool setup_one(int& occ) { return setup_ww<T, GU, 4>(occ) && setup_ww<T, GU, 2>(occ); }
+// STRATA_PF_FUSED_NT=256 (sm_80+, opt-in): the 256-thread kernels (native_kernel's NT)
+bool nt256_requested() {
+ static const bool on = [] { const char* v = std::getenv("STRATA_PF_FUSED_NT"); return v != nullptr && std::atoi(v) == 256; }();
+ return on;
+}
+template <int T, bool GU> bool setup_nt(int& occ) {
+ return setup_ww<T, GU, 2, 256>(occ);
+}
#endif
#if defined(__HIPCC__)
@@ -1316,6 +1395,15 @@ const DevInfo& dev_info() {
static const bool q2n = [] { const char* v = std::getenv("STRATA_PF_FUSED_Q2N"); return v != nullptr && v[0] == '1'; }();
if (int o2 = occ; q2n && d.ok && setup_one<T_Q2_0, true>(o2)) { d.q2n = true; occ = o2; }
d.occ = d.ok ? occ : 1;
+ if (d.ok && nt256_requested()) {
+ int o2 = 1 << 20;
+ const bool ok2 = setup_nt<T_IQ2_XXS, true>(o2) && setup_nt<T_IQ2_XS, true>(o2) && setup_nt<T_IQ2_S, true>(o2) &&
+ setup_nt<T_IQ3_XXS, true>(o2) && setup_nt<T_IQ3_S, true>(o2) && setup_nt<T_IQ4_XS, true>(o2) &&
+ setup_nt<T_Q2_0, false>(o2) && setup_nt<T_IQ4_NL, false>(o2) &&
+ (!d.q2n || setup_nt<T_Q2_0, true>(o2)); // (the Q2_0 gate/up of STRATA_PF_FUSED_Q2N=1 too)
+ d.occ_nt = ok2 ? o2 : 0;
+ if (!ok2) std::fprintf(stderr, "prefill fused experts (native): STRATA_PF_FUSED_NT=256 is not available here\n");
+ }
cudaGetLastError();
#endif
return d;
@@ -1375,6 +1463,13 @@ void launch(int ww, unsigned grid, const Batch& b, const NativeGeom& g, const Ta
if (ww == 4) native_kernel<T, GU, 4><<<grid, THREADS, smem_bytes(T, 4, host_astages()), s>>>(b, g, tb, a, src, o, dm);
else native_kernel<T, GU, 2><<<grid, THREADS, smem_bytes(T, 2, host_astages()), s>>>(b, g, tb, a, src, o, dm);
}
+// the NT = 256 kernels (WW 2)
+template <int T, bool GU>
+void launch(unsigned grid, const WideBatch& b, const NativeGeom& g, const Tables& tb, const void* act, const int32_t* src,
+ void* out, float* dm, cudaStream_t s) {
+ native_kernel<T, GU, 2, 256><<<grid, 256, smem_bytes(T, 2, host_astages(), 256), s>>>(b, g, tb, (const uint8_t*) act,
+ src, (uint8_t*) out, dm);
+}
// The work item's shape for a layer of `n` routed rows over `n_expert` experts: 128 routed rows when an expert has
// about one such tile (56 to 112 rows on average), else 64. Measured on the RTX 5070 (tests/cuda/prefill_fused_iq_test,
@@ -1530,6 +1625,14 @@ void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n
return;
}
#endif
+ if (d.occ_nt > 0) { // STRATA_PF_FUSED_NT=256: the NT = 256 kernels, which take a WideBatch
+ WideBatch w;
+ w.e0 = b.e0;
+ w.e1 = b.e1;
+ std::copy(b.blob, b.blob + (b.e1 - b.e0), w.blob);
+ experts_native(w, g, n_expert, n, scratch, xa, src, ha, dm, stream);
+ return;
+ }
const int ww = pick_ww(n, n_expert);
// the most 64-row tiles the batch can have (every row in it, plus a partial tile per expert) - the items of the
// ones inside a larger item end at once
@@ -1550,4 +1653,35 @@ void experts_native(const Batch& b, const NativeGeom& g, int n_expert, int64_t n
ck(cudaGetLastError(), "experts_native");
}
+bool native_wide() { return dev_info().occ_nt > 0; }
+
+void experts_native(const WideBatch& b, const NativeGeom& g, int n_expert, int64_t n, const void* scratch, const void* xa,
+ const int32_t* src, void* ha, float* dm, void* stream) {
+ if (b.e1 <= b.e0 || n <= 0) return;
+ const DevInfo& d = dev_info();
+ if (d.occ_nt <= 0 || !gu_covered(g.gu_type) || !d_covered_on(g.d_type, d)) {
+ std::fprintf(stderr, "prefill fused experts (native, NT = 256): types %d / %d are not covered here\n", g.gu_type,
+ g.d_type);
+ std::exit(1);
+ }
+ const cudaStream_t s = (cudaStream_t) stream;
+ const Tables tb = tables(const_cast<void*>(scratch), n_expert);
+ // WW 2 (128 x 64 items), as many blocks as are resident; the tiles as in experts_native(const Batch&)
+ const int64_t tiles = (n + kTileRows - 1) / kTileRows + (b.e1 - b.e0);
+ const unsigned g_gu = (unsigned) std::min<int64_t>(tiles * (1280 / weight_rows(2)), (int64_t) d.sms * d.occ_nt);
+ const unsigned g_d = (unsigned) std::min<int64_t>(tiles * (2560 / weight_rows(2)), (int64_t) d.sms * d.occ_nt);
+ switch (g.gu_type) {
+ case T_IQ2_XXS: launch<T_IQ2_XXS, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ case T_IQ2_XS: launch<T_IQ2_XS, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ case T_IQ2_S: launch<T_IQ2_S, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ case T_IQ3_XXS: launch<T_IQ3_XXS, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ case T_IQ3_S: launch<T_IQ3_S, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ case T_Q2_0: launch<T_Q2_0, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ default: launch<T_IQ4_XS, true>(g_gu, b, g, tb, xa, src, ha, nullptr, s); break;
+ }
+ if (g.d_type == T_Q2_0) launch<T_Q2_0, false>(g_d, b, g, tb, ha, src, nullptr, dm, s);
+ else launch<T_IQ4_NL, false>(g_d, b, g, tb, ha, src, nullptr, dm, s);
+ ck(cudaGetLastError(), "experts_native (NT = 256)");
+}
+
} // namespace strata::prefill::fused
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 8b08b511c..a96c2a03d 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -63,6 +63,7 @@
#include <cstring>
#include <future>
#include <memory>
+#include <optional>
#include <mutex>
#include <thread>
#include <utility>
@@ -115,6 +116,9 @@ bool native_supported(int, int) { return false; }
void quantize_act_native(const float*, int64_t, int64_t, void*, void*) {}
void experts_native(const Batch&, const NativeGeom&, int, int64_t, const void*, const void*, const int32_t*, void*,
float*, void*) {}
+bool native_wide() { return false; }
+void experts_native(const WideBatch&, const NativeGeom&, int, int64_t, const void*, const void*, const int32_t*, void*,
+ float*, void*) {}
} // namespace strata::prefill::fused
#endif
@@ -3962,23 +3966,36 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (fused_nat) fused::quantize_act_native(m.mixed, T, N, m.Xq, m.cs);
else fused::quantize_act(m.mixed, T, N, m.Xq, m.cs);
fused::group(m.ids, T * K, (int) K, (int) m.g->n_expert, m.GU, m.slot_dev, m.src_dev, m.cs);
- // launches over the experts in id order, each at most kMaxBatch experts of which at most a
- // third of the ring streamed: the next batch's blobs arrive while one computes. An expert
- // the routing did not pick has no tiles; its ring slot is given back with its batch.
+ // launches over the experts in id order, each at most kMaxBatch experts (STRATA_PF_BATCH: below)
+ // of which at most a third of the ring streamed: the next batch's blobs arrive while one computes.
+ // An expert the routing did not pick has no tiles; its ring slot is given back with its batch.
const size_t per = (size_t) std::max(1, m.ring / 3);
size_t k = seq_start[(size_t) l];
const size_t kend = seq_start[(size_t) l + 1];
+ // STRATA_PF_BATCH=N (opt-in): experts per launch (the same rows, so the same bits; a launch is one
+ // persistent grid, so fewer launches leave fewer grid tails) - at most kMaxBatch, or up to
+ // kBatchRoom in a WideBatch on the native NT = 256 kernels (STRATA_PF_FUSED_NT=256)
+ static const int pf_batch = [] {
+ const char* v = std::getenv("STRATA_PF_BATCH");
+ const int n = v ? std::atoi(v) : 0;
+ return n > 0 ? std::min(n, fused::kBatchRoom) : fused::kMaxBatch;
+ }();
+ const bool wide = fused_nat && pf_batch > fused::kMaxBatch && fused::native_wide();
+ const int per_launch = wide ? pf_batch : std::min(pf_batch, fused::kMaxBatch);
for (int32_t e = 0; e < m.g->n_expert;) {
fused::Batch b;
+ std::optional<fused::WideBatch> wb; // wide: the launch's pointers (b keeps e0 / e1)
+ if (wide) wb.emplace();
+ const uint8_t** blob = wb ? wb->blob : b.blob;
b.e0 = e;
const size_t k0 = k;
- for (; e < m.g->n_expert && e - b.e0 < fused::kMaxBatch; ++e) {
+ for (; e < m.g->n_expert && e - b.e0 < per_launch; ++e) {
if (k < kend && seq[k].e == e) {
if (k - k0 == per) break;
- b.blob[e - b.e0] = m.stage_dev[k % (size_t) m.ring];
+ blob[e - b.e0] = m.stage_dev[k % (size_t) m.ring];
++k;
} else { // not streamed: resident (the walk streams all others)
- b.blob[e - b.e0] = m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]);
+ blob[e - b.e0] = m.cache->device_slot(m.host_res[(size_t) l * m.g->n_expert + e]);
// counted whether routed or not (the routing stays on the GPU): at a streamed
// chunk's size (>= 1024 tokens x 10 of 512) nearly every expert is routed
++stats_.experts_resident;
@@ -3995,8 +4012,15 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (fused_nat) {
const auto& f = lay.fmt[(size_t) l];
const fused::NativeGeom ng{f.gu_type, f.d_type, f.gu_row, f.d_row, f.up_off, f.down_off};
- fused::experts_native(b, ng, (int) m.g->n_expert, T * K, m.GU, m.Xq, m.src_dev, m.H,
- m.Dm, m.cs);
+ if (wb) {
+ wb->e0 = b.e0;
+ wb->e1 = b.e1;
+ fused::experts_native(*wb, ng, (int) m.g->n_expert, T * K, m.GU, m.Xq, m.src_dev,
+ m.H, m.Dm, m.cs);
+ } else {
+ fused::experts_native(b, ng, (int) m.g->n_expert, T * K, m.GU, m.Xq, m.src_dev, m.H,
+ m.Dm, m.cs);
+ }
} else {
fused::experts(b, (int) m.g->n_expert, T * K, m.GU, m.Xq, m.src_dev, m.H, m.Dm, m.cs);
}