Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-146-int (b7505b72a, 0.1.46 engine: Volta perf PRs + QSA_WARP sm<75 fix) into f-split-146

Niko1221a6759787c3e4
diff --cc src/prefill/prefill.cpp
index 56ae3b358,bceacbda3..5fee9fe98
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -3870,22 -3847,15 +3915,27 @@@ bool Prefill::run_impl(const int64_t* t
const size_t mmq_gub = use_mmq ? mmq::matrix_bytes(mmq_gt, 1280, N) : 0;
const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;
pt.mark(kPfGather, cs);
+ // the routed experts' input: `mixed`, or (APR) its rotated copy, which the peer / the layer
+ // split's helper also take (one rotation for both cards; `mixed` itself feeds the shared expert)
+ const float* xsrc = m.mixed;
if (use_mmq) {
- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`:
- // each token quantized once and written to its K rows (slot_dev, the inverse of src_dev).
+ // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed` (APR:
+ // from its rotated copy `Xr`): each token quantized once and written to its K rows
+ // (slot_dev, the inverse of src_dev).
// NVFP4 w4a4 activations carry a per-row scale (Xscale): that mode keeps the gather.
- if (had_gu) {
- strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);
- xsrc = m.Xr;
- if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows (pf_wmma)
- else if (mmq::fp4_activations(mmq_gt))
- mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
- else
- mmq::quantize_scatter(m.mixed, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
++ if (wmma_l) { // FP16 rows (pf_wmma); APR: Hadamard-folded gate/up take H, rotated in fp16 storage as the FP16 path does
++ gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);
++ if (had_gu) strata::kernels::hadamard_rows_f16(m.Xs, T * K, N, had_block, had_sx, m.cs);
++ } else {
++ if (had_gu) {
++ strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);
++ xsrc = m.Xr;
++ }
++ if (mmq::fp4_activations(mmq_gt))
++ mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
++ else
++ mmq::quantize_scatter(xsrc, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
+ }
- if (mmq::fp4_activations(mmq_gt))
- mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);
- else
- mmq::quantize_scatter(xsrc, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);
// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down
// reads the group's own quantized H)
const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;
@@@ -4394,6 -4356,19 +4444,21 @@@
int64_t maxr = 0;
for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, m.cnt[(size_t) order[i]]);
pt.mark(kPfGemmGU, cs);
+ if (wmma_l) { // pf_wmma: gate/up, SwiGLU to FP16, down - the group in 2 tile launches
+ const int32_t* dnb = m.bounds_dev + n + 1 + g * (MMQ_GROUP + 1);
+ const bool ok1 = strata::kernels::gemm_iq_f16_grouped(
+ mmq_gt, m.grp_gu, mmq_gub, 1280, (int) N, m.Xs, (int) N, m.bounds_dev + j0, ngx,
+ (int) maxr, m.GU, 1280, m.cs);
+ swiglu_split_f16(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, 640, !lay.native, m.cs);
++ if (had_d) // APR: folded down weights take H (s * h), rotated in fp16 storage as the FP16 path does
++ strata::kernels::hadamard_rows_f16(m.Hh + r0 * 640, nr, 640, had_block, had_sh, m.cs);
+ pt.mark(kPfGemmD, cs);
+ const bool ok2 = strata::kernels::gemm_iq_f16_grouped(
+ mmq_dt, m.grp_d, mmq_db, (int) N, 640, m.Hh + r0 * 640, 640, dnb, ngx, (int) maxr,
+ m.Dm + r0 * N, (int) N, m.cs);
+ if (!ok1 || !ok2) { err = "prefill: the WMMA expert tiles could not launch"; return false; }
+ return true;
+ }
if (direct) {
// one launch: the direct experts' guards (their down over-read) and the zeroed tail
// after the last gathered expert (see MMQ_TAIL)