Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-146-int (b7505b72a, 0.1.46 engine: Volta perf PRs + QSA_WARP sm<75 fix) into f-split-146
Niko1221a6759787c3e4
diff --cc src/prefill/prefill.cppindex 56ae3b358,bceacbda3..5fee9fe98--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@@ -3870,22 -3847,15 +3915,27 @@@ bool Prefill::run_impl(const int64_t* tconst size_t mmq_gub = use_mmq ? mmq::matrix_bytes(mmq_gt, 1280, N) : 0;const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;pt.mark(kPfGather, cs);+ // the routed experts' input: `mixed`, or (APR) its rotated copy, which the peer / the layer+ // split's helper also take (one rotation for both cards; `mixed` itself feeds the shared expert)+ const float* xsrc = m.mixed;if (use_mmq) {- // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`:- // each token quantized once and written to its K rows (slot_dev, the inverse of src_dev).+ // step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed` (APR:+ // from its rotated copy `Xr`): each token quantized once and written to its K rows+ // (slot_dev, the inverse of src_dev).// NVFP4 w4a4 activations carry a per-row scale (Xscale): that mode keeps the gather.- if (had_gu) {- strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);- xsrc = m.Xr;- if (wmma_l) gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs); // FP16 rows (pf_wmma)- else if (mmq::fp4_activations(mmq_gt))- mmq::quantize(m.mixed, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);- else- mmq::quantize_scatter(m.mixed, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);++ if (wmma_l) { // FP16 rows (pf_wmma); APR: Hadamard-folded gate/up take H, rotated in fp16 storage as the FP16 path does++ gather_rows16(m.mixed_h, m.src_dev, m.Xs, T * K, N, m.cs);++ if (had_gu) strata::kernels::hadamard_rows_f16(m.Xs, T * K, N, had_block, had_sx, m.cs);++ } else {++ if (had_gu) {++ strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);++ xsrc = m.Xr;++ }++ if (mmq::fp4_activations(mmq_gt))++ mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);++ else++ mmq::quantize_scatter(xsrc, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);+ }- if (mmq::fp4_activations(mmq_gt))- mmq::quantize(xsrc, m.src_dev, m.Xq, mmq_gt, N, N, T * K, m.cs, m.Xscale);- else- mmq::quantize_scatter(xsrc, m.slot_dev, m.src_dev, m.Xq, mmq_gt, N, N, T, (int) K, m.cs);// each group's rows: absolute bounds (gate/up reads the layer's rows), relative ones (down// reads the group's own quantized H)const size_t n = order.size(), ng = (n + MMQ_GROUP - 1) / MMQ_GROUP;@@@ -4394,6 -4356,19 +4444,21 @@@int64_t maxr = 0;for (size_t i = j0; i <= j; ++i) maxr = std::max<int64_t>(maxr, m.cnt[(size_t) order[i]]);pt.mark(kPfGemmGU, cs);+ if (wmma_l) { // pf_wmma: gate/up, SwiGLU to FP16, down - the group in 2 tile launches+ const int32_t* dnb = m.bounds_dev + n + 1 + g * (MMQ_GROUP + 1);+ const bool ok1 = strata::kernels::gemm_iq_f16_grouped(+ mmq_gt, m.grp_gu, mmq_gub, 1280, (int) N, m.Xs, (int) N, m.bounds_dev + j0, ngx,+ (int) maxr, m.GU, 1280, m.cs);+ swiglu_split_f16(m.GU + r0 * 1280, m.Hh + r0 * 640, nr, 640, !lay.native, m.cs);++ if (had_d) // APR: folded down weights take H (s * h), rotated in fp16 storage as the FP16 path does++ strata::kernels::hadamard_rows_f16(m.Hh + r0 * 640, nr, 640, had_block, had_sh, m.cs);+ pt.mark(kPfGemmD, cs);+ const bool ok2 = strata::kernels::gemm_iq_f16_grouped(+ mmq_dt, m.grp_d, mmq_db, (int) N, 640, m.Hh + r0 * 640, 640, dnb, ngx, (int) maxr,+ m.Dm + r0 * N, (int) N, m.cs);+ if (!ok1 || !ok2) { err = "prefill: the WMMA expert tiles could not launch"; return false; }+ return true;+ }if (direct) {// one launch: the direct experts' guards (their down over-read) and the zeroed tail// after the last gathered expert (see MMQ_TAIL)