Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
layer-major prefill: STRATA_HC_Q8 dequant, guards for APR/NVFP4/WMMA layers, QSA indexer calls of this tree (#3976)
Norihiro Nishimuradf7068b6be3e
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 7a8a8de2f..c3fca2d9a 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -5643,6 +5643,18 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why) {if (!mp.any || (int64_t) mp.layer.size() < g.n_layers) { why = "the experts are not on MMQ"; return 0; }for (int64_t l = 0; l < g.n_layers; ++l)if (!mp.layer[(size_t) l]) { why = "a layer's experts are not on MMQ"; return 0; }+ // 0.2.2's expert formats this path's MMQ pass does not carry (port, 2026-10-24): APR's Hadamard-folded layers,+ // NVFP4's tails and w4a4 activations, and IQ1_M layers that only the WMMA tiles run+ {+ const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();+ for (int64_t l = 0; l < g.n_layers; ++l) {+ if (l < (int64_t) mp.wmma_only.size() && mp.wmma_only[(size_t) l]) { why = "a layer needs the WMMA tiles"; return 0; }+ if (!lay.native) continue;+ const auto& f = lay.fmt[(size_t) l];+ if (f.had_gu || f.had_d) { why = "an APR (Hadamard-folded) layer"; return 0; }+ if (f.tail_off || mmq::fp4_activations(f.gu_type)) { why = "an NVFP4 layer"; return 0; }+ }+ }if (m.fused_bufs) { why = "the fused expert layout"; return 0; }if (!m.grp_gu || !m.grp_d || !m.Xq || !m.GU || !m.H || !m.Hq || !m.Dm || !m.ids_identity || !m.tok_dev) {why = "the MoE buffers are missing"; return 0;@@ -5821,12 +5833,12 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, ststd::fill(pf_job.begin(), pf_job.end(), -1);if (m.src == nullptr || !pf_on) return true;std::vector<Stager::Job> js;- const bool pdirect = m.src->prefill_direct();+ const bool pdirect = false; // (STRATA_PREFILL_DIRECT, #1670's other commit, is not ported)for (int32_t e = 0; e < NE; ++e) {if ((m.host_res && m.cache && m.host_res[(size_t) l * NE + e] >= 0) || m.src->pinned(l, e)) continue;pf_job[(size_t) e] = (int) js.size();if (pdirect || m.src->transient(l, e)) {- js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e});continue;}const uint8_t* b = m.src->blob_stable(l, e);@@ -5883,13 +5895,13 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stif (cnt[(size_t) e] == 0 && pf_job[(size_t) e] >= 0) m.stager->skip(pf_job[(size_t) e]);if (!pf_on && m.src != nullptr) { // STRATA_LM_PREFETCH=0: the routed experts only, started nowstd::vector<Stager::Job> js;- const bool pdirect = m.src->prefill_direct();+ const bool pdirect = false; // (STRATA_PREFILL_DIRECT, #1670's other commit, is not ported)for (size_t u = 0; u < uniq.size(); ++u) {const int32_t e = uniq[u];if (resident(e) || m.src->pinned(l, e)) continue;job_of[u] = (int) js.size();if (pdirect || m.src->transient(l, e)) {- js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e});continue;}const uint8_t* b = m.src->blob_stable(l, e);@@ -6288,6 +6300,16 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stconst core::WeightRef *wn = need(v, sn.c_str(), err), *wd = need(v, sd.c_str(), err),*wu = need(v, su.c_str(), err), *wi = need(v, si.c_str(), err);if (!wn || !wd || !wu || !wi) return false;+ // STRATA_HC_Q8 without a bf16 copy (0.2.2's run_impl): this path's GEMMs read bf16, so the matrices+ // are dequantized per half+ core::WeightRef wd_q8, wu_q8;+ if (!wd->data && wd->hc_q8 && !wu->data && wu->hc_q8) {+ wd_q8 = *wd; wu_q8 = *wu;+ wd_q8.data = core::NativeDense::hc_bf16_view(*wd, cs, 0, 0);+ wu_q8.data = core::NativeDense::hc_bf16_view(*wu, cs, 1, 0);+ if (!wd_q8.data || !wu_q8.data) { err = "prefill: the Q8_0 hyper-connection projections could not be dequantized"; return false; }+ wd = &wd_q8; wu = &wu_q8;+ }pt.mark(kPfHc, cs);if (gr_unfused()) gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, cs, m.xn16_lo);else if (!normed) gr_norm_rs(m.R, (const float*) wn->data, EPS, m.grs, m.xn16, T, cs, m.xn16_lo);@@ -6299,6 +6321,10 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stif (gr_unfused()) gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h, m.mixed_bf_lo);else gr_mix_r(m.R, m.grs, (const float*) wn->data, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h,m.mixed_bf_lo);+ if (wd_q8.data) { // the HC GEMMs that read the scratch are enqueued: the next user waits for them+ core::NativeDense::hc_bf16_release(cs, 0, 0);+ core::NativeDense::hc_bf16_release(cs, 1, 0);+ }if (half == 0 && !core::is_qsa_layer(g, l)) {// ======================= GDN (run_impl's) =======================const core::WeightRef *wqkv = need(v, "attn_qkv.weight", err), *wg = need(v, "attn_gate.weight", err),@@ -6392,7 +6418,7 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stif (st.kv_rot) strata::kernels::fwht256_inplace_cuda(m.q, T * 24, cs);rms_rows(m.q_idx, (const float*) wiqn->data, T * 4, 128, 128, EPS, cs);rope(m.q_idx, T, 4, 128, 512, p0, strata::kernels::rope_scaling(), cs);- const strata::kernels::QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos, st.idx_pooled_i8, st.idx_pooled_scale};+ const strata::kernels::QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos};pt.mark(kPfQsaIdx, cs);try {static const bool per_token = std::getenv("STRATA_INDEXER_PER_TOKEN") != nullptr;@@ -6415,10 +6441,10 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stconst int64_t active = (int64_t) m.steps_host[(size_t) ((q0 + nb - 1) * strata::kernels::kStepCount +strata::kernels::kStepNBid)] + 1;static const bool old_sel = std::getenv("STRATA_SELECT_OLD") != nullptr;- if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512,+ if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_dead, m.q_idx + q0 * 512,steps0, nb, m.max_blocks, s, m.sel_scores,cs, active))- strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512, steps0, nb,+ strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_dead, m.q_idx + q0 * 512, steps0, nb,m.max_blocks, s, m.sel_scores, cs, active);strata::kernels::qsa_block_topk(m.sel_scores, steps0, nb, m.max_blocks, m.cap, s,m.sel_ids + q0 * m.cap, cs, active);