Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

APR port: multi-GPU prompt offload (peer, layer split helper) for Hadamard-folded packs

Laurent Zuijdwijk28651a4079fd
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 8d14fe46c..4ea168e88 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -756,6 +756,9 @@ struct PeerPrefill {
int32_t* host_pair = nullptr;
std::vector<int32_t> bounds_host;
core::PinnedRing up_ring; // #1996: this device's pageable uploads (bounds, routed rows) go up from pinned slots
+ // APR (prism.hadamard): per layer, this device's copy of the down projection's sign vector (nullptr: identity or
+ // not folded). The gate/up input arrives already rotated (the primary sends its rotated copy, m.Xr).
+ std::vector<const float*> had_sh;
std::vector<void*> owned;
int64_t layers = 0, experts = 0, rows = 0, over_cap = 0; // stats
~PeerPrefill() {
@@ -1733,14 +1736,24 @@ double split_help_frac(int64_t T) {
}
} // namespace
+namespace {
+// APR: the down projections' sign vectors on the current device (the peer's or the helper's), one per layer, made at
+// setup (cudaMalloc) so a prompt chunk only looks them up.
+void peer_had_signs(PeerPrefill& P) {
+ const auto& lay = strata::kernels::cpu::expert_layout();
+ P.had_sh.assign(lay.native ? lay.fmt.size() : 0, nullptr);
+ for (size_t l = 0; l < P.had_sh.size(); ++l)
+ if (lay.fmt[l].had_d)
+ P.had_sh[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);
+}
+} // namespace
+
bool Prefill::set_stage_helper(Prefill* helper, std::string& err) {
Impl& m = *impl_;
m.help_pp.reset();
if (helper == nullptr || helper == this || !split_help_env()) return true;
const Impl& h = *helper->impl_;
if (m.device < 0 || h.device < 0 || h.device == m.device || !mmq_plan().any) return true;
- // APR: the helper's MMQ share does not rotate the activations of Hadamard-folded experts; not used for those packs
- if (strata::kernels::cpu::hadamard_spec().any()) return true;
auto pp = std::make_unique<PeerPrefill>();
pp->peer = nullptr; // stream-only: it holds no experts, it streams a share of this stage's
pp->compact = true;
@@ -1768,6 +1781,7 @@ bool Prefill::set_stage_helper(Prefill* helper, std::string& err) {
for (int i = 0; ok && i < kSplitHelpRing; ++i)
ok = cudaEventCreateWithFlags(&pp->pcopied[(size_t) i], cudaEventDisableTiming) == cudaSuccess &&
cudaEventCreateWithFlags(&pp->pused[(size_t) i], cudaEventDisableTiming) == cudaSuccess;
+ if (ok) peer_had_signs(*pp); // APR: on the helper's device
cudaSetDevice(prev);
if (!ok) { err = "prefill: the layer split's prompt help (streams and events)"; return false; }
helper_ = helper;
@@ -1966,6 +1980,7 @@ bool Prefill::set_peer(core::PeerExperts* peer, int64_t cap_rows, std::string& e
pp->ctx = std::make_unique<mmq::Context>();
pp->run_ctx = pp->ctx.get();
mmq::iota(pp->ident, pp->cap_rows, pp->s);
+ peer_had_signs(*pp); // APR: on the peer's device
ok = cudaStreamSynchronize(pp->s) == cudaSuccess;
}
if (ok && !pp->p2p) { // the host route's mapped buffers, sized for the largest chunk
@@ -3214,8 +3229,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// (moe_fused_iq.hpp) where they cover its two formats, else MMQ (or the FP16 path: IQ1_M).
const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
// APR (prism.hadamard): folded gate/up take H (s * x), folded down H (s * h), on the FP16 and MMQ
- // paths; the fused kernels do not rotate (and have no trellis types), and the multi-GPU prompt
- // paths are not set up for a folded pack (set_peer, set_stage_helper)
+ // paths and on the multi-GPU prompt paths (the peer, the layer split's helper: MMQ); the fused
+ // kernels do not rotate (and have no trellis types), so a folded layer takes MMQ instead
const bool had_gu = lay.native && lay.fmt[(size_t) l].had_gu;
const bool had_d = lay.native && lay.fmt[(size_t) l].had_d;
const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];
@@ -3653,10 +3668,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
const size_t mmq_gub = use_mmq ? mmq::matrix_bytes(mmq_gt, 1280, N) : 0;
const size_t mmq_db = use_mmq ? mmq::matrix_bytes(mmq_dt, N, 640) : 0;
pt.mark(kPfGather, cs);
+ // the routed experts' input: `mixed`, or (APR) its rotated copy, which the peer / the layer
+ // split's helper also take (one rotation for both cards; `mixed` itself feeds the shared expert)
+ const float* xsrc = m.mixed;
if (use_mmq) {
// step 2b: the layer's activations as q8_1 rows in expert order, straight from `mixed`
- // (APR: from a rotated copy of `mixed`; `mixed` itself feeds the shared expert)
- const float* xsrc = m.mixed;
if (had_gu) {
strata::kernels::hadamard_rows(m.mixed, m.Xr, T, N, had_block, had_sx, m.cs);
xsrc = m.Xr;
@@ -3855,6 +3871,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
gu.total_rows = nr; gu.max_rows = maxr; gu.dst = P.GU_g; gu.ld_dst = 1280;
P.run_ctx->run(gu, ps);
mmq::swiglu(P.GU_g, P.H_g, nr, 640, !lay.native, ps);
+ if (had_d) // APR: folded down weights take H (s * h), on this card as on the primary
+ strata::kernels::hadamard_rows(P.H_g, P.H_g, nr, 640, had_block, P.had_sh[(size_t) l], ps);
mmq::quantize(P.H_g, nullptr, P.Hq_g, mmq_dt, 640, 640, nr, ps);
const int b = (int) (g2 & 1);
if (P.dm_live[b]) cudaStreamWaitEvent(ps, P.ev_dm[b], 0); // its last rows have left
@@ -3932,6 +3950,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
gu.total_rows = rows_peer; gu.max_rows = maxr; gu.dst = P.GU; gu.ld_dst = 1280;
P.run_ctx->run(gu, ps);
mmq::swiglu(P.GU + r0 * 1280, P.H + r0 * 640, nr, 640, !lay.native, ps);
+ if (had_d) // APR: as above
+ strata::kernels::hadamard_rows(P.H + r0 * 640, P.H + r0 * 640, nr, 640, had_block,
+ P.had_sh[(size_t) l], ps);
mmq::quantize(P.H + r0 * 640, nullptr, P.Hq, mmq_dt, 640, 640, nr, ps);
mmq::Product dn;
dn.w = P.grp_d; dn.type = mmq_dt; dn.w_rows = N; dn.w_cols = 640; dn.expert_bytes = mmq_db;