Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: resolve the APR sign vectors once per device, not per layer and chunk
Laurent Zuijdwijk8fd8e6da3c65
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex ddd7288a0..5ebfc931c 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -969,6 +969,9 @@ struct Prefill::Impl {std::vector<int32_t> tok_host;PinnedUpload tok_up;core::PinnedRing up_ring; // #1996: every other upload of the prompt path whose host side is pageable+ // APR (prism.hadamard): per layer, this device's copies of the gate/up (n_embd) and down (n_ff) sign vectors+ // (nullptr: identity signs or not folded), resolved once in init so a prompt chunk reads a pointer+ std::vector<const float*> had_sx, had_sh;std::unique_ptr<PeerPrefill> pp; // multi-GPU: the peer GPU's expert share (set_peer)// layer split: the next stage's GPU as a stream-only peer (set_stage_helper); its buffers are that stage's own// prompt buffers, bound per prompt, and it takes the place of `pp` for a run it helps@@ -1241,6 +1244,16 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co" x " + std::to_string(lay.fmt[l].n_ff) + ", not the artifact's 2560 x 640";return false;}+ // APR: the sign vectors on this device, once (hadamard_device_signs takes a mutex and, the first time per+ // device, cudaMallocs its copy): run_impl reads m.had_sx / m.had_sh+ m.had_sx.assign(lay.native ? lay.fmt.size() : 0, nullptr);+ m.had_sh.assign(m.had_sx.size(), nullptr);+ for (size_t l = 0; l < m.had_sx.size(); ++l) {+ if (lay.fmt[l].had_gu)+ m.had_sx[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);+ if (lay.fmt[l].had_d)+ m.had_sh[l] = strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);+ }}cudaGetDevice(&m.device);if (stage_le_ < 0) stage_le_ = g.n_layers;@@ -1476,13 +1489,6 @@ bool Prefill::carve(size_t T, void* alloc) {if (!m.mmq_ctx) m.mmq_ctx = std::make_unique<mmq::Context>();}m.ring = ring_slots(T);- { // APR: the device sign vectors now (cudaMalloc), not on first use inside a run- const auto& lay = strata::kernels::cpu::expert_layout();- for (size_t l = 0; lay.native && l < lay.fmt.size(); ++l) {- if (lay.fmt[l].had_gu) strata::kernels::hadamard_device_signs(lay.fmt[l].gu_signs, (int) lay.fmt[l].n_embd);- if (lay.fmt[l].had_d) strata::kernels::hadamard_device_signs(lay.fmt[l].d_signs, (int) lay.fmt[l].n_ff);- }- }if (o.base == nullptr && m.ring > 0) {// OWNED buffers: the ring in ONE allocation. 384 separate 2.7 MiB cudaMallocs each round up to a 2 MiB page// (~1.3 MiB a slot, ~0.5 GiB in all) that no count ever saw. A borrowed region keeps its per-slot layout (and@@ -1747,8 +1753,10 @@ double split_help_frac(int64_t T) {} // namespacenamespace {-// APR: the down projections' sign vectors on the current device (the peer's or the helper's), one per layer, made at-// setup (cudaMalloc) so a prompt chunk only looks them up.+// APR: the down projections' sign vectors on the current device (the peer's or the helper's), one per layer,+// resolved once at setup (hadamard_device_signs: a mutex and, the first time on that device, a cudaMalloc) into+// P.had_sh, which a prompt chunk reads directly. The gate/up signs are not needed there: the primary sends its+// already rotated input (m.Xr).void peer_had_signs(PeerPrefill& P) {const auto& lay = strata::kernels::cpu::expert_layout();P.had_sh.assign(lay.native ? lay.fmt.size() : 0, nullptr);@@ -3250,8 +3258,8 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striconst bool had_d = lay.native && lay.fmt[(size_t) l].had_d;const bool use_mmq = mmq_plan().any && mmq_plan().layer[(size_t) l];const int had_block = lay.native ? lay.fmt[(size_t) l].had_block : 0;- const float* had_sx = had_gu ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].gu_signs, (int) lay.fmt[(size_t) l].n_embd) : nullptr;- const float* had_sh = had_d ? strata::kernels::hadamard_device_signs(lay.fmt[(size_t) l].d_signs, (int) lay.fmt[(size_t) l].n_ff) : nullptr;+ const float* had_sx = had_gu ? m.had_sx[(size_t) l] : nullptr; // resolved in init+ const float* had_sh = had_d ? m.had_sh[(size_t) l] : nullptr;const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;// --peer-device: MMQ only, whether or not the peer took the prompt path (set_peer can decline), as