Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
layer-major prefill on a CUDA layer split: each stage runs its layers over the super-chunk and hands its rows to the next stage (#3976)
Norihiro Nishimura6eff377a791a
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex 0861279af..56ff63d0c 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -259,15 +259,23 @@ public:/// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): true while `on_chunk` reports a sub-chunk before the end of/// its super-chunk. The session state is already at the super-chunk's end then, so no checkpoint may be taken.- bool layer_major_mid() const { return lm_mid_; }+ /// On a layer split it is the last stage's (the one that reports the rows), asked through the first.+ bool layer_major_mid() const {+ const Prefill* p = this;+ while (p->next_ != nullptr) p = p->next_;+ return p->lm_mid_;+ }private:static uint64_t bytes_needed_impl(const core::ModelGeometry& g, const core::SessionState& ss, int64_t chunk,bool owned_pages, bool src);kernels::cpu::ExpertPool* cpu_pool_ = nullptr; ///< set_cpu_pool// Layer-major prefill: the super-chunk's buffers for an n-token prompt (its size, 0 = not this prompt: `why`),- // the run, and the buffers given back after it.- int64_t lm_prepare(int64_t n, std::string& why);+ // the run, and the buffers given back after it. `cap` (> 0): at most that many tokens (a layer split's stages+ // share one size); lm_prepare_chain sizes every stage of a split.+ int64_t lm_prepare(int64_t n, std::string& why, int64_t cap = 0);+ int64_t lm_prepare_chain(int64_t n, std::string& why);+ void lm_release_chain();bool run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);void lm_release();bool lm_mid_ = false;diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 6074ca1a6..67a932f7e 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -5618,6 +5618,7 @@ int64_t lm_table_ints(int64_t rows, int64_t G) { return 5 * rows + 3 * lm_entrievoid Prefill::lm_release() {Impl& m = *impl_;+ const core::OnDevice on_device(m.device);if (m.lm.dev) {cudaStreamSynchronize(m.cs);cudaFree(m.lm.dev);@@ -5629,12 +5630,18 @@ void Prefill::lm_release() {m.lm.S = 0;}-int64_t Prefill::lm_prepare(int64_t n, std::string& why) {+int64_t Prefill::lm_prepare(int64_t n, std::string& why, int64_t cap) {Impl& m = *impl_;if (m.g == nullptr) { why = "not initialized"; return 0; }const core::ModelGeometry& g = *m.g;- if (stage_lb_ != 0 || stage_le_ != g.n_layers || next_ != nullptr || hand_in_ != nullptr) { why = "a layer split"; return 0; }+ const core::OnDevice on_device(m.device); // a layer split's later stage is sized on its own GPU+ // layer split (2026-10-24): each stage runs its own layers layer-major over the same super-chunk and hands the+ // super-chunk's rows to the next one (lm_prepare_chain); the peer GPU's expert share and the stage helper have+ // no layer-major form+ const bool split = stage_lb_ != 0 || stage_le_ != g.n_layers || next_ != nullptr;+ if (hand_in_ != nullptr) { why = "a layer split's chunk hand-off is in use"; return 0; }if (m.pp || helper_ != nullptr || core::peer_portable()) { why = "a peer GPU"; return 0; }+ if (split && m.ss->ple.ready()) { why = "a layer split with PLE layers"; return 0; }if (embd_rows != nullptr) { why = "an image prompt"; return 0; }// run_impl's debug hooks on the routing and the rows have no layer-major formif (std::getenv("STRATA_DBG_FEAT") || std::getenv("STRATA_DBG_ROUTE_OVERRIDE") || std::getenv("STRATA_PREFILL_DUMP_R_ALL")) {@@ -5672,6 +5679,10 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why) {const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : kLmDefaultTokens;int64_t S = std::max<int64_t>(min_s, want / m.T * m.T);S = std::min<int64_t>(S, (n + m.T - 1) / m.T * m.T);+ if (cap > 0) {+ if (cap % m.T != 0 || cap < min_s) { why = "the layer split's stages read different chunks"; return 0; }+ S = std::min(S, cap);+ }const int64_t G = m.T * K;const auto dev_bytes = [&](int64_t s, bool r_dev) {const uint64_t per = (uint64_t) (2 * N + HC + 2 * K) * 4 + (uint64_t) N * 2 + (r_dev ? (uint64_t) D * 4 : 0);@@ -5687,7 +5698,8 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why) {while (s >= min_s && (int64_t) dev_bytes(s, r_dev) > room) s -= m.T;return s >= min_s ? s : (int64_t) 0;};- const int mode = lm_r_env();+ // a stage after the first: its rows arrive in R_host (the previous stage writes them there), so they stay there+ const int mode = stage_lb_ > 0 ? 2 : lm_r_env();bool r_dev = false;int64_t s_use = 0;if (mode == 1) { s_use = fit(true); r_dev = true; }@@ -5754,6 +5766,37 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why) {return s_use;}+// A layer split: every stage gets the same super-chunk (the smallest any of them fits), re-sized until they agree.+int64_t Prefill::lm_prepare_chain(int64_t n, std::string& why) {+ int64_t cap = 0;+ for (int round = 0; round < 4; ++round) {+ bool same = true;+ for (Prefill* p = this; p != nullptr; p = p->next_) {+ Impl::LmBufs& lb = p->impl_->lm;+ if (lb.S > 0 && lb.S == cap) continue;+ if (lb.S > 0) p->lm_release();+ const int64_t s = p->lm_prepare(n, why, cap);+ if (s <= 0) {+ if (p != this)+ why = "layers " + std::to_string(p->stage_lb_) + ".." + std::to_string(p->stage_le_) + ": " + why;+ lm_release_chain();+ return 0;+ }+ if (cap > 0 && s != cap) same = false;+ cap = s;+ }+ if (same) return cap;+ }+ why = "the layer split's stages found no common super-chunk";+ lm_release_chain();+ return 0;+}++void Prefill::lm_release_chain() {+ for (Prefill* p = this; p != nullptr; p = p->next_)+ if (p->impl_) p->lm_release();+}+bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {err.clear();Impl& m = *impl_;@@ -5764,6 +5807,11 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stconst cudaStream_t cs = (cudaStream_t) m.cs;Impl::LmBufs& lb = m.lm;const int64_t S = lb.S, L = g.n_layers;+ // a layer split's stage (2026-10-24): its layers [LB, LE); a later stage's rows arrive in R_host, written by the+ // stage before it, and a stage before the last writes its final rows into the next stage's R_host, then runs+ // the next stage on that super-chunk on a thread while it reads its own next super-chunk (one hand-off buffer:+ // its final pass waits until the next stage has finished the previous one)+ const int64_t LB = stage_lb_, LE = stage_le_;strata::kernels::QsaShapes s = core::qsa_shapes(g); // as 0.2.2's run_implconst uint64_t gdn_floats = (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size +(uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);@@ -6162,9 +6210,9 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, stconst int64_t nc = (Sn + m.T - 1) / m.T;++lm_supers;lm_mid_ = true;- for (int64_t l = 0; l <= L; ++l) {+ for (int64_t l = LB; l <= LE; ++l) {const auto ta = Clock::now();- if (l < L && !lm_prefetch(l)) return false;+ if (l < LE && !lm_prefetch(l)) return false;// host mode: sub-chunk c's rows go up on the copy stream into buffer c % 2 (issued one sub-chunk ahead)auto r_buf = [&](int64_t c) { return (c & 1) ? lb.R2 : R_chunk; };auto issue_up = [&](int64_t c) {@@ -6241,13 +6289,29 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, streturn ple_gather(c0, b, ple_next_err);});for (int64_t t = 0; t < T; ++t) { prev[0] = prev[1]; prev[1] = (int32_t) tokens[c0 + t]; }- } else {+ } else if (l > LB) { // (a later stage's first layer: the stage before wrote layer LB - 1)// the previous layer's experts: the hyper-connection write of its FFN halfgr_write(m.R, acc, m.inj, HC, T, cs);if (strata::kernels::cvec().covers(l - 1))strata::kernels::cvec_apply(m.R, l - 1, T, D, nullptr, 0, nullptr, 0, false, cs);}- if (l == L) {+ if (l == LE && next_ != nullptr) {+ // ---- a layer split: this sub-chunk's rows into the next stage's R_host (on the copy stream, as+ // the per-layer download) once the next stage has finished the previous super-chunk+ stats_.tokens += T;+ ++stats_.chunks;+ if (c == 0 && next_run_.valid() && !next_run_.get()) { err = next_err_; return false; }+ cudaEventRecord(lb.ev_done[c & 1], cs);+ cudaStreamWaitEvent(m.copy, lb.ev_done[c & 1], 0);+ if (cudaMemcpyAsync(next_->impl_->lm.R_host + (size_t) t0 * D, m.R, (size_t) T * D * 4,+ cudaMemcpyDeviceToHost, m.copy) != cudaSuccess) {+ err = "prefill (layer-major): the layer split's hand-off failed";+ return false;+ }+ cudaEventRecord(lb.ev_out[c & 1], m.copy);+ continue;+ }+ if (l == LE) {// ---- the final rows of this sub-chunk: reported in order, as run_impl's chunks arestats_.tokens += T;++stats_.chunks;@@ -6270,7 +6334,7 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, streturn false;}lm_mid_ = c + 1 < nc;- if (on_stage_chunk && !on_stage_chunk(p0 + T, err)) return false;+ if (on_stage_chunk && !lm_mid_ && !on_stage_chunk(p0 + T, err)) return false;if (on_chunk && !on_chunk(m.R, T, p0, err)) return false;}continue;@@ -6562,9 +6626,22 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, st}}ms_attn += ms_since(ta);- if (l == L) break;+ if (l == LE) break;if (!lm_experts(l, Sn)) return false;}+ if (next_ != nullptr) { // the super-chunk's rows are in the next stage's R_host: it reads them on a thread+ if (cudaStreamSynchronize(cs) != cudaSuccess || cudaStreamSynchronize(m.copy) != cudaSuccess) {+ err = std::string("prefill (layer-major): the layer split's hand-off: ") ++ cudaGetErrorString(cudaGetLastError());+ return false;+ }+ if (on_stage_chunk && !on_stage_chunk(pos0 + sc0 + Sn, err)) return false;+ next_err_.clear();+ next_run_ = std::async(std::launch::async, [this, tokens, sc0, Sn, pos0] {+ strata::aux_cpus::pin_current_thread();+ return next_->run_layer_major(tokens + sc0, Sn, pos0 + sc0, next_err_);+ });+ }}lm_mid_ = false;ss.ple_prev[0] = prev[0];@@ -6579,11 +6656,12 @@ bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, streturn false;}stats_.ms_total += ms_since(t_start);- std::fprintf(stderr, "strata prefill: layer-major: %lld tokens in %lld super-chunk(s) of up to %lld (R in %s, %lld MiB VRAM free before), "+ std::fprintf(stderr, "strata prefill: layer-major (layers %lld..%lld): %lld tokens in %lld super-chunk(s) of up to %lld (R in %s, %lld MiB VRAM free before), ""%lld expert blobs streamed (%.1f MB), %lld resident, %lld expert groups (%lld on the FP16 path); %.0f ms (host: ""enqueueing the attention halves %.0f, the experts %.0f: waiting for the attention halves ""%.0f, the routing tables %.0f)\n",- (long long) n, (long long) lm_supers, (long long) S, lb.r_dev ? "VRAM" : "pinned RAM", (long long) lb.free_mib,+ (long long) LB, (long long) LE, (long long) n, (long long) lm_supers, (long long) S,+ lb.r_dev ? "VRAM" : "pinned RAM", (long long) lb.free_mib,(long long) lm_streamed, (double) lm_bytes / 1e6, (long long) lm_resident, (long long) lm_groups,(long long) lm_groups_f16, ms_since(t_start), ms_attn, ms_experts, ms_wait_a, ms_host_tables);if (pt.on) {@@ -6653,10 +6731,15 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e#endifif (lm_env() && hand_in_ == nullptr && impl_) {std::string why;- if (lm_prepare(n, why) > 0) {- const bool ok = run_layer_major(tokens, n, pos0, err);- lm_mid_ = false;- lm_release();+ if (lm_prepare_chain(n, why) > 0) {+ bool ok = run_layer_major(tokens, n, pos0, err);+ std::string drain_err; // a layer split: the later stages' last super-chunk+ if (!drain_pipeline(drain_err) && ok) {+ ok = false;+ err = drain_err;+ }+ for (Prefill* p = this; p != nullptr; p = p->next_) p->lm_mid_ = false;+ lm_release_chain();return ok;}static std::string said; // once per reason (a short prompt is the common one)