Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1) (#3976, #1670)

vizdatomffe40c1695e9
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hpp
index 39ed85577..0861279af 100644
--- a/include/strata/prefill/prefill.hpp
+++ b/include/strata/prefill/prefill.hpp
@@ -257,10 +257,20 @@ public:
/// the share is on for chunks below 1024 tokens (STRATA_PREFILL_CPU_SHARE=0 turns it off).
static void arm_cpu_share(bool applies, bool by_default = false);
+ /// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): true while `on_chunk` reports a sub-chunk before the end of
+ /// its super-chunk. The session state is already at the super-chunk's end then, so no checkpoint may be taken.
+ bool layer_major_mid() const { return lm_mid_; }
+
private:
static uint64_t bytes_needed_impl(const core::ModelGeometry& g, const core::SessionState& ss, int64_t chunk,
bool owned_pages, bool src);
kernels::cpu::ExpertPool* cpu_pool_ = nullptr; ///< set_cpu_pool
+ // Layer-major prefill: the super-chunk's buffers for an n-token prompt (its size, 0 = not this prompt: `why`),
+ // the run, and the buffers given back after it.
+ int64_t lm_prepare(int64_t n, std::string& why);
+ bool run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);
+ void lm_release();
+ bool lm_mid_ = false;
// Stage-1 pipeline: intermediate stages return after handing their chunk to
// the direct successor. The public run() drains the chain once at prompt end.
bool run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index e1f221c76..7a8a8de2f 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1092,6 +1092,27 @@ struct Prefill::Impl {
// layer split: the next stage's GPU as a stream-only peer (set_stage_helper); its buffers are that stage's own
// prompt buffers, bound per prompt, and it takes the place of `pp` for a run it helps
std::unique_ptr<PeerPrefill> help_pp;
+ // layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): a super-chunk's state between a layer's attention halves
+ // and its experts - device buffers per prompt (lm_prepare / lm_release), pinned host buffers kept
+ struct LmBufs {
+ int64_t S = 0; // tokens of a super-chunk (0: not allocated)
+ bool r_dev = false; // its residual rows in VRAM (else R_host, copied per sub-chunk)
+ void* dev = nullptr; // the one device allocation the pointers below are carved from
+ float *mixed = nullptr, *acc = nullptr, *inj = nullptr, *R = nullptr, *w = nullptr;
+ int32_t *ids = nullptr, *tab = nullptr;
+ size_t tab_n = 0;
+ float* R_host = nullptr;
+ size_t R_host_n = 0;
+ int32_t* tab_host = nullptr;
+ size_t tab_host_n = 0;
+ int32_t* idsw_host = nullptr;
+ size_t idsw_host_n = 0;
+ int64_t free_mib = 0; // VRAM free when the buffers were sized (the log line)
+ // host mode: the residual rows of two sub-chunks on the device in turn (the chunk's R and R2), so the copy
+ // stream uploads the next one and downloads the last one while the compute stream works on this one
+ float* R2 = nullptr;
+ cudaEvent_t ev_in[2] = {}, ev_done[2] = {}, ev_out[2] = {};
+ } lm;
};
namespace {
@@ -1179,6 +1200,13 @@ void Prefill::release() {
if (impl_->kv_released) cudaEventDestroy(impl_->kv_released);
if (impl_->kv_ready) cudaEventDestroy(impl_->kv_ready);
if (impl_->grp_host) cudaFreeHost(impl_->grp_host);
+ if (impl_->lm.dev) cudaFree(impl_->lm.dev);
+ if (impl_->lm.R_host) cudaFreeHost(impl_->lm.R_host);
+ if (impl_->lm.tab_host) cudaFreeHost(impl_->lm.tab_host);
+ if (impl_->lm.idsw_host) cudaFreeHost(impl_->lm.idsw_host);
+ for (int b = 0; b < 2; ++b)
+ for (cudaEvent_t e : {impl_->lm.ev_in[b], impl_->lm.ev_done[b], impl_->lm.ev_out[b]})
+ if (e) cudaEventDestroy(e);
impl_->steps_up.release();
impl_->tok_up.release();
impl_->up_ring.release();
@@ -5537,6 +5565,969 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
return true;
}
+// ============================================================================================================
+// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1).
+//
+// The default prompt path reads a chunk (here 512 tokens) through all 48 layers, so every chunk streams every routed
+// expert of every layer to the GPU again - ~25-34 GB of expert blobs per chunk on Q2_0, which is what bounds a prompt
+// on a small card (PCIe / NVMe). This path reads a SUPER-CHUNK of S tokens layer by layer instead: for layer l it
+// runs each sub-chunk's attention half, the FFN half's hyper-connection read, the router and the shared expert
+// (in order, so the GDN / QSA / PLE state advance exactly as in the default path), parks what the experts need
+// (the FFN input rows, the routing, the shared expert's part of the output) and then runs the layer's experts ONCE
+// for all S tokens: each routed expert is staged once, its rows of every sub-chunk computed in one MMQ group, and
+// the results added per token into the super-chunk's accumulator. The FFN half's hyper-connection write of layer l
+// runs at the start of the sub-chunk's layer l + 1 (or in the final pass). Expert traffic drops by ~S / 512.
+//
+// The residual rows of the super-chunk live in VRAM when they fit (STRATA_PREFILL_LM_R=vram) or in pinned host
+// memory, copied in and out per sub-chunk and layer (host). Only the experts' sum differs from the default path:
+// it is added per expert group in id order, not in the router's k order (FP32 rounding), and the MMQ products run on
+// other row groupings. on_chunk reports every sub-chunk in order from the final pass; while it reports one before
+// the super-chunk's end, layer_major_mid() is true (the session state is the super-chunk's end: no checkpoint).
+namespace {
+bool lm_env() {
+ static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LAYER_MAJOR"); return e && std::atoi(e) != 0; }();
+ return v;
+}
+int64_t lm_tokens_env() { // the super-chunk's target size (0: the default)
+ static const int64_t v = [] { const char* e = std::getenv("STRATA_PREFILL_LM_TOKENS"); return e ? (int64_t) std::atoll(e) : (int64_t) 0; }();
+ return v;
+}
+int lm_r_env() { // 0 auto, 1 vram, 2 host
+ static const int v = [] {
+ const char* e = std::getenv("STRATA_PREFILL_LM_R");
+ if (!e) return 0;
+ const std::string s(e);
+ return s == "vram" ? 1 : s == "host" ? 2 : 0;
+ }();
+ return v;
+}
+int64_t lm_margin_bytes() { // VRAM left free while a prompt runs (the verify windows, the driver)
+ static const int64_t v = [] {
+ const char* e = std::getenv("STRATA_PREFILL_LM_MARGIN_MIB");
+ return (int64_t) (e ? std::atoll(e) : 96) << 20;
+ }();
+ return v;
+}
+constexpr int64_t kLmDefaultTokens = 8192;
+// the device tables of one layer, in int32s: row_tok (rows) | bounds (2 E) | u_tok (rows) | u_off (rows + E) |
+// u_rows (rows) | u_w (rows), E = the most entries a layer can split into
+int64_t lm_entries_max(int64_t rows, int64_t G) { return NE + rows / std::max<int64_t>(G, 1) + 2; }
+int64_t lm_table_ints(int64_t rows, int64_t G) { return 5 * rows + 3 * lm_entries_max(rows, G); }
+} // namespace
+
+void Prefill::lm_release() {
+ Impl& m = *impl_;
+ if (m.lm.dev) {
+ cudaStreamSynchronize(m.cs);
+ cudaFree(m.lm.dev);
+ }
+ m.lm.dev = nullptr;
+ m.lm.mixed = m.lm.acc = m.lm.inj = m.lm.R = m.lm.w = m.lm.R2 = nullptr;
+ m.lm.ids = m.lm.tab = nullptr;
+ m.lm.S = 0;
+}
+
+int64_t Prefill::lm_prepare(int64_t n, std::string& why) {
+ Impl& m = *impl_;
+ if (m.g == nullptr) { why = "not initialized"; return 0; }
+ const core::ModelGeometry& g = *m.g;
+ if (stage_lb_ != 0 || stage_le_ != g.n_layers || next_ != nullptr || hand_in_ != nullptr) { why = "a layer split"; return 0; }
+ if (m.pp || helper_ != nullptr || core::peer_portable()) { why = "a peer GPU"; return 0; }
+ if (embd_rows != nullptr) { why = "an image prompt"; return 0; }
+ // run_impl's debug hooks on the routing and the rows have no layer-major form
+ if (std::getenv("STRATA_DBG_FEAT") || std::getenv("STRATA_DBG_ROUTE_OVERRIDE") || std::getenv("STRATA_PREFILL_DUMP_R_ALL")) {
+ why = "a routing or row debug dump";
+ return 0;
+ }
+ const MmqPlan& mp = mmq_plan();
+ if (!mp.any || (int64_t) mp.layer.size() < g.n_layers) { why = "the experts are not on MMQ"; return 0; }
+ for (int64_t l = 0; l < g.n_layers; ++l)
+ if (!mp.layer[(size_t) l]) { why = "a layer's experts are not on MMQ"; return 0; }
+ if (m.fused_bufs) { why = "the fused expert layout"; return 0; }
+ if (!m.grp_gu || !m.grp_d || !m.Xq || !m.GU || !m.H || !m.Hq || !m.Dm || !m.ids_identity || !m.tok_dev) {
+ why = "the MoE buffers are missing"; return 0;
+ }
+ // (an explicit STRATA_PREFILL_LM_TOKENS may be one chunk: the A/B against the default path, experts aside)
+ const int64_t min_s = lm_tokens_env() > 0 ? m.T : 2 * m.T;
+ if (n < min_s) { why = "a short prompt"; return 0; }
+ const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : kLmDefaultTokens;
+ int64_t S = std::max<int64_t>(min_s, want / m.T * m.T);
+ S = std::min<int64_t>(S, (n + m.T - 1) / m.T * m.T);
+ const int64_t G = m.T * K;
+ const auto dev_bytes = [&](int64_t s, bool r_dev) {
+ const uint64_t per = (uint64_t) (2 * N + HC + 2 * K) * 4 + (r_dev ? (uint64_t) D * 4 : 0);
+ return (uint64_t) s * per + (uint64_t) lm_table_ints(s * K, G) * 4 + 4096 +
+ (r_dev ? 0 : (uint64_t) m.T * D * 4 + 256);
+ };
+ size_t free_b = 0, total_b = 0;
+ cudaMemGetInfo(&free_b, &total_b);
+ const int64_t room = (int64_t) free_b - lm_margin_bytes();
+ m.lm.free_mib = (int64_t) (free_b >> 20);
+ auto fit = [&](bool r_dev) { // the largest super-chunk (a multiple of the chunk) that fits `room`
+ int64_t s = S;
+ while (s >= min_s && (int64_t) dev_bytes(s, r_dev) > room) s -= m.T;
+ return s >= min_s ? s : (int64_t) 0;
+ };
+ const int mode = lm_r_env();
+ bool r_dev = false;
+ int64_t s_use = 0;
+ if (mode == 1) { s_use = fit(true); r_dev = true; }
+ else if (mode == 2) { s_use = fit(false); }
+ else { // auto: VRAM when the whole target fits there, else pinned host (the bigger super-chunk wins)
+ const int64_t sv = fit(true);
+ if (sv == S) { s_use = sv; r_dev = true; } else s_use = fit(false);
+ }
+ if (s_use <= 0) {
+ why = "no VRAM for a super-chunk (" + std::to_string(free_b >> 20) + " MiB free)";
+ return 0;
+ }
+ Impl::LmBufs& lb = m.lm;
+ // pinned host: the routing's copy, the tables, and (host mode) the residual rows
+ const size_t idsw = (size_t) (2 * s_use * K), tab = (size_t) lm_table_ints(s_use * K, G);
+ auto host_grow = [](int32_t*& p, size_t& have, size_t want_n) -> bool {
+ if (have >= want_n) return true;
+ if (p) cudaFreeHost(p);
+ p = nullptr;
+ have = 0;
+ if (cudaHostAlloc((void**) &p, want_n * 4, cudaHostAllocDefault) != cudaSuccess) { cudaGetLastError(); p = nullptr; return false; }
+ have = want_n;
+ return true;
+ };
+ if (!host_grow(lb.idsw_host, lb.idsw_host_n, idsw) || !host_grow(lb.tab_host, lb.tab_host_n, tab)) {
+ why = "no pinned RAM for the routing tables"; return 0;
+ }
+ if (!r_dev) {
+ int32_t* rh = (int32_t*) lb.R_host;
+ if (!host_grow(rh, lb.R_host_n, (size_t) s_use * D)) { why = "no pinned RAM for the residual rows"; lb.R_host = nullptr; return 0; }
+ lb.R_host = (float*) rh;
+ }
+ const uint64_t bytes = dev_bytes(s_use, r_dev);
+ void* d = nullptr;
+ if (cudaMalloc(&d, bytes) != cudaSuccess) {
+ cudaGetLastError();
+ why = "the super-chunk's buffers (" + std::to_string(bytes >> 20) + " MiB) do not fit";
+ return 0;
+ }
+ lb.dev = d;
+ uint8_t* q = (uint8_t*) d;
+ auto carve_b = [&](size_t b) { uint8_t* p = q; q += (b + 255) & ~(size_t) 255; return p; };
+ lb.mixed = (float*) carve_b((size_t) s_use * N * 4);
+ lb.acc = (float*) carve_b((size_t) s_use * N * 4);
+ lb.inj = (float*) carve_b((size_t) s_use * HC * 4);
+ lb.ids = (int32_t*) carve_b((size_t) s_use * K * 4);
+ lb.w = (float*) carve_b((size_t) s_use * K * 4);
+ lb.R = r_dev ? (float*) carve_b((size_t) s_use * D * 4) : nullptr;
+ lb.tab = (int32_t*) carve_b(tab * 4);
+ lb.tab_n = tab;
+ lb.R2 = r_dev ? nullptr : (float*) carve_b((size_t) m.T * D * 4);
+ for (int b = 0; b < 2; ++b)
+ for (cudaEvent_t* e : {&lb.ev_in[b], &lb.ev_done[b], &lb.ev_out[b]})
+ if (*e == nullptr && cudaEventCreateWithFlags(e, cudaEventDisableTiming) != cudaSuccess) {
+ cudaGetLastError();
+ *e = nullptr;
+ why = "the copy events";
+ lm_release();
+ return 0;
+ }
+ lb.r_dev = r_dev;
+ lb.S = s_use;
+ return s_use;
+}
+
+bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {
+ err.clear();
+ Impl& m = *impl_;
+ const core::OnDevice on_device(m.device);
+ const core::ModelGeometry& g = *m.g;
+ core::SessionState& ss = *m.ss;
+ const auto t_start = Clock::now();
+ const cudaStream_t cs = (cudaStream_t) m.cs;
+ Impl::LmBufs& lb = m.lm;
+ const int64_t S = lb.S, L = g.n_layers;
+ strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes();
+ s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads;
+ s.idx_dim = g.idx_key_dim;
+ const uint64_t gdn_floats = (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size +
+ (uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);
+ int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};
+ PfTimer pt;
+ // the MMQ row table lives in the borrowed cache slots when the prompt path borrows: written for every prompt
+ mmq::iota(m.ids_identity, m.T * K, m.cs);
+ // ---- the PLE rows, read per chunk on a thread as run_impl does (the same state machine, in chunk order)
+ const bool ple_on = ss.ple.ready();
+ static const bool ple_batch_env = [] {
+ const char* v = std::getenv("STRATA_PLE_BATCH");
+ return v == nullptr || std::atoi(v) != 0;
+ }();
+ const bool ple_batch = ple_on && ple_batch_env && strata::kernels::ple_native_postops_enabled() &&
+ (ss.ple.w.key_bf16 != nullptr || ss.ple.w.key_native_data != nullptr) &&
+ m.region_bytes / ((uint64_t) (3 * strata::kernels::NG_HC_DIM + N + 4) * 4 + (uint64_t) N * 2 + 4096) >= 64;
+ const int32_t prev0[2] = {prev[0], prev[1]};
+ auto ple_gather = [&m, &ss, tokens, n, prev0](int64_t c0, int buf, std::string& e) -> bool {
+ const int64_t T = std::min(m.T, n - c0);
+ auto at = [&](int64_t i) { return i < 2 ? prev0[i] : (int32_t) tokens[i - 2]; };
+ int32_t pv[2] = {at(c0), at(c0 + 1)};
+ for (int64_t t = 0; t < T; ++t) {
+ const int32_t tok = (int32_t) tokens[c0 + t];
+ strata::kernels::ngram_rows(&tok, pv, 1, ss.ple.consts,
+ m.ple_rows[buf].data() + t * strata::kernels::PLE_N_HEADS);
+ pv[0] = pv[1];
+ pv[1] = tok;
+ }
+ return ss.ple.table->gather_batch(m.ple_rows[buf].data(), (size_t) T, m.ple_emb_host[buf], e);
+ };
+ std::string ple_next_err;
+ std::future<bool> ple_next;
+ int ple_buf = 0;
+ auto ple_land = [&](int64_t c0, int64_t T) -> bool {
+ const auto tp = Clock::now();
+ if (!ple_next.get()) { err = ple_next_err; return false; }
+ // #1996: the pageable fallback (no RAM to pin) goes up through a pinned slot; the pinned buffers as before
+ if ((m.ple_pageable[ple_buf].empty()
+ ? cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, cs)
+ : m.up_ring.h2d(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cs)) != cudaSuccess ||
+ cudaEventRecord(m.ple_copied[ple_buf], cs) != cudaSuccess) {
+ err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ if (c0 + m.T < n) {
+ if (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) {
+ err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c1 = c0 + m.T, b = ple_buf ^ 1] {
+ strata::aux_cpus::pin_current_thread();
+ return ple_gather(c1, b, ple_next_err);
+ });
+ }
+ ple_buf ^= 1;
+ stats_.ms_ple += ms_since(tp);
+ return true;
+ };
+ // the chunk buffers this path points into the super-chunk (restored on every return)
+ struct Restore {
+ Impl& m;
+ float *R, *mixed, *inj;
+ ~Restore() { m.R = R; m.mixed = mixed; m.inj = inj; }
+ } restore{m, m.R, m.mixed, m.inj};
+ float* const R_chunk = m.R;
+ const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();
+ double ms_attn = 0, ms_experts = 0, ms_host_tables = 0, ms_wait_a = 0;
+ std::vector<int> pf_job((size_t) NE, -1); // the layer's prefetch: expert -> stager job (-1: not streamed)
+ // every streamed expert of layer l, in id order, to the stager - started with the layer's attention halves, so
+ // the NVMe / page-cache reads overlap them (at a super-chunk's size nearly every expert is routed)
+ static const bool pf_on = [] { const char* e = std::getenv("STRATA_LM_PREFETCH"); return e == nullptr || e[0] != '0'; }();
+ auto lm_prefetch = [&](int64_t l) -> bool {
+ std::fill(pf_job.begin(), pf_job.end(), -1);
+ if (m.src == nullptr || !pf_on) return true;
+ std::vector<Stager::Job> js;
+ const bool pdirect = m.src->prefill_direct();
+ for (int32_t e = 0; e < NE; ++e) {
+ if ((m.host_res && m.cache && m.host_res[(size_t) l * NE + e] >= 0) || m.src->pinned(l, e)) continue;
+ pf_job[(size_t) e] = (int) js.size();
+ if (pdirect || m.src->transient(l, e)) {
+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});
+ continue;
+ }
+ const uint8_t* b = m.src->blob_stable(l, e);
+ if (!b) { err = "prefill (layer-major): expert source has no blob"; return false; }
+ js.push_back({b, (size_t) lay.blob_bytes(l)});
+ }
+ m.stager->start(std::move(js));
+ return true;
+ };
+ struct StagerStop {
+ Stager* st;
+ ~StagerStop() { if (st) st->finish(); } // an early return in a layer's attention halves
+ } stager_stop{m.stager.get()};
+ int64_t lm_streamed = 0, lm_resident = 0, lm_groups = 0, lm_supers = 0;
+ uint64_t lm_bytes = 0;
+ const int64_t G = m.T * K; // the MoE buffers' rows (Xq, GU, H, Hq, Dm): a group's most
+
+ // ---- one layer's experts over the super-chunk's Sn tokens
+ auto lm_experts = [&](int64_t l, int64_t Sn) -> bool {
+ const auto th0 = Clock::now();
+ const int64_t rows = Sn * K;
+ int32_t* idsw = lb.idsw_host;
+ cudaMemcpyAsync(idsw, lb.ids, (size_t) rows * 4, cudaMemcpyDeviceToHost, cs);
+ cudaMemcpyAsync(idsw + rows, lb.w, (size_t) rows * 4, cudaMemcpyDeviceToHost, cs);
+ core::progress_at("reading the prompt (layer-major): waiting for the GPU (attention, router) at layer", l, pos0);
+ if (cudaStreamSynchronize(cs) != cudaSuccess) {
+ err = std::string("prefill (layer-major): ") + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ core::progress_at("reading the prompt (layer-major): the experts of layer", l, pos0);
+ pt.fold();
+ ms_wait_a += ms_since(th0);
+ const auto th1 = Clock::now();
+ const int32_t* ids_h = idsw;
+ const float* w_h = (const float*) (idsw + rows);
+ std::vector<int32_t> cnt((size_t) NE, 0), off((size_t) NE + 1, 0);
+ for (int64_t i = 0; i < rows; ++i) {
+ const int32_t e = ids_h[(size_t) i];
+ if (e < 0 || e >= NE) { err = "prefill (layer-major): routed id out of range"; return false; }
+ ++cnt[(size_t) e];
+ }
+ for (int32_t e = 0; e < NE; ++e) off[(size_t) e + 1] = off[(size_t) e] + cnt[(size_t) e];
+ // ---- the routed experts in id order: resident ones from the cache, the others staged once (run_impl's walk)
+ // (started before the tables are built: the stager's threads read while the host builds them)
+ std::vector<int32_t> uniq;
+ for (int32_t e = 0; e < NE; ++e)
+ if (cnt[(size_t) e] > 0) uniq.push_back(e);
+ auto resident = [&](int32_t e) { return m.host_res && m.cache && m.host_res[(size_t) l * NE + e] >= 0; };
+ // the stager was started with every streamed expert of the layer when its attention halves began (the
+ // prefetch); an expert the routing skipped is never waited for (a later job's issue covers its buffer)
+ std::vector<int> job_of(uniq.size(), -1), stage_of(uniq.size(), -1);
+ for (size_t u = 0; u < uniq.size(); ++u) job_of[u] = pf_job[(size_t) uniq[u]];
+ for (int32_t e = 0; e < NE; ++e) // prefetched but not routed: not read, if not started yet
+ if (cnt[(size_t) e] == 0 && pf_job[(size_t) e] >= 0) m.stager->skip(pf_job[(size_t) e]);
+ if (!pf_on && m.src != nullptr) { // STRATA_LM_PREFETCH=0: the routed experts only, started now
+ std::vector<Stager::Job> js;
+ const bool pdirect = m.src->prefill_direct();
+ for (size_t u = 0; u < uniq.size(); ++u) {
+ const int32_t e = uniq[u];
+ if (resident(e) || m.src->pinned(l, e)) continue;
+ job_of[u] = (int) js.size();
+ if (pdirect || m.src->transient(l, e)) {
+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});
+ continue;
+ }
+ const uint8_t* b = m.src->blob_stable(l, e);
+ if (!b) { err = "prefill (layer-major): expert source has no blob"; return false; }
+ js.push_back({b, (size_t) lay.blob_bytes(l)});
+ }
+ m.stager->start(std::move(js));
+ }
+ struct StagerDone {
+ Stager* st;
+ ~StagerDone() { if (st) st->finish(); }
+ } stager_done{m.stager.get()};
+ // the tables: rows sorted by expert (token order within one), then per group its bounds and per-token CSR
+ int32_t* th = lb.tab_host;
+ int32_t* row_tok = th;
+ std::vector<float> row_w((size_t) rows);
+ {
+ std::vector<int32_t> fill(off.begin(), off.end() - 1);
+ for (int64_t i = 0; i < rows; ++i) {
+ const int32_t e = ids_h[(size_t) i];
+ const int32_t p = fill[(size_t) e]++;
+ row_tok[p] = (int32_t) (i / K);
+ row_w[(size_t) p] = w_h[(size_t) i];
+ }
+ }
+ struct Entry { int32_t e, r0, nr; };
+ std::vector<Entry> ents;
+ for (int32_t e = 0; e < NE; ++e)
+ for (int32_t r = off[(size_t) e]; r < off[(size_t) e + 1]; r += (int32_t) G)
+ ents.push_back({e, r, (int32_t) std::min<int64_t>(G, off[(size_t) e + 1] - r)});
+ struct Grp { size_t j0, j1; int32_t r0, nr, maxr; int64_t b_at, u_at, uo_at, nu; };
+ std::vector<Grp> grps;
+ for (size_t j = 0; j < ents.size();) {
+ Grp gp{};
+ gp.j0 = j;
+ gp.r0 = ents[j].r0;
+ int32_t nr = 0, maxr = 0;
+ while (j < ents.size() && j - gp.j0 < (size_t) MMQ_GROUP && nr + ents[j].nr <= G) {
+ nr += ents[j].nr;
+ maxr = std::max(maxr, ents[j].nr);
+ ++j;
+ }
+ gp.j1 = j;
+ gp.nr = nr;
+ gp.maxr = maxr;
+ grps.push_back(gp);
+ }
+ const int64_t E = (int64_t) ents.size();
+ if (E > lm_entries_max(rows, G)) { err = "prefill (layer-major): more expert entries than the tables hold"; return false; }
+ int64_t at = rows;
+ int32_t* bounds = th + at;
+ int64_t b_used = 0;
+ at += 2 * lm_entries_max(rows, G);
+ int32_t* u_tok = th + at;
+ at += rows;
+ int32_t* u_off = th + at;
+ at += rows + lm_entries_max(rows, G);
+ int32_t* u_rows = th + at;
+ at += rows;
+ float* u_w = (float*) (th + at);
+ at += rows;
+ if (at > (int64_t) lb.tab_n || at > (int64_t) lb.tab_host_n) { err = "prefill (layer-major): the tables overflow"; return false; }
+ int64_t u_used = 0, uo_used = 0;
+ // per group, the tokens in order of first appearance and each token's rows in row order (counting, O(rows));
+ // `slot` maps a token to its place in the group (-1: not in it), reset through the group's own token list
+ std::vector<int32_t> slot((size_t) Sn, -1), fillp;
+ for (Grp& gp : grps) {
+ gp.b_at = b_used;
+ for (size_t j = gp.j0; j < gp.j1; ++j) bounds[b_used++] = ents[j].r0 - gp.r0;
+ bounds[b_used++] = gp.nr;
+ gp.u_at = u_used;
+ gp.uo_at = uo_used;
+ int32_t* ut = u_tok + u_used;
+ int32_t* uo = u_off + uo_used;
+ int64_t nu = 0;
+ for (int32_t r = 0; r < gp.nr; ++r) {
+ const int32_t t = row_tok[gp.r0 + r];
+ if (slot[(size_t) t] < 0) { slot[(size_t) t] = (int32_t) nu; ut[nu] = t; uo[nu] = 0; ++nu; }
+ ++uo[slot[(size_t) t]];
+ }
+ fillp.assign((size_t) nu, 0);
+ for (int64_t i = 0, acc_n = 0; i < nu; ++i) { const int32_t c = uo[i]; uo[i] = (int32_t) acc_n; fillp[(size_t) i] = (int32_t) acc_n; acc_n += c; }
+ uo[nu] = gp.nr;
+ for (int32_t r = 0; r < gp.nr; ++r) {
+ const int32_t p = fillp[(size_t) slot[(size_t) row_tok[gp.r0 + r]]]++;
+ u_rows[gp.r0 + p] = r;
+ u_w[gp.r0 + p] = row_w[(size_t) (gp.r0 + r)];
+ }
+ for (int64_t i = 0; i < nu; ++i) slot[(size_t) ut[i]] = -1;
+ gp.nu = nu;
+ u_used += nu;
+ uo_used += nu + 1;
+ }
+ // (u_rows / u_w are indexed by the group's own row range [r0, r0 + nr) of the layer)
+ cudaMemcpyAsync(lb.tab, th, (size_t) at * 4, cudaMemcpyHostToDevice, cs);
+ const int32_t* d_row_tok = lb.tab;
+ const int32_t* d_bounds = lb.tab + rows;
+ const int32_t* d_u_tok = lb.tab + rows + 2 * lm_entries_max(rows, G);
+ const int32_t* d_u_off = d_u_tok + rows;
+ const int32_t* d_u_rows = d_u_off + rows + lm_entries_max(rows, G);
+ const float* d_u_w = (const float*) (d_u_rows + rows);
+ ms_host_tables += ms_since(th1);
+ // (the entries' last index per expert, for releasing a staged blob's ring slot)
+ std::vector<size_t> last_ent;
+ for (size_t j = 0; j < ents.size(); ++j) {
+ if (j == 0 || ents[j - 1].e != ents[j].e) last_ent.push_back(j);
+ else last_ent.back() = j;
+ }
+ int stage_next = 0;
+ auto stage_one = [&](size_t u) -> bool {
+ const int32_t e = uniq[u];
+ if (resident(e)) return true;
+ if (m.src == nullptr) { err = "prefill (layer-major): an expert is neither resident nor in a source"; return false; }
+ const int sl = stage_next;
+ stage_next = (stage_next + 1) % STAGE;
+ const auto tq = Clock::now();
+ const bool pinned = m.src->pinned(l, e);
+ const uint8_t* b = pinned ? m.src->blob_stable(l, e) : nullptr;
+ if (pinned && !b) { err = "prefill (layer-major): expert source has no blob"; return false; }
+ if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
+ if (pinned) {
+ cudaMemcpyAsync(m.stage_dev[sl], b, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);
+ ++stats_.experts_dma;
+ } else {
+ m.stager->pass(job_of[u]); // the prefetched jobs below it the routing skipped get no DMA
+ const uint8_t* hb = m.stager->wait(job_of[u]);
+ cudaMemcpyAsync(m.stage_dev[sl], hb, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);
+ m.stager->issued_one(job_of[u], m.copy);
+ }
+ cudaEventRecord(m.copied[sl], m.copy);
+ m.stage_live[sl] = true;
+ stage_of[u] = sl;
+ stats_.ms_experts_host += ms_since(tq);
+ ++stats_.experts_streamed;
+ ++lm_streamed;
+ lm_bytes += (uint64_t) lay.blob_bytes(l);
+ return true;
+ };
+ const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;
+ const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;
+ const size_t mmq_gub = mmq::matrix_bytes(mmq_gt, 1280, N), mmq_db = mmq::matrix_bytes(mmq_dt, N, 640);
+ size_t staged = 0, u = 0;
+ const uint8_t* blob_dev = nullptr;
+ size_t gi = 0;
+ for (size_t j = 0; j < ents.size(); ++j) {
+ const int32_t e = ents[j].e;
+ if (j == 0 || ents[j - 1].e != e) { // the next expert: staged ahead, waited for, its blob located
+ if (j > 0) ++u;
+ while (staged < uniq.size() && staged <= u + (size_t) STAGE - 1) {
+ if (!stage_one(staged)) return false;
+ ++staged;
+ }
+ if (stage_of[u] < 0) {
+ blob_dev = m.cache->device_slot(m.host_res[(size_t) l * NE + e]);
+ ++stats_.experts_resident;
+ ++lm_resident;
+ } else {
+ pt.mark(kPfWaitCopy, cs);
+ cudaStreamWaitEvent(cs, m.copied[stage_of[u]], 0);
+ blob_dev = m.stage_dev[stage_of[u]];
+ }
+ }
+ const Grp& gp = grps[gi];
+ const size_t q = j - gp.j0;
+ pt.mark(kPfDequant, cs);
+ if (lay.native) {
+ const auto& f = lay.fmt[(size_t) l];
+ mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off, mmq_db,
+ m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, cs);
+ } else {
+ mmq::gather_strata_q2(blob_dev, m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, cs);
+ }
+ if (j == last_ent[u] && stage_of[u] >= 0) { // the blob is read: its ring slot is free
+ cudaEventRecord(m.used[stage_of[u]], cs);
+ m.used_of[stage_of[u]] = stage_of[u];
+ }
+ if (j + 1 < gp.j1) continue;
+ // ---- the group's products over all of its rows of the super-chunk, then the rows added per token
+ const int ngx = (int) (gp.j1 - gp.j0);
+ pt.mark(kPfGemmGU, cs);
+ cudaMemsetAsync(m.grp_gu + (size_t) ngx * mmq_gub, 0, MMQ_TAIL, cs);
+ cudaMemsetAsync(m.grp_d + (size_t) ngx * mmq_db, 0, MMQ_TAIL, cs);
+ mmq::quantize(lb.mixed, d_row_tok + gp.r0, m.Xq, mmq_gt, N, N, gp.nr, cs);
+ mmq::Product gu;
+ gu.w = m.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;
+ gu.n = ngx; gu.xq = m.Xq; gu.bounds = d_bounds + gp.b_at; gu.ids = m.ids_identity;
+ gu.total_rows = gp.nr; gu.max_rows = gp.maxr; gu.dst = m.GU; gu.ld_dst = 1280;
+ m.mmq_ctx->run(gu, cs);
+ mmq::swiglu(m.GU, m.H, gp.nr, 640, !lay.native, cs);
+ pt.mark(kPfGemmD, cs);
+ mmq::quantize(m.H, nullptr, m.Hq, mmq_dt, 640, 640, gp.nr, cs);
+ mmq::Product dn;
+ dn.w = m.grp_d; dn.type = mmq_dt; dn.w_rows = N; dn.w_cols = 640; dn.expert_bytes = mmq_db;
+ dn.n = ngx; dn.xq = m.Hq; dn.bounds = d_bounds + gp.b_at; dn.ids = m.ids_identity;
+ dn.total_rows = gp.nr; dn.max_rows = gp.maxr; dn.dst = m.Dm; dn.ld_dst = N;
+ m.mmq_ctx->run(dn, cs);
+ pt.mark(kPfCombine, cs);
+ moe_scatter_add(m.Dm, d_u_tok + gp.u_at, d_u_off + gp.uo_at, d_u_rows + gp.r0,
+ d_u_w + gp.r0, gp.nu, lb.acc, cs);
+ ++lm_groups;
+ ++gi;
+ core::progress_beat();
+ }
+ ms_experts += ms_since(th0);
+ return true;
+ };
+
+ for (int64_t sc0 = 0; sc0 < n; sc0 += S) {
+ const int64_t Sn = std::min(S, n - sc0);
+ const int64_t nc = (Sn + m.T - 1) / m.T;
+ ++lm_supers;
+ lm_mid_ = true;
+ for (int64_t l = 0; l <= L; ++l) {
+ const auto ta = Clock::now();
+ if (l < L && !lm_prefetch(l)) return false;
+ // host mode: sub-chunk c's rows go up on the copy stream into buffer c % 2 (issued one sub-chunk ahead)
+ auto r_buf = [&](int64_t c) { return (c & 1) ? lb.R2 : R_chunk; };
+ auto issue_up = [&](int64_t c) {
+ const int64_t cc0 = sc0 + c * m.T, TT = std::min(m.T, n - cc0);
+ cudaMemcpyAsync(r_buf(c), lb.R_host + (size_t) (c * m.T) * D, (size_t) TT * D * 4, cudaMemcpyHostToDevice,
+ m.copy);
+ cudaEventRecord(lb.ev_in[c & 1], m.copy);
+ };
+ if (!lb.r_dev && l > 0) issue_up(0);
+ int64_t qsa_index = 0, gdn_index = 0;
+ for (int64_t i = 0; i < std::min(l, L); ++i) (core::is_qsa_layer(g, i) ? qsa_index : gdn_index) += 1;
+ const int64_t qsa_index0 = qsa_index, gdn_index0 = gdn_index;
+ for (int64_t c = 0; c < nc; ++c) {
+ if (should_stop && should_stop()) { err = "cancelled"; return false; }
+ core::progress_beat();
+ core::progress_at("reading the prompt (layer-major): layer", l, pos0 + sc0 + c * m.T);
+ qsa_index = qsa_index0;
+ gdn_index = gdn_index0;
+ const int64_t c0 = sc0 + c * m.T; // in this run's tokens
+ const int64_t T = std::min(m.T, n - c0), p0 = pos0 + c0;
+ const int64_t t0 = c * m.T; // in the super-chunk
+ m.R = lb.r_dev ? lb.R + t0 * D : r_buf(c);
+ if (!lb.r_dev) {
+ if (l > 0) {
+ if (c + 1 < nc) issue_up(c + 1);
+ cudaStreamWaitEvent(cs, lb.ev_in[c & 1], 0);
+ } else {
+ cudaStreamWaitEvent(cs, lb.ev_out[c & 1], 0); // its last download (sub-chunk c - 2) is done
+ }
+ }
+ m.mixed = lb.mixed + t0 * N;
+ m.inj = lb.inj + t0 * HC;
+ float* const acc = lb.acc + t0 * N;
+ if (l == 0) {
+ // ---- embeddings, broadcast to the four streams (run_impl's C-4 batched gather, else per token)
+ pt.mark(kPfStart, cs);
+ bool batched = m.tok_dev != nullptr;
+ const core::NativeEmbed* nemb = core::native_embed();
+ const core::WeightRef* wemb = nemb ? nullptr : m.wt->find("token_embd.weight");
+ if (batched && nemb == nullptr &&
+ (wemb == nullptr || wemb->codebook_iq4nl || wemb->ne0 != g.n_embd || wemb->group_elems <= 0 ||
+ (wemb->code_bits != 2 && wemb->code_bits != 4 && wemb->code_bits != 8)))
+ batched = false;
+ for (int64_t t = 0; batched && t < T; ++t) {
+ const int64_t tok = tokens[c0 + t];
+ if (tok < 0 || (wemb && tok >= wemb->ne1)) batched = false;
+ else m.tok_host[(size_t) t] = (int32_t) tok;
+ }
+ if (batched) {
+ if (!m.tok_up.upload(m.tok_dev, m.tok_host.data(), (size_t) T, cs)) {
+ err = "prefill: the token id upload failed";
+ return false;
+ }
+ if (nemb) {
+ nemb->gather_dev(m.tok_dev, T, m.emb, cs);
+ } else {
+ const auto* codes = (const uint8_t*) wemb->data;
+ const auto* scales = (const float*) (codes + wemb->codes_bytes);
+ const auto* offsets = wemb->has_offset ? (const float*) (codes + wemb->codes_bytes + wemb->scales_bytes)
+ : nullptr;
+ strata::kernels::embedding_gather_dev(codes, scales, offsets, m.tok_dev, (int) T, wemb->ne0,
+ wemb->code_bits, wemb->code_bias, wemb->group_elems,
+ (uint64_t) (wemb->ne0 / (8 / wemb->code_bits)),
+ (uint64_t) (wemb->ne0 / wemb->group_elems), m.emb, cs);
+ }
+ }
+ for (int64_t t = 0; !batched && t < T; ++t)
+ if (!core::embed_row(*m.wt, g, tokens[c0 + t], m.emb + t * N, cs, err)) return false;
+ gr_broadcast(m.emb, m.R, T, cs);
+ if (ple_on && !ple_next.valid())
+ ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c0, b = ple_buf] {
+ strata::aux_cpus::pin_current_thread();
+ return ple_gather(c0, b, ple_next_err);
+ });
+ for (int64_t t = 0; t < T; ++t) { prev[0] = prev[1]; prev[1] = (int32_t) tokens[c0 + t]; }
+ } else {
+ // the previous layer's experts: the hyper-connection write of its FFN half
+ gr_write(m.R, acc, m.inj, HC, T, cs);
+ if (strata::kernels::cvec().covers(l - 1))
+ strata::kernels::cvec_apply(m.R, l - 1, T, D, nullptr, 0, nullptr, 0, false, cs);
+ }
+ if (l == L) {
+ // ---- the final rows of this sub-chunk: reported in order, as run_impl's chunks are
+ stats_.tokens += T;
+ ++stats_.chunks;
+ if (const char* dump = std::getenv("STRATA_PREFILL_DUMP_R")) { // debug: run_impl's dump format
+ cudaStreamSynchronize(cs);
+ if (std::FILE* f = std::fopen(dump, c0 == 0 ? "wb" : "ab")) {
+ std::vector<float> row((size_t) D);
+ for (int64_t t = (64 - p0 % 64) % 64; t < T; t += 64) {
+ cudaMemcpy(row.data(), m.R + t * D, (size_t) D * 4, cudaMemcpyDeviceToHost);
+ const int64_t pos = p0 + t;
+ std::fwrite(&pos, sizeof pos, 1, f);
+ std::fwrite(row.data(), 4, row.size(), f);
+ }
+ std::fclose(f);
+ }
+ }
+ if (on_chunk || on_stage_chunk) {
+ if (cudaStreamSynchronize(cs) != cudaSuccess) {
+ err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ lm_mid_ = c + 1 < nc;
+ if (on_stage_chunk && !on_stage_chunk(p0 + T, err)) return false;
+ if (on_chunk && !on_chunk(m.R, T, p0, err)) return false;
+ }
+ continue;
+ }
+ const core::LayerView v(*m.wt, l);
+ if (l == 1 && ple_on && !ple_land(c0, T)) return false;
+ // ---- the PLE block at layer 1 (run_impl's, batched or token by token)
+ if (l == 1 && ple_on && ple_batch) {
+ pt.mark(kPfPle, cs);
+ const auto tp = Clock::now();
+ const strata::kernels::PleWeights& pw = ss.ple.w;
+ constexpr int64_t HD = strata::kernels::NG_HC_DIM;
+ const uint64_t per_token = (uint64_t) (3 * HD + N + 4) * 4 + (uint64_t) N * (bf16x2(m.f16_io) ? 4 : 2) + 4096;
+ const int64_t SB = std::min<int64_t>(T, (int64_t) (m.region_bytes / per_token));
+ for (int64_t s0 = 0; s0 < T; s0 += SB) {
+ const int64_t nb = std::min(SB, T - s0);
+ uint8_t* q = m.region;
+ auto carve_f = [&](size_t nn) { float* p = (float*) q; q += (nn * 4 + 255) & ~(size_t) 255; return p; };
+ float* key = carve_f((size_t) nb * HD);
+ float* qn = carve_f((size_t) nb * HD);
+ float* gated = carve_f((size_t) nb * HD);
+ float* val = carve_f((size_t) nb * N);
+ float* gate = carve_f((size_t) nb * 4);
+ uint16_t* e16 = (uint16_t*) carve_f((size_t) nb * N / 2);
+ uint16_t* e16_lo = bf16x2(m.f16_io) ? (uint16_t*) carve_f((size_t) nb * N / 2) : nullptr;
+ const float* emb = m.ple_emb + s0 * N;
+ if (pw.key_bf16 != nullptr) {
+ to_bf16(emb, e16, nb * N, cs, e16_lo);
+ m.gemm.bf16(e16, pw.key_bf16, key, nb, HD, N);
+ if (e16_lo) m.gemm.bf16(e16_lo, pw.key_bf16, key, nb, HD, N, 0, 1.0f);
+ } else {
+ to_f16(emb, e16, nb * N, cs);
+ m.gemm.native(e16, pw.key_native_type, pw.key_native_data, key, nb, HD, N);
+ to_bf16(emb, e16, nb * N, cs, e16_lo);
+ }
+ m.gemm.bf16(e16, pw.value_bf16, val, nb, N, N);
+ if (e16_lo) m.gemm.bf16(e16_lo, pw.value_bf16, val, nb, N, N, 0, 1.0f);
+ try {
+ strata::kernels::native_ple_postops_batch(key, m.R + s0 * D, val, ss.ple.hist, pw, qn, gated,
+ gate, (int) nb, cs);
+ } catch (const std::exception& e) { err = std::string("prefill PLE: ") + e.what(); return false; }
+ }
+ stats_.ms_ple += ms_since(tp);
+ } else if (l == 1 && ple_on) {
+ pt.mark(kPfPle, cs);
+ const auto tp = Clock::now();
+ for (int64_t t = 0; t < T; ++t) {
+ strata::kernels::PleOut po;
+ po.normalized = m.ple_norm;
+ po.result = m.R + t * D;
+ try {
+ strata::kernels::ple_block(m.ple_emb + t * N, m.R + t * D, ss.ple.hist, ss.ple.w, po,
+ ss.ple.scratch, cs);
+ } catch (const std::exception& e) { err = std::string("prefill PLE: ") + e.what(); return false; }
+ strata::kernels::ple_history_advance(ss.ple.hist, m.ple_norm, cs);
+ }
+ stats_.ms_ple += ms_since(tp);
+ }
+ if (core::is_qsa_layer(g, l)) { // the QSA step records of this sub-chunk's positions
+ for (int64_t t = 0; t < T; ++t)
+ strata::kernels::qsa_step_fill(m.steps_host.data() + t * strata::kernels::kStepCount, p0 + t, s);
+ // the host table is rewritten by the next QSA sub-chunk: its upload has read it by then (#1996: from a
+ // pinned slot)
+ if (m.up_ring.h2d(m.steps_dev, m.steps_host.data(), (size_t) T * strata::kernels::kStepCount * 4, cs) !=
+ cudaSuccess) {
+ err = "prefill: the QSA step records' upload failed";
+ return false;
+ }
+ }
+ bool normed = false;
+ for (int half = 0; half < 2; ++half) {
+ // ---- the hyper-connection read of this half (run_impl's)
+ const char* pre = half == 0 ? "hc_attn_" : "hc_ffn_";
+ const std::string sn = std::string(pre) + "norm.weight", sd = std::string(pre) + "down.weight",
+ su = std::string(pre) + "up.weight", si = std::string(pre) + "inject.weight";
+ const core::WeightRef *wn = need(v, sn.c_str(), err), *wd = need(v, sd.c_str(), err),
+ *wu = need(v, su.c_str(), err), *wi = need(v, si.c_str(), err);
+ if (!wn || !wd || !wu || !wi) return false;
+ pt.mark(kPfHc, cs);
+ if (gr_unfused()) gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, cs, m.xn16_lo);
+ else if (!normed) gr_norm_rs(m.R, (const float*) wn->data, EPS, m.grs, m.xn16, T, cs, m.xn16_lo);
+ normed = false;
+ if (!bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo)) return false;
+ gr_silu(m.lo, m.lo16, T, cs, m.lo16_lo);
+ if (!bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo)) return false;
+ if (!bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo)) return false;
+ if (gr_unfused()) gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h, m.mixed_bf_lo);
+ else gr_mix_r(m.R, m.grs, (const float*) wn->data, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h,
+ m.mixed_bf_lo);
+ if (half == 0 && !core::is_qsa_layer(g, l)) {
+ // ======================= GDN (run_impl's) =======================
+ const core::WeightRef *wqkv = need(v, "attn_qkv.weight", err), *wg = need(v, "attn_gate.weight", err),
+ *wo = need(v, "ssm_out.weight", err), *wa = need(v, "ssm_alpha.weight", err),
+ *wb = need(v, "ssm_beta.weight", err), *wc = need(v, "ssm_conv1d.weight", err),
+ *wnm = need(v, "ssm_norm.weight", err), *wdt = need(v, "ssm_dt.bias", err),
+ *wsa = need(v, "ssm_a", err);
+ if (!wqkv || !wg || !wo || !wa || !wb || !wc || !wnm || !wdt || !wsa) return false;
+ pt.mark(kPfGdn, cs);
+ float* state = ss.gdn_state + (size_t) (gdn_index - ss.gdn_ord0) * gdn_floats;
+ float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;
+ if (!native_proj(m.gemm, wqkv, m.mixed_h, m.qkv, T, v.name("attn_qkv.weight"), err)) return false;
+ if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;
+ if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;
+ if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;
+ pt.mark(kPfGdnConv, cs);
+ gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, cs);
+ gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, cs);
+ pt.mark(kPfGdnRec, cs);
+ gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, cs);
+ pt.mark(kPfGdnOut, cs);
+ if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err)) return false;
+ ++gdn_index;
+ } else if (half == 0) {
+ // ======================= QSA (run_impl's, without its debug dumps) =======================
+ const core::QsaState& st = ss.qsa_states[qsa_index];
+ const core::WeightRef *wq = need(v, "attn_q.weight", err), *wk = need(v, "attn_k.weight", err),
+ *wv = need(v, "attn_v.weight", err), *wo = need(v, "attn_output.weight", err),
+ *wik = need(v, "indexer.k_proj.weight", err),
+ *wiq = need(v, "indexer.q_proj.weight", err),
+ *wqn = need(v, "attn_q_norm.weight", err), *wkn = need(v, "attn_k_norm.weight", err),
+ *wiqn = need(v, "indexer.q_norm.weight", err),
+ *wikn = need(v, "indexer.k_norm.weight", err);
+ if (!wq || !wk || !wv || !wo || !wik || !wiq || !wqn || !wkn || !wiqn || !wikn) return false;
+ pt.mark(kPfQsa, cs);
+ if (!native_proj(m.gemm, wk, m.mixed_h, m.Kc, T, v.name("attn_k.weight"), err)) return false;
+ if (!native_proj(m.gemm, wv, m.mixed_h, m.Vc, T, v.name("attn_v.weight"), err)) return false;
+ if (!native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;
+ if (!bf16_proj(m.gemm, wik, m.mixed_bf, m.idx_raw, T, v.name("indexer.k_proj.weight"), err, 0, m.mixed_bf_lo)) return false;
+ if (!bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;
+ rms_rows(m.Kc, (const float*) wkn->data, T * 2, 256, 256, EPS, cs);
+ rope(m.Kc, T, 2, 256, 512, p0, strata::kernels::rope_scaling(), cs);
+ const bool staged = st.kv_mode == 1;
+ if (staged) {
+ pt.mark(kPfKvStage, cs);
+ strata::kernels::kv_stage_from_host(pools_of(m.stage, m.ident_table), st.host,
+ core::qsa_kv_format(st),
+ (p0 + s.page_size - 1) / s.page_size, s, cs);
+ pt.mark(kPfQsa, cs);
+ }
+#if defined(STRATA_USE_HIP)
+ static const bool kv_host_dma = [] { const char* e = std::getenv("STRATA_KV_HOST_DMA"); return e && e[0] == '1'; }();
+#else
+ constexpr bool kv_host_dma = false;
+#endif
+ const bool host_by_dma = staged && kv_host_dma;
+ const strata::kernels::KvHostPools* host_w = host_by_dma ? nullptr : &st.host;
+ if (st.kv_hybrid) {
+ // streamed: each half writes its part of the host copy and of the staging pool
+ strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, cs);
+ const strata::kernels::KvHostPools hk = strata::kernels::kv_hybrid_k_half(st.host),
+ hv = strata::kernels::kv_hybrid_v_half(st.host),
+ sk = strata::kernels::kv_hybrid_k_half(m.stage),
+ sv = strata::kernels::kv_hybrid_v_half(m.stage);
+ const bool mirror = st.host.present();
+ kv_append(m.Kc, m.Kc, T, p0, st.page_table, s.page_size, nullptr, nullptr,
+ st.k_q, st.k_q, st.k_scale, st.k_scale, cs, mirror ? &hk : nullptr,
+ staged ? &sk : nullptr);
+ strata::kernels::kv_append_q4(st.v_q4, st.v_q4, st.page_table, p0, T, m.Vc, m.Vc, s, cs,
+ mirror ? &hv : nullptr, staged ? &sv : nullptr);
+ } else {
+ if (st.kv_rot) {
+ strata::kernels::fwht256_inplace_cuda(m.Kc, T * 2, cs);
+ strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, cs);
+ }
+ if (st.kv_q4)
+ strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, p0, T, m.Kc, m.Vc, s, cs,
+ host_w, staged ? &m.stage : nullptr);
+ else
+ kv_append(m.Kc, m.Vc, T, p0, st.page_table, s.page_size, st.kv_int8 ? nullptr : st.k_pool,
+ st.kv_int8 ? nullptr : st.v_pool, st.k_q, st.v_q, st.k_scale, st.v_scale, cs,
+ host_w, staged ? &m.stage : nullptr);
+ if (host_by_dma)
+ strata::kernels::kv_unstage_to_host(pools_of(m.stage, m.ident_table), st.host,
+ core::qsa_kv_format(st), p0 / s.page_size,
+ (p0 + T + s.page_size - 1) / s.page_size, s, cs);
+ }
+ split_q(m.Qf, m.q, T, cs);
+ rms_rows(m.q, (const float*) wqn->data, T * 24, 256, 256, EPS, cs);
+ rope(m.q, T, 24, 256, 6144, p0, strata::kernels::rope_scaling(), cs);
+ if (st.kv_rot) strata::kernels::fwht256_inplace_cuda(m.q, T * 24, cs);
+ rms_rows(m.q_idx, (const float*) wiqn->data, T * 4, 128, 128, EPS, cs);
+ rope(m.q_idx, T, 4, 128, 512, p0, strata::kernels::rope_scaling(), cs);
+ const strata::kernels::QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos, st.idx_pooled_i8, st.idx_pooled_scale};
+ pt.mark(kPfQsaIdx, cs);
+ try {
+ static const bool per_token = std::getenv("STRATA_INDEXER_PER_TOKEN") != nullptr;
+ if (!per_token) {
+ strata::kernels::native_qsa_indexer_append_batch(m.idx_raw, T, p0, 0, (const float*) wikn->data,
+ EPS, ib, s, st.max_cells,
+ strata::kernels::rope_scaling(), cs);
+ }
+ for (int64_t t = 0; per_token && t < T; ++t) {
+ const int32_t* step_t = m.steps_dev + t * strata::kernels::kStepCount;
+ strata::kernels::native_qsa_indexer_append(m.idx_raw + t * 128, step_t + strata::kernels::kStepPos, 0,
+ (const float*) wikn->data, EPS, ib, s, st.max_cells,
+ strata::kernels::rope_scaling(), cs);
+ }
+ } catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; }
+ pt.mark(kPfQsaSel, cs);
+ for (int64_t q0 = 0; q0 < T; q0 += m.sel_batch) {
+ const int64_t nb = std::min(m.sel_batch, T - q0);
+ const int32_t* steps0 = m.steps_dev + q0 * strata::kernels::kStepCount;
+ const int64_t active = (int64_t) m.steps_host[(size_t) ((q0 + nb - 1) * strata::kernels::kStepCount +
+ strata::kernels::kStepNBid)] + 1;
+ static const bool old_sel = std::getenv("STRATA_SELECT_OLD") != nullptr;
+ if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512,
+ steps0, nb, m.max_blocks, s, m.sel_scores,
+ cs, active))
+ strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512, steps0, nb,
+ m.max_blocks, s, m.sel_scores, cs, active);
+ strata::kernels::qsa_block_topk(m.sel_scores, steps0, nb, m.max_blocks, m.cap, s,
+ m.sel_ids + q0 * m.cap, cs, active);
+ }
+ const strata::kernels::QsaAttnPools pools = staged ? pools_of(m.stage, m.ident_table)
+ : core::qsa_attn_pools(st);
+ pt.mark(kPfQsaAttn, cs);
+ static const bool old_attn = std::getenv("STRATA_PROMPT_ATTN_OLD") != nullptr;
+ if (old_attn || !strata::kernels::qsa_prompt_attn_batch(m.q, pools, m.sel_ids, m.steps_dev, m.cap, s,
+ m.attn, T, cs))
+ for (int64_t q0 = 0; q0 < T; q0 += m.attn_batch) {
+ const int64_t nb = std::min(m.attn_batch, T - q0);
+ strata::kernels::qsa_decode_attn_batch(m.q + q0 * ZV, pools, m.sel_ids + q0 * m.cap,
+ m.steps_dev + q0 * strata::kernels::kStepCount, m.cap,
+ s, m.attn_scratch, m.attn + q0 * ZV, nb, cs);
+ }
+ if (st.kv_rot || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, cs);
+ pt.mark(kPfQsa, cs);
+ gate_attn(m.attn, m.Qf, m.attn_h, T, cs);
+ if (!native_proj(m.gemm, wo, m.attn_h, m.bo, T, v.name("attn_output.weight"), err)) return false;
+ ++qsa_index;
+ } else {
+ // ======================= the FFN half: router + shared expert now, the experts later =========
+ const core::WeightRef *wr = need(v, "ffn_gate_inp.weight", err),
+ *wgi = need(v, "ffn_gate_inp_shexp.weight", err),
+ *wsg = need(v, "ffn_gate_shexp.weight", err),
+ *wsu = need(v, "ffn_up_shexp.weight", err),
+ *wsd = need(v, "ffn_down_shexp.weight", err);
+ if (!wr || !wgi || !wsg || !wsu || !wsd) return false;
+ pt.mark(kPfRouter, cs);
+ if (!bf16_proj(m.gemm, wr, m.mixed_bf, m.logits, T, v.name("ffn_gate_inp.weight"), err, 0, m.mixed_bf_lo)) return false;
+ route(m.logits, lb.ids + t0 * K, lb.w + t0 * K, T, m.g->n_expert, cs);
+ if (!native_proj(m.gemm, wsg, m.mixed_h, m.sgate, T, v.name("ffn_gate_shexp.weight"), err)) return false;
+ if (!native_proj(m.gemm, wsu, m.mixed_h, m.sup, T, v.name("ffn_up_shexp.weight"), err)) return false;
+ swiglu_pair(m.sgate, m.sup, m.sh_h, T, cs);
+ if (!native_proj(m.gemm, wsd, m.sh_h, m.shared, T, v.name("ffn_down_shexp.weight"), err)) return false;
+ if (wgi->kind != core::WeightKind::Bf16InF32) { err = "prefill: shared gate is not BF16"; return false; }
+ m.gemm.bf16(m.mixed_bf, (const uint16_t*) wgi->data, m.sg, T, 1, N);
+ if (m.mixed_bf_lo) m.gemm.bf16(m.mixed_bf_lo, (const uint16_t*) wgi->data, m.sg, T, 1, N, 0, 1.0f);
+ moe_shared_init(m.shared, m.sg, acc, T, cs);
+ break; // the FFN half's write waits for the experts (the next layer, or the final pass)
+ }
+ // ---- the attention half's write, fused with the FFN half's norm (run_impl's F-2)
+ const core::WeightRef* wnn = nullptr;
+ if (!gr_unfused()) {
+ wnn = need(v, "hc_ffn_norm.weight", err);
+ if (!wnn) return false;
+ }
+ if (wnn) {
+ gr_write_norm_rs(m.R, m.bo, m.inj, HC, (const float*) wnn->data, EPS, m.grs, m.xn16, T, cs,
+ m.xn16_lo);
+ normed = true;
+ } else {
+ gr_write(m.R, m.bo, m.inj, HC, T, cs);
+ }
+ }
+ if (!lb.r_dev) { // down on the copy stream once this sub-chunk's work is done
+ cudaEventRecord(lb.ev_done[c & 1], cs);
+ cudaStreamWaitEvent(m.copy, lb.ev_done[c & 1], 0);
+ if (cudaMemcpyAsync(lb.R_host + (size_t) t0 * D, m.R, (size_t) T * D * 4, cudaMemcpyDeviceToHost,
+ m.copy) != cudaSuccess) {
+ err = "prefill (layer-major): the residual rows' download failed";
+ return false;
+ }
+ cudaEventRecord(lb.ev_out[c & 1], m.copy);
+ }
+ }
+ ms_attn += ms_since(ta);
+ if (l == L) break;
+ if (!lm_experts(l, Sn)) return false;
+ }
+ }
+ lm_mid_ = false;
+ ss.ple_prev[0] = prev[0];
+ ss.ple_prev[1] = prev[1];
+ if (ple_next.valid()) ple_next.wait();
+ if (cudaStreamSynchronize(cs) != cudaSuccess) {
+ err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ if (const cudaError_t cst = cudaStreamSynchronize(m.copy); cst != cudaSuccess) {
+ err = std::string("prefill: expert copy stream: ") + cudaGetErrorString(cst);
+ return false;
+ }
+ stats_.ms_total += ms_since(t_start);
+ std::fprintf(stderr, "strata prefill: layer-major: %lld tokens in %lld super-chunk(s) of up to %lld (R in %s, %lld MiB VRAM free before), "
+ "%lld expert blobs streamed (%.1f MB), %lld resident, %lld MMQ groups; %.0f ms (host: "
+ "enqueueing the attention halves %.0f, the experts %.0f: waiting for the attention halves "
+ "%.0f, the routing tables %.0f)\n",
+ (long long) n, (long long) lm_supers, (long long) S, lb.r_dev ? "VRAM" : "pinned RAM", (long long) lb.free_mib,
+ (long long) lm_streamed, (double) lm_bytes / 1e6, (long long) lm_resident, (long long) lm_groups,
+ ms_since(t_start), ms_attn, ms_experts, ms_wait_a, ms_host_tables);
+ if (pt.on) {
+ pt.fold();
+ double total = 0.0;
+ for (double x : pt.ms) total += x;
+ std::string line;
+ char b[96];
+ for (int i = 0; i < kPfCount; ++i) {
+ if (pt.ms[i] <= 0.0) continue;
+ std::snprintf(b, sizeof b, " %s %.0f (%.1f%%)", kPfNames[i], pt.ms[i], total > 0 ? 100.0 * pt.ms[i] / total : 0.0);
+ line += b;
+ }
+ std::fprintf(stderr, "strata prefill timing (layer-major): %lld tokens, GPU timeline %.0f ms:%s\n",
+ (long long) n, total, line.c_str());
+ }
+ return true;
+}
+
bool Prefill::drain_pipeline(std::string& err) {
bool ok = true;
@@ -5585,6 +6576,20 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
return ok;
}
#endif
+ if (lm_env() && hand_in_ == nullptr && impl_) {
+ std::string why;
+ if (lm_prepare(n, why) > 0) {
+ const bool ok = run_layer_major(tokens, n, pos0, err);
+ lm_mid_ = false;
+ lm_release();
+ return ok;
+ }
+ static std::string said; // once per reason (a short prompt is the common one)
+ if (why != said) {
+ said = why;
+ std::fprintf(stderr, "strata prefill: layer-major off for this prompt: %s\n", why.c_str());
+ }
+ }
const bool body_ok = run_impl(tokens, n, pos0, err);
std::string drain_err;
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 54d221cd4..3b6537ce4 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -10127,12 +10127,15 @@ int main(int argc, char** argv) {
strata::core::progress_at("reading the prompt (batched), done up to token", done);
strata::core::progress_beat();
std::fflush(stdout);
- const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check;
+ // layer-major prefill: a sub-chunk before its super-chunk's end has the super-chunk's state - the
+ // checkpoint is taken at that end instead
+ const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check && !sp.layer_major_mid();
// Keep the prefill's existing chunk geometry. One extra near-tail checkpoint
// can serve a branch whose shared prefix ends before the final cached state.
const bool tail_checkpoint = o.prompt_cache_tail && o.prompt_cache > 0 && o.prompt_cache_every > 0 &&
!multi_gpu && !pp_tail_saved && pp_total - done > std::max<int64_t>(1, o.short_read) &&
- pp_total - done <= T; // this chunk's real size (--prefill auto chunks differ from o.prefill_chunk)
+ pp_total - done <= T && // this chunk's real size (--prefill auto chunks differ from o.prefill_chunk)
+ !sp.layer_major_mid();
if (periodic_checkpoint || tail_checkpoint) {
bool saved = false;
if (multi_gpu) { // the stages' parts, saved when each of them read this chunk