Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1) (#3976, #1670)
vizdatomffe40c1695e9
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex 39ed85577..0861279af 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -257,10 +257,20 @@ public:/// the share is on for chunks below 1024 tokens (STRATA_PREFILL_CPU_SHARE=0 turns it off).static void arm_cpu_share(bool applies, bool by_default = false);+ /// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): true while `on_chunk` reports a sub-chunk before the end of+ /// its super-chunk. The session state is already at the super-chunk's end then, so no checkpoint may be taken.+ bool layer_major_mid() const { return lm_mid_; }+private:static uint64_t bytes_needed_impl(const core::ModelGeometry& g, const core::SessionState& ss, int64_t chunk,bool owned_pages, bool src);kernels::cpu::ExpertPool* cpu_pool_ = nullptr; ///< set_cpu_pool+ // Layer-major prefill: the super-chunk's buffers for an n-token prompt (its size, 0 = not this prompt: `why`),+ // the run, and the buffers given back after it.+ int64_t lm_prepare(int64_t n, std::string& why);+ bool run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);+ void lm_release();+ bool lm_mid_ = false;// Stage-1 pipeline: intermediate stages return after handing their chunk to// the direct successor. The public run() drains the chain once at prompt end.bool run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex e1f221c76..7a8a8de2f 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1092,6 +1092,27 @@ struct Prefill::Impl {// layer split: the next stage's GPU as a stream-only peer (set_stage_helper); its buffers are that stage's own// prompt buffers, bound per prompt, and it takes the place of `pp` for a run it helpsstd::unique_ptr<PeerPrefill> help_pp;+ // layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1): a super-chunk's state between a layer's attention halves+ // and its experts - device buffers per prompt (lm_prepare / lm_release), pinned host buffers kept+ struct LmBufs {+ int64_t S = 0; // tokens of a super-chunk (0: not allocated)+ bool r_dev = false; // its residual rows in VRAM (else R_host, copied per sub-chunk)+ void* dev = nullptr; // the one device allocation the pointers below are carved from+ float *mixed = nullptr, *acc = nullptr, *inj = nullptr, *R = nullptr, *w = nullptr;+ int32_t *ids = nullptr, *tab = nullptr;+ size_t tab_n = 0;+ float* R_host = nullptr;+ size_t R_host_n = 0;+ int32_t* tab_host = nullptr;+ size_t tab_host_n = 0;+ int32_t* idsw_host = nullptr;+ size_t idsw_host_n = 0;+ int64_t free_mib = 0; // VRAM free when the buffers were sized (the log line)+ // host mode: the residual rows of two sub-chunks on the device in turn (the chunk's R and R2), so the copy+ // stream uploads the next one and downloads the last one while the compute stream works on this one+ float* R2 = nullptr;+ cudaEvent_t ev_in[2] = {}, ev_done[2] = {}, ev_out[2] = {};+ } lm;};namespace {@@ -1179,6 +1200,13 @@ void Prefill::release() {if (impl_->kv_released) cudaEventDestroy(impl_->kv_released);if (impl_->kv_ready) cudaEventDestroy(impl_->kv_ready);if (impl_->grp_host) cudaFreeHost(impl_->grp_host);+ if (impl_->lm.dev) cudaFree(impl_->lm.dev);+ if (impl_->lm.R_host) cudaFreeHost(impl_->lm.R_host);+ if (impl_->lm.tab_host) cudaFreeHost(impl_->lm.tab_host);+ if (impl_->lm.idsw_host) cudaFreeHost(impl_->lm.idsw_host);+ for (int b = 0; b < 2; ++b)+ for (cudaEvent_t e : {impl_->lm.ev_in[b], impl_->lm.ev_done[b], impl_->lm.ev_out[b]})+ if (e) cudaEventDestroy(e);impl_->steps_up.release();impl_->tok_up.release();impl_->up_ring.release();@@ -5537,6 +5565,969 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strireturn true;}+// ============================================================================================================+// Layer-major prefill (STRATA_PREFILL_LAYER_MAJOR=1).+//+// The default prompt path reads a chunk (here 512 tokens) through all 48 layers, so every chunk streams every routed+// expert of every layer to the GPU again - ~25-34 GB of expert blobs per chunk on Q2_0, which is what bounds a prompt+// on a small card (PCIe / NVMe). This path reads a SUPER-CHUNK of S tokens layer by layer instead: for layer l it+// runs each sub-chunk's attention half, the FFN half's hyper-connection read, the router and the shared expert+// (in order, so the GDN / QSA / PLE state advance exactly as in the default path), parks what the experts need+// (the FFN input rows, the routing, the shared expert's part of the output) and then runs the layer's experts ONCE+// for all S tokens: each routed expert is staged once, its rows of every sub-chunk computed in one MMQ group, and+// the results added per token into the super-chunk's accumulator. The FFN half's hyper-connection write of layer l+// runs at the start of the sub-chunk's layer l + 1 (or in the final pass). Expert traffic drops by ~S / 512.+//+// The residual rows of the super-chunk live in VRAM when they fit (STRATA_PREFILL_LM_R=vram) or in pinned host+// memory, copied in and out per sub-chunk and layer (host). Only the experts' sum differs from the default path:+// it is added per expert group in id order, not in the router's k order (FP32 rounding), and the MMQ products run on+// other row groupings. on_chunk reports every sub-chunk in order from the final pass; while it reports one before+// the super-chunk's end, layer_major_mid() is true (the session state is the super-chunk's end: no checkpoint).+namespace {+bool lm_env() {+ static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LAYER_MAJOR"); return e && std::atoi(e) != 0; }();+ return v;+}+int64_t lm_tokens_env() { // the super-chunk's target size (0: the default)+ static const int64_t v = [] { const char* e = std::getenv("STRATA_PREFILL_LM_TOKENS"); return e ? (int64_t) std::atoll(e) : (int64_t) 0; }();+ return v;+}+int lm_r_env() { // 0 auto, 1 vram, 2 host+ static const int v = [] {+ const char* e = std::getenv("STRATA_PREFILL_LM_R");+ if (!e) return 0;+ const std::string s(e);+ return s == "vram" ? 1 : s == "host" ? 2 : 0;+ }();+ return v;+}+int64_t lm_margin_bytes() { // VRAM left free while a prompt runs (the verify windows, the driver)+ static const int64_t v = [] {+ const char* e = std::getenv("STRATA_PREFILL_LM_MARGIN_MIB");+ return (int64_t) (e ? std::atoll(e) : 96) << 20;+ }();+ return v;+}+constexpr int64_t kLmDefaultTokens = 8192;+// the device tables of one layer, in int32s: row_tok (rows) | bounds (2 E) | u_tok (rows) | u_off (rows + E) |+// u_rows (rows) | u_w (rows), E = the most entries a layer can split into+int64_t lm_entries_max(int64_t rows, int64_t G) { return NE + rows / std::max<int64_t>(G, 1) + 2; }+int64_t lm_table_ints(int64_t rows, int64_t G) { return 5 * rows + 3 * lm_entries_max(rows, G); }+} // namespace++void Prefill::lm_release() {+ Impl& m = *impl_;+ if (m.lm.dev) {+ cudaStreamSynchronize(m.cs);+ cudaFree(m.lm.dev);+ }+ m.lm.dev = nullptr;+ m.lm.mixed = m.lm.acc = m.lm.inj = m.lm.R = m.lm.w = m.lm.R2 = nullptr;+ m.lm.ids = m.lm.tab = nullptr;+ m.lm.S = 0;+}++int64_t Prefill::lm_prepare(int64_t n, std::string& why) {+ Impl& m = *impl_;+ if (m.g == nullptr) { why = "not initialized"; return 0; }+ const core::ModelGeometry& g = *m.g;+ if (stage_lb_ != 0 || stage_le_ != g.n_layers || next_ != nullptr || hand_in_ != nullptr) { why = "a layer split"; return 0; }+ if (m.pp || helper_ != nullptr || core::peer_portable()) { why = "a peer GPU"; return 0; }+ if (embd_rows != nullptr) { why = "an image prompt"; return 0; }+ // run_impl's debug hooks on the routing and the rows have no layer-major form+ if (std::getenv("STRATA_DBG_FEAT") || std::getenv("STRATA_DBG_ROUTE_OVERRIDE") || std::getenv("STRATA_PREFILL_DUMP_R_ALL")) {+ why = "a routing or row debug dump";+ return 0;+ }+ const MmqPlan& mp = mmq_plan();+ if (!mp.any || (int64_t) mp.layer.size() < g.n_layers) { why = "the experts are not on MMQ"; return 0; }+ for (int64_t l = 0; l < g.n_layers; ++l)+ if (!mp.layer[(size_t) l]) { why = "a layer's experts are not on MMQ"; return 0; }+ if (m.fused_bufs) { why = "the fused expert layout"; return 0; }+ if (!m.grp_gu || !m.grp_d || !m.Xq || !m.GU || !m.H || !m.Hq || !m.Dm || !m.ids_identity || !m.tok_dev) {+ why = "the MoE buffers are missing"; return 0;+ }+ // (an explicit STRATA_PREFILL_LM_TOKENS may be one chunk: the A/B against the default path, experts aside)+ const int64_t min_s = lm_tokens_env() > 0 ? m.T : 2 * m.T;+ if (n < min_s) { why = "a short prompt"; return 0; }+ const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : kLmDefaultTokens;+ int64_t S = std::max<int64_t>(min_s, want / m.T * m.T);+ S = std::min<int64_t>(S, (n + m.T - 1) / m.T * m.T);+ const int64_t G = m.T * K;+ const auto dev_bytes = [&](int64_t s, bool r_dev) {+ const uint64_t per = (uint64_t) (2 * N + HC + 2 * K) * 4 + (r_dev ? (uint64_t) D * 4 : 0);+ return (uint64_t) s * per + (uint64_t) lm_table_ints(s * K, G) * 4 + 4096 ++ (r_dev ? 0 : (uint64_t) m.T * D * 4 + 256);+ };+ size_t free_b = 0, total_b = 0;+ cudaMemGetInfo(&free_b, &total_b);+ const int64_t room = (int64_t) free_b - lm_margin_bytes();+ m.lm.free_mib = (int64_t) (free_b >> 20);+ auto fit = [&](bool r_dev) { // the largest super-chunk (a multiple of the chunk) that fits `room`+ int64_t s = S;+ while (s >= min_s && (int64_t) dev_bytes(s, r_dev) > room) s -= m.T;+ return s >= min_s ? s : (int64_t) 0;+ };+ const int mode = lm_r_env();+ bool r_dev = false;+ int64_t s_use = 0;+ if (mode == 1) { s_use = fit(true); r_dev = true; }+ else if (mode == 2) { s_use = fit(false); }+ else { // auto: VRAM when the whole target fits there, else pinned host (the bigger super-chunk wins)+ const int64_t sv = fit(true);+ if (sv == S) { s_use = sv; r_dev = true; } else s_use = fit(false);+ }+ if (s_use <= 0) {+ why = "no VRAM for a super-chunk (" + std::to_string(free_b >> 20) + " MiB free)";+ return 0;+ }+ Impl::LmBufs& lb = m.lm;+ // pinned host: the routing's copy, the tables, and (host mode) the residual rows+ const size_t idsw = (size_t) (2 * s_use * K), tab = (size_t) lm_table_ints(s_use * K, G);+ auto host_grow = [](int32_t*& p, size_t& have, size_t want_n) -> bool {+ if (have >= want_n) return true;+ if (p) cudaFreeHost(p);+ p = nullptr;+ have = 0;+ if (cudaHostAlloc((void**) &p, want_n * 4, cudaHostAllocDefault) != cudaSuccess) { cudaGetLastError(); p = nullptr; return false; }+ have = want_n;+ return true;+ };+ if (!host_grow(lb.idsw_host, lb.idsw_host_n, idsw) || !host_grow(lb.tab_host, lb.tab_host_n, tab)) {+ why = "no pinned RAM for the routing tables"; return 0;+ }+ if (!r_dev) {+ int32_t* rh = (int32_t*) lb.R_host;+ if (!host_grow(rh, lb.R_host_n, (size_t) s_use * D)) { why = "no pinned RAM for the residual rows"; lb.R_host = nullptr; return 0; }+ lb.R_host = (float*) rh;+ }+ const uint64_t bytes = dev_bytes(s_use, r_dev);+ void* d = nullptr;+ if (cudaMalloc(&d, bytes) != cudaSuccess) {+ cudaGetLastError();+ why = "the super-chunk's buffers (" + std::to_string(bytes >> 20) + " MiB) do not fit";+ return 0;+ }+ lb.dev = d;+ uint8_t* q = (uint8_t*) d;+ auto carve_b = [&](size_t b) { uint8_t* p = q; q += (b + 255) & ~(size_t) 255; return p; };+ lb.mixed = (float*) carve_b((size_t) s_use * N * 4);+ lb.acc = (float*) carve_b((size_t) s_use * N * 4);+ lb.inj = (float*) carve_b((size_t) s_use * HC * 4);+ lb.ids = (int32_t*) carve_b((size_t) s_use * K * 4);+ lb.w = (float*) carve_b((size_t) s_use * K * 4);+ lb.R = r_dev ? (float*) carve_b((size_t) s_use * D * 4) : nullptr;+ lb.tab = (int32_t*) carve_b(tab * 4);+ lb.tab_n = tab;+ lb.R2 = r_dev ? nullptr : (float*) carve_b((size_t) m.T * D * 4);+ for (int b = 0; b < 2; ++b)+ for (cudaEvent_t* e : {&lb.ev_in[b], &lb.ev_done[b], &lb.ev_out[b]})+ if (*e == nullptr && cudaEventCreateWithFlags(e, cudaEventDisableTiming) != cudaSuccess) {+ cudaGetLastError();+ *e = nullptr;+ why = "the copy events";+ lm_release();+ return 0;+ }+ lb.r_dev = r_dev;+ lb.S = s_use;+ return s_use;+}++bool Prefill::run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {+ err.clear();+ Impl& m = *impl_;+ const core::OnDevice on_device(m.device);+ const core::ModelGeometry& g = *m.g;+ core::SessionState& ss = *m.ss;+ const auto t_start = Clock::now();+ const cudaStream_t cs = (cudaStream_t) m.cs;+ Impl::LmBufs& lb = m.lm;+ const int64_t S = lb.S, L = g.n_layers;+ strata::kernels::QsaShapes s = strata::kernels::qsa_real_shapes();+ s.n_head = g.n_head; s.n_head_kv = g.n_head_kv; s.head_dim = g.head_dim; s.idx_n_head = g.idx_q_heads;+ s.idx_dim = g.idx_key_dim;+ const uint64_t gdn_floats = (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size ++ (uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);+ int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};+ PfTimer pt;+ // the MMQ row table lives in the borrowed cache slots when the prompt path borrows: written for every prompt+ mmq::iota(m.ids_identity, m.T * K, m.cs);+ // ---- the PLE rows, read per chunk on a thread as run_impl does (the same state machine, in chunk order)+ const bool ple_on = ss.ple.ready();+ static const bool ple_batch_env = [] {+ const char* v = std::getenv("STRATA_PLE_BATCH");+ return v == nullptr || std::atoi(v) != 0;+ }();+ const bool ple_batch = ple_on && ple_batch_env && strata::kernels::ple_native_postops_enabled() &&+ (ss.ple.w.key_bf16 != nullptr || ss.ple.w.key_native_data != nullptr) &&+ m.region_bytes / ((uint64_t) (3 * strata::kernels::NG_HC_DIM + N + 4) * 4 + (uint64_t) N * 2 + 4096) >= 64;+ const int32_t prev0[2] = {prev[0], prev[1]};+ auto ple_gather = [&m, &ss, tokens, n, prev0](int64_t c0, int buf, std::string& e) -> bool {+ const int64_t T = std::min(m.T, n - c0);+ auto at = [&](int64_t i) { return i < 2 ? prev0[i] : (int32_t) tokens[i - 2]; };+ int32_t pv[2] = {at(c0), at(c0 + 1)};+ for (int64_t t = 0; t < T; ++t) {+ const int32_t tok = (int32_t) tokens[c0 + t];+ strata::kernels::ngram_rows(&tok, pv, 1, ss.ple.consts,+ m.ple_rows[buf].data() + t * strata::kernels::PLE_N_HEADS);+ pv[0] = pv[1];+ pv[1] = tok;+ }+ return ss.ple.table->gather_batch(m.ple_rows[buf].data(), (size_t) T, m.ple_emb_host[buf], e);+ };+ std::string ple_next_err;+ std::future<bool> ple_next;+ int ple_buf = 0;+ auto ple_land = [&](int64_t c0, int64_t T) -> bool {+ const auto tp = Clock::now();+ if (!ple_next.get()) { err = ple_next_err; return false; }+ // #1996: the pageable fallback (no RAM to pin) goes up through a pinned slot; the pinned buffers as before+ if ((m.ple_pageable[ple_buf].empty()+ ? cudaMemcpyAsync(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cudaMemcpyHostToDevice, cs)+ : m.up_ring.h2d(m.ple_emb, m.ple_emb_host[ple_buf], (size_t) T * N * 4, cs)) != cudaSuccess ||+ cudaEventRecord(m.ple_copied[ple_buf], cs) != cudaSuccess) {+ err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ if (c0 + m.T < n) {+ if (cudaEventSynchronize(m.ple_copied[ple_buf ^ 1]) != cudaSuccess) {+ err = std::string("prefill: the PLE rows' upload failed: ") + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c1 = c0 + m.T, b = ple_buf ^ 1] {+ strata::aux_cpus::pin_current_thread();+ return ple_gather(c1, b, ple_next_err);+ });+ }+ ple_buf ^= 1;+ stats_.ms_ple += ms_since(tp);+ return true;+ };+ // the chunk buffers this path points into the super-chunk (restored on every return)+ struct Restore {+ Impl& m;+ float *R, *mixed, *inj;+ ~Restore() { m.R = R; m.mixed = mixed; m.inj = inj; }+ } restore{m, m.R, m.mixed, m.inj};+ float* const R_chunk = m.R;+ const strata::kernels::cpu::ExpertLayout& lay = strata::kernels::cpu::expert_layout();+ double ms_attn = 0, ms_experts = 0, ms_host_tables = 0, ms_wait_a = 0;+ std::vector<int> pf_job((size_t) NE, -1); // the layer's prefetch: expert -> stager job (-1: not streamed)+ // every streamed expert of layer l, in id order, to the stager - started with the layer's attention halves, so+ // the NVMe / page-cache reads overlap them (at a super-chunk's size nearly every expert is routed)+ static const bool pf_on = [] { const char* e = std::getenv("STRATA_LM_PREFETCH"); return e == nullptr || e[0] != '0'; }();+ auto lm_prefetch = [&](int64_t l) -> bool {+ std::fill(pf_job.begin(), pf_job.end(), -1);+ if (m.src == nullptr || !pf_on) return true;+ std::vector<Stager::Job> js;+ const bool pdirect = m.src->prefill_direct();+ for (int32_t e = 0; e < NE; ++e) {+ if ((m.host_res && m.cache && m.host_res[(size_t) l * NE + e] >= 0) || m.src->pinned(l, e)) continue;+ pf_job[(size_t) e] = (int) js.size();+ if (pdirect || m.src->transient(l, e)) {+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});+ continue;+ }+ const uint8_t* b = m.src->blob_stable(l, e);+ if (!b) { err = "prefill (layer-major): expert source has no blob"; return false; }+ js.push_back({b, (size_t) lay.blob_bytes(l)});+ }+ m.stager->start(std::move(js));+ return true;+ };+ struct StagerStop {+ Stager* st;+ ~StagerStop() { if (st) st->finish(); } // an early return in a layer's attention halves+ } stager_stop{m.stager.get()};+ int64_t lm_streamed = 0, lm_resident = 0, lm_groups = 0, lm_supers = 0;+ uint64_t lm_bytes = 0;+ const int64_t G = m.T * K; // the MoE buffers' rows (Xq, GU, H, Hq, Dm): a group's most++ // ---- one layer's experts over the super-chunk's Sn tokens+ auto lm_experts = [&](int64_t l, int64_t Sn) -> bool {+ const auto th0 = Clock::now();+ const int64_t rows = Sn * K;+ int32_t* idsw = lb.idsw_host;+ cudaMemcpyAsync(idsw, lb.ids, (size_t) rows * 4, cudaMemcpyDeviceToHost, cs);+ cudaMemcpyAsync(idsw + rows, lb.w, (size_t) rows * 4, cudaMemcpyDeviceToHost, cs);+ core::progress_at("reading the prompt (layer-major): waiting for the GPU (attention, router) at layer", l, pos0);+ if (cudaStreamSynchronize(cs) != cudaSuccess) {+ err = std::string("prefill (layer-major): ") + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ core::progress_at("reading the prompt (layer-major): the experts of layer", l, pos0);+ pt.fold();+ ms_wait_a += ms_since(th0);+ const auto th1 = Clock::now();+ const int32_t* ids_h = idsw;+ const float* w_h = (const float*) (idsw + rows);+ std::vector<int32_t> cnt((size_t) NE, 0), off((size_t) NE + 1, 0);+ for (int64_t i = 0; i < rows; ++i) {+ const int32_t e = ids_h[(size_t) i];+ if (e < 0 || e >= NE) { err = "prefill (layer-major): routed id out of range"; return false; }+ ++cnt[(size_t) e];+ }+ for (int32_t e = 0; e < NE; ++e) off[(size_t) e + 1] = off[(size_t) e] + cnt[(size_t) e];+ // ---- the routed experts in id order: resident ones from the cache, the others staged once (run_impl's walk)+ // (started before the tables are built: the stager's threads read while the host builds them)+ std::vector<int32_t> uniq;+ for (int32_t e = 0; e < NE; ++e)+ if (cnt[(size_t) e] > 0) uniq.push_back(e);+ auto resident = [&](int32_t e) { return m.host_res && m.cache && m.host_res[(size_t) l * NE + e] >= 0; };+ // the stager was started with every streamed expert of the layer when its attention halves began (the+ // prefetch); an expert the routing skipped is never waited for (a later job's issue covers its buffer)+ std::vector<int> job_of(uniq.size(), -1), stage_of(uniq.size(), -1);+ for (size_t u = 0; u < uniq.size(); ++u) job_of[u] = pf_job[(size_t) uniq[u]];+ for (int32_t e = 0; e < NE; ++e) // prefetched but not routed: not read, if not started yet+ if (cnt[(size_t) e] == 0 && pf_job[(size_t) e] >= 0) m.stager->skip(pf_job[(size_t) e]);+ if (!pf_on && m.src != nullptr) { // STRATA_LM_PREFETCH=0: the routed experts only, started now+ std::vector<Stager::Job> js;+ const bool pdirect = m.src->prefill_direct();+ for (size_t u = 0; u < uniq.size(); ++u) {+ const int32_t e = uniq[u];+ if (resident(e) || m.src->pinned(l, e)) continue;+ job_of[u] = (int) js.size();+ if (pdirect || m.src->transient(l, e)) {+ js.push_back({nullptr, (size_t) lay.blob_bytes(l), m.src, (int32_t) l, e, pdirect});+ continue;+ }+ const uint8_t* b = m.src->blob_stable(l, e);+ if (!b) { err = "prefill (layer-major): expert source has no blob"; return false; }+ js.push_back({b, (size_t) lay.blob_bytes(l)});+ }+ m.stager->start(std::move(js));+ }+ struct StagerDone {+ Stager* st;+ ~StagerDone() { if (st) st->finish(); }+ } stager_done{m.stager.get()};+ // the tables: rows sorted by expert (token order within one), then per group its bounds and per-token CSR+ int32_t* th = lb.tab_host;+ int32_t* row_tok = th;+ std::vector<float> row_w((size_t) rows);+ {+ std::vector<int32_t> fill(off.begin(), off.end() - 1);+ for (int64_t i = 0; i < rows; ++i) {+ const int32_t e = ids_h[(size_t) i];+ const int32_t p = fill[(size_t) e]++;+ row_tok[p] = (int32_t) (i / K);+ row_w[(size_t) p] = w_h[(size_t) i];+ }+ }+ struct Entry { int32_t e, r0, nr; };+ std::vector<Entry> ents;+ for (int32_t e = 0; e < NE; ++e)+ for (int32_t r = off[(size_t) e]; r < off[(size_t) e + 1]; r += (int32_t) G)+ ents.push_back({e, r, (int32_t) std::min<int64_t>(G, off[(size_t) e + 1] - r)});+ struct Grp { size_t j0, j1; int32_t r0, nr, maxr; int64_t b_at, u_at, uo_at, nu; };+ std::vector<Grp> grps;+ for (size_t j = 0; j < ents.size();) {+ Grp gp{};+ gp.j0 = j;+ gp.r0 = ents[j].r0;+ int32_t nr = 0, maxr = 0;+ while (j < ents.size() && j - gp.j0 < (size_t) MMQ_GROUP && nr + ents[j].nr <= G) {+ nr += ents[j].nr;+ maxr = std::max(maxr, ents[j].nr);+ ++j;+ }+ gp.j1 = j;+ gp.nr = nr;+ gp.maxr = maxr;+ grps.push_back(gp);+ }+ const int64_t E = (int64_t) ents.size();+ if (E > lm_entries_max(rows, G)) { err = "prefill (layer-major): more expert entries than the tables hold"; return false; }+ int64_t at = rows;+ int32_t* bounds = th + at;+ int64_t b_used = 0;+ at += 2 * lm_entries_max(rows, G);+ int32_t* u_tok = th + at;+ at += rows;+ int32_t* u_off = th + at;+ at += rows + lm_entries_max(rows, G);+ int32_t* u_rows = th + at;+ at += rows;+ float* u_w = (float*) (th + at);+ at += rows;+ if (at > (int64_t) lb.tab_n || at > (int64_t) lb.tab_host_n) { err = "prefill (layer-major): the tables overflow"; return false; }+ int64_t u_used = 0, uo_used = 0;+ // per group, the tokens in order of first appearance and each token's rows in row order (counting, O(rows));+ // `slot` maps a token to its place in the group (-1: not in it), reset through the group's own token list+ std::vector<int32_t> slot((size_t) Sn, -1), fillp;+ for (Grp& gp : grps) {+ gp.b_at = b_used;+ for (size_t j = gp.j0; j < gp.j1; ++j) bounds[b_used++] = ents[j].r0 - gp.r0;+ bounds[b_used++] = gp.nr;+ gp.u_at = u_used;+ gp.uo_at = uo_used;+ int32_t* ut = u_tok + u_used;+ int32_t* uo = u_off + uo_used;+ int64_t nu = 0;+ for (int32_t r = 0; r < gp.nr; ++r) {+ const int32_t t = row_tok[gp.r0 + r];+ if (slot[(size_t) t] < 0) { slot[(size_t) t] = (int32_t) nu; ut[nu] = t; uo[nu] = 0; ++nu; }+ ++uo[slot[(size_t) t]];+ }+ fillp.assign((size_t) nu, 0);+ for (int64_t i = 0, acc_n = 0; i < nu; ++i) { const int32_t c = uo[i]; uo[i] = (int32_t) acc_n; fillp[(size_t) i] = (int32_t) acc_n; acc_n += c; }+ uo[nu] = gp.nr;+ for (int32_t r = 0; r < gp.nr; ++r) {+ const int32_t p = fillp[(size_t) slot[(size_t) row_tok[gp.r0 + r]]]++;+ u_rows[gp.r0 + p] = r;+ u_w[gp.r0 + p] = row_w[(size_t) (gp.r0 + r)];+ }+ for (int64_t i = 0; i < nu; ++i) slot[(size_t) ut[i]] = -1;+ gp.nu = nu;+ u_used += nu;+ uo_used += nu + 1;+ }+ // (u_rows / u_w are indexed by the group's own row range [r0, r0 + nr) of the layer)+ cudaMemcpyAsync(lb.tab, th, (size_t) at * 4, cudaMemcpyHostToDevice, cs);+ const int32_t* d_row_tok = lb.tab;+ const int32_t* d_bounds = lb.tab + rows;+ const int32_t* d_u_tok = lb.tab + rows + 2 * lm_entries_max(rows, G);+ const int32_t* d_u_off = d_u_tok + rows;+ const int32_t* d_u_rows = d_u_off + rows + lm_entries_max(rows, G);+ const float* d_u_w = (const float*) (d_u_rows + rows);+ ms_host_tables += ms_since(th1);+ // (the entries' last index per expert, for releasing a staged blob's ring slot)+ std::vector<size_t> last_ent;+ for (size_t j = 0; j < ents.size(); ++j) {+ if (j == 0 || ents[j - 1].e != ents[j].e) last_ent.push_back(j);+ else last_ent.back() = j;+ }+ int stage_next = 0;+ auto stage_one = [&](size_t u) -> bool {+ const int32_t e = uniq[u];+ if (resident(e)) return true;+ if (m.src == nullptr) { err = "prefill (layer-major): an expert is neither resident nor in a source"; return false; }+ const int sl = stage_next;+ stage_next = (stage_next + 1) % STAGE;+ const auto tq = Clock::now();+ const bool pinned = m.src->pinned(l, e);+ const uint8_t* b = pinned ? m.src->blob_stable(l, e) : nullptr;+ if (pinned && !b) { err = "prefill (layer-major): expert source has no blob"; return false; }+ if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);+ if (pinned) {+ cudaMemcpyAsync(m.stage_dev[sl], b, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);+ ++stats_.experts_dma;+ } else {+ m.stager->pass(job_of[u]); // the prefetched jobs below it the routing skipped get no DMA+ const uint8_t* hb = m.stager->wait(job_of[u]);+ cudaMemcpyAsync(m.stage_dev[sl], hb, (size_t) lay.blob_bytes(l), cudaMemcpyHostToDevice, m.copy);+ m.stager->issued_one(job_of[u], m.copy);+ }+ cudaEventRecord(m.copied[sl], m.copy);+ m.stage_live[sl] = true;+ stage_of[u] = sl;+ stats_.ms_experts_host += ms_since(tq);+ ++stats_.experts_streamed;+ ++lm_streamed;+ lm_bytes += (uint64_t) lay.blob_bytes(l);+ return true;+ };+ const int mmq_gt = lay.native ? lay.fmt[(size_t) l].gu_type : 42;+ const int mmq_dt = lay.native ? lay.fmt[(size_t) l].d_type : 42;+ const size_t mmq_gub = mmq::matrix_bytes(mmq_gt, 1280, N), mmq_db = mmq::matrix_bytes(mmq_dt, N, 640);+ size_t staged = 0, u = 0;+ const uint8_t* blob_dev = nullptr;+ size_t gi = 0;+ for (size_t j = 0; j < ents.size(); ++j) {+ const int32_t e = ents[j].e;+ if (j == 0 || ents[j - 1].e != e) { // the next expert: staged ahead, waited for, its blob located+ if (j > 0) ++u;+ while (staged < uniq.size() && staged <= u + (size_t) STAGE - 1) {+ if (!stage_one(staged)) return false;+ ++staged;+ }+ if (stage_of[u] < 0) {+ blob_dev = m.cache->device_slot(m.host_res[(size_t) l * NE + e]);+ ++stats_.experts_resident;+ ++lm_resident;+ } else {+ pt.mark(kPfWaitCopy, cs);+ cudaStreamWaitEvent(cs, m.copied[stage_of[u]], 0);+ blob_dev = m.stage_dev[stage_of[u]];+ }+ }+ const Grp& gp = grps[gi];+ const size_t q = j - gp.j0;+ pt.mark(kPfDequant, cs);+ if (lay.native) {+ const auto& f = lay.fmt[(size_t) l];+ mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off, mmq_db,+ m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, cs);+ } else {+ mmq::gather_strata_q2(blob_dev, m.grp_gu + q * mmq_gub, m.grp_d + q * mmq_db, cs);+ }+ if (j == last_ent[u] && stage_of[u] >= 0) { // the blob is read: its ring slot is free+ cudaEventRecord(m.used[stage_of[u]], cs);+ m.used_of[stage_of[u]] = stage_of[u];+ }+ if (j + 1 < gp.j1) continue;+ // ---- the group's products over all of its rows of the super-chunk, then the rows added per token+ const int ngx = (int) (gp.j1 - gp.j0);+ pt.mark(kPfGemmGU, cs);+ cudaMemsetAsync(m.grp_gu + (size_t) ngx * mmq_gub, 0, MMQ_TAIL, cs);+ cudaMemsetAsync(m.grp_d + (size_t) ngx * mmq_db, 0, MMQ_TAIL, cs);+ mmq::quantize(lb.mixed, d_row_tok + gp.r0, m.Xq, mmq_gt, N, N, gp.nr, cs);+ mmq::Product gu;+ gu.w = m.grp_gu; gu.type = mmq_gt; gu.w_rows = 1280; gu.w_cols = N; gu.expert_bytes = mmq_gub;+ gu.n = ngx; gu.xq = m.Xq; gu.bounds = d_bounds + gp.b_at; gu.ids = m.ids_identity;+ gu.total_rows = gp.nr; gu.max_rows = gp.maxr; gu.dst = m.GU; gu.ld_dst = 1280;+ m.mmq_ctx->run(gu, cs);+ mmq::swiglu(m.GU, m.H, gp.nr, 640, !lay.native, cs);+ pt.mark(kPfGemmD, cs);+ mmq::quantize(m.H, nullptr, m.Hq, mmq_dt, 640, 640, gp.nr, cs);+ mmq::Product dn;+ dn.w = m.grp_d; dn.type = mmq_dt; dn.w_rows = N; dn.w_cols = 640; dn.expert_bytes = mmq_db;+ dn.n = ngx; dn.xq = m.Hq; dn.bounds = d_bounds + gp.b_at; dn.ids = m.ids_identity;+ dn.total_rows = gp.nr; dn.max_rows = gp.maxr; dn.dst = m.Dm; dn.ld_dst = N;+ m.mmq_ctx->run(dn, cs);+ pt.mark(kPfCombine, cs);+ moe_scatter_add(m.Dm, d_u_tok + gp.u_at, d_u_off + gp.uo_at, d_u_rows + gp.r0,+ d_u_w + gp.r0, gp.nu, lb.acc, cs);+ ++lm_groups;+ ++gi;+ core::progress_beat();+ }+ ms_experts += ms_since(th0);+ return true;+ };++ for (int64_t sc0 = 0; sc0 < n; sc0 += S) {+ const int64_t Sn = std::min(S, n - sc0);+ const int64_t nc = (Sn + m.T - 1) / m.T;+ ++lm_supers;+ lm_mid_ = true;+ for (int64_t l = 0; l <= L; ++l) {+ const auto ta = Clock::now();+ if (l < L && !lm_prefetch(l)) return false;+ // host mode: sub-chunk c's rows go up on the copy stream into buffer c % 2 (issued one sub-chunk ahead)+ auto r_buf = [&](int64_t c) { return (c & 1) ? lb.R2 : R_chunk; };+ auto issue_up = [&](int64_t c) {+ const int64_t cc0 = sc0 + c * m.T, TT = std::min(m.T, n - cc0);+ cudaMemcpyAsync(r_buf(c), lb.R_host + (size_t) (c * m.T) * D, (size_t) TT * D * 4, cudaMemcpyHostToDevice,+ m.copy);+ cudaEventRecord(lb.ev_in[c & 1], m.copy);+ };+ if (!lb.r_dev && l > 0) issue_up(0);+ int64_t qsa_index = 0, gdn_index = 0;+ for (int64_t i = 0; i < std::min(l, L); ++i) (core::is_qsa_layer(g, i) ? qsa_index : gdn_index) += 1;+ const int64_t qsa_index0 = qsa_index, gdn_index0 = gdn_index;+ for (int64_t c = 0; c < nc; ++c) {+ if (should_stop && should_stop()) { err = "cancelled"; return false; }+ core::progress_beat();+ core::progress_at("reading the prompt (layer-major): layer", l, pos0 + sc0 + c * m.T);+ qsa_index = qsa_index0;+ gdn_index = gdn_index0;+ const int64_t c0 = sc0 + c * m.T; // in this run's tokens+ const int64_t T = std::min(m.T, n - c0), p0 = pos0 + c0;+ const int64_t t0 = c * m.T; // in the super-chunk+ m.R = lb.r_dev ? lb.R + t0 * D : r_buf(c);+ if (!lb.r_dev) {+ if (l > 0) {+ if (c + 1 < nc) issue_up(c + 1);+ cudaStreamWaitEvent(cs, lb.ev_in[c & 1], 0);+ } else {+ cudaStreamWaitEvent(cs, lb.ev_out[c & 1], 0); // its last download (sub-chunk c - 2) is done+ }+ }+ m.mixed = lb.mixed + t0 * N;+ m.inj = lb.inj + t0 * HC;+ float* const acc = lb.acc + t0 * N;+ if (l == 0) {+ // ---- embeddings, broadcast to the four streams (run_impl's C-4 batched gather, else per token)+ pt.mark(kPfStart, cs);+ bool batched = m.tok_dev != nullptr;+ const core::NativeEmbed* nemb = core::native_embed();+ const core::WeightRef* wemb = nemb ? nullptr : m.wt->find("token_embd.weight");+ if (batched && nemb == nullptr &&+ (wemb == nullptr || wemb->codebook_iq4nl || wemb->ne0 != g.n_embd || wemb->group_elems <= 0 ||+ (wemb->code_bits != 2 && wemb->code_bits != 4 && wemb->code_bits != 8)))+ batched = false;+ for (int64_t t = 0; batched && t < T; ++t) {+ const int64_t tok = tokens[c0 + t];+ if (tok < 0 || (wemb && tok >= wemb->ne1)) batched = false;+ else m.tok_host[(size_t) t] = (int32_t) tok;+ }+ if (batched) {+ if (!m.tok_up.upload(m.tok_dev, m.tok_host.data(), (size_t) T, cs)) {+ err = "prefill: the token id upload failed";+ return false;+ }+ if (nemb) {+ nemb->gather_dev(m.tok_dev, T, m.emb, cs);+ } else {+ const auto* codes = (const uint8_t*) wemb->data;+ const auto* scales = (const float*) (codes + wemb->codes_bytes);+ const auto* offsets = wemb->has_offset ? (const float*) (codes + wemb->codes_bytes + wemb->scales_bytes)+ : nullptr;+ strata::kernels::embedding_gather_dev(codes, scales, offsets, m.tok_dev, (int) T, wemb->ne0,+ wemb->code_bits, wemb->code_bias, wemb->group_elems,+ (uint64_t) (wemb->ne0 / (8 / wemb->code_bits)),+ (uint64_t) (wemb->ne0 / wemb->group_elems), m.emb, cs);+ }+ }+ for (int64_t t = 0; !batched && t < T; ++t)+ if (!core::embed_row(*m.wt, g, tokens[c0 + t], m.emb + t * N, cs, err)) return false;+ gr_broadcast(m.emb, m.R, T, cs);+ if (ple_on && !ple_next.valid())+ ple_next = std::async(std::launch::async, [&ple_gather, &ple_next_err, c0, b = ple_buf] {+ strata::aux_cpus::pin_current_thread();+ return ple_gather(c0, b, ple_next_err);+ });+ for (int64_t t = 0; t < T; ++t) { prev[0] = prev[1]; prev[1] = (int32_t) tokens[c0 + t]; }+ } else {+ // the previous layer's experts: the hyper-connection write of its FFN half+ gr_write(m.R, acc, m.inj, HC, T, cs);+ if (strata::kernels::cvec().covers(l - 1))+ strata::kernels::cvec_apply(m.R, l - 1, T, D, nullptr, 0, nullptr, 0, false, cs);+ }+ if (l == L) {+ // ---- the final rows of this sub-chunk: reported in order, as run_impl's chunks are+ stats_.tokens += T;+ ++stats_.chunks;+ if (const char* dump = std::getenv("STRATA_PREFILL_DUMP_R")) { // debug: run_impl's dump format+ cudaStreamSynchronize(cs);+ if (std::FILE* f = std::fopen(dump, c0 == 0 ? "wb" : "ab")) {+ std::vector<float> row((size_t) D);+ for (int64_t t = (64 - p0 % 64) % 64; t < T; t += 64) {+ cudaMemcpy(row.data(), m.R + t * D, (size_t) D * 4, cudaMemcpyDeviceToHost);+ const int64_t pos = p0 + t;+ std::fwrite(&pos, sizeof pos, 1, f);+ std::fwrite(row.data(), 4, row.size(), f);+ }+ std::fclose(f);+ }+ }+ if (on_chunk || on_stage_chunk) {+ if (cudaStreamSynchronize(cs) != cudaSuccess) {+ err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ lm_mid_ = c + 1 < nc;+ if (on_stage_chunk && !on_stage_chunk(p0 + T, err)) return false;+ if (on_chunk && !on_chunk(m.R, T, p0, err)) return false;+ }+ continue;+ }+ const core::LayerView v(*m.wt, l);+ if (l == 1 && ple_on && !ple_land(c0, T)) return false;+ // ---- the PLE block at layer 1 (run_impl's, batched or token by token)+ if (l == 1 && ple_on && ple_batch) {+ pt.mark(kPfPle, cs);+ const auto tp = Clock::now();+ const strata::kernels::PleWeights& pw = ss.ple.w;+ constexpr int64_t HD = strata::kernels::NG_HC_DIM;+ const uint64_t per_token = (uint64_t) (3 * HD + N + 4) * 4 + (uint64_t) N * (bf16x2(m.f16_io) ? 4 : 2) + 4096;+ const int64_t SB = std::min<int64_t>(T, (int64_t) (m.region_bytes / per_token));+ for (int64_t s0 = 0; s0 < T; s0 += SB) {+ const int64_t nb = std::min(SB, T - s0);+ uint8_t* q = m.region;+ auto carve_f = [&](size_t nn) { float* p = (float*) q; q += (nn * 4 + 255) & ~(size_t) 255; return p; };+ float* key = carve_f((size_t) nb * HD);+ float* qn = carve_f((size_t) nb * HD);+ float* gated = carve_f((size_t) nb * HD);+ float* val = carve_f((size_t) nb * N);+ float* gate = carve_f((size_t) nb * 4);+ uint16_t* e16 = (uint16_t*) carve_f((size_t) nb * N / 2);+ uint16_t* e16_lo = bf16x2(m.f16_io) ? (uint16_t*) carve_f((size_t) nb * N / 2) : nullptr;+ const float* emb = m.ple_emb + s0 * N;+ if (pw.key_bf16 != nullptr) {+ to_bf16(emb, e16, nb * N, cs, e16_lo);+ m.gemm.bf16(e16, pw.key_bf16, key, nb, HD, N);+ if (e16_lo) m.gemm.bf16(e16_lo, pw.key_bf16, key, nb, HD, N, 0, 1.0f);+ } else {+ to_f16(emb, e16, nb * N, cs);+ m.gemm.native(e16, pw.key_native_type, pw.key_native_data, key, nb, HD, N);+ to_bf16(emb, e16, nb * N, cs, e16_lo);+ }+ m.gemm.bf16(e16, pw.value_bf16, val, nb, N, N);+ if (e16_lo) m.gemm.bf16(e16_lo, pw.value_bf16, val, nb, N, N, 0, 1.0f);+ try {+ strata::kernels::native_ple_postops_batch(key, m.R + s0 * D, val, ss.ple.hist, pw, qn, gated,+ gate, (int) nb, cs);+ } catch (const std::exception& e) { err = std::string("prefill PLE: ") + e.what(); return false; }+ }+ stats_.ms_ple += ms_since(tp);+ } else if (l == 1 && ple_on) {+ pt.mark(kPfPle, cs);+ const auto tp = Clock::now();+ for (int64_t t = 0; t < T; ++t) {+ strata::kernels::PleOut po;+ po.normalized = m.ple_norm;+ po.result = m.R + t * D;+ try {+ strata::kernels::ple_block(m.ple_emb + t * N, m.R + t * D, ss.ple.hist, ss.ple.w, po,+ ss.ple.scratch, cs);+ } catch (const std::exception& e) { err = std::string("prefill PLE: ") + e.what(); return false; }+ strata::kernels::ple_history_advance(ss.ple.hist, m.ple_norm, cs);+ }+ stats_.ms_ple += ms_since(tp);+ }+ if (core::is_qsa_layer(g, l)) { // the QSA step records of this sub-chunk's positions+ for (int64_t t = 0; t < T; ++t)+ strata::kernels::qsa_step_fill(m.steps_host.data() + t * strata::kernels::kStepCount, p0 + t, s);+ // the host table is rewritten by the next QSA sub-chunk: its upload has read it by then (#1996: from a+ // pinned slot)+ if (m.up_ring.h2d(m.steps_dev, m.steps_host.data(), (size_t) T * strata::kernels::kStepCount * 4, cs) !=+ cudaSuccess) {+ err = "prefill: the QSA step records' upload failed";+ return false;+ }+ }+ bool normed = false;+ for (int half = 0; half < 2; ++half) {+ // ---- the hyper-connection read of this half (run_impl's)+ const char* pre = half == 0 ? "hc_attn_" : "hc_ffn_";+ const std::string sn = std::string(pre) + "norm.weight", sd = std::string(pre) + "down.weight",+ su = std::string(pre) + "up.weight", si = std::string(pre) + "inject.weight";+ const core::WeightRef *wn = need(v, sn.c_str(), err), *wd = need(v, sd.c_str(), err),+ *wu = need(v, su.c_str(), err), *wi = need(v, si.c_str(), err);+ if (!wn || !wd || !wu || !wi) return false;+ pt.mark(kPfHc, cs);+ if (gr_unfused()) gr_norm(m.R, (const float*) wn->data, EPS, m.xn, m.xn16, T, cs, m.xn16_lo);+ else if (!normed) gr_norm_rs(m.R, (const float*) wn->data, EPS, m.grs, m.xn16, T, cs, m.xn16_lo);+ normed = false;+ if (!bf16_proj(m.gemm, wd, m.xn16, m.lo, T, sd, err, 0, m.xn16_lo)) return false;+ gr_silu(m.lo, m.lo16, T, cs, m.lo16_lo);+ if (!bf16_proj(m.gemm, wu, m.lo16, m.gated, T, su, err, 0, m.lo16_lo)) return false;+ if (!bf16_proj(m.gemm, wi, m.xn16, m.inj, T, si, err, 0, m.xn16_lo)) return false;+ if (gr_unfused()) gr_mix(m.xn, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h, m.mixed_bf_lo);+ else gr_mix_r(m.R, m.grs, (const float*) wn->data, m.gated, m.mixed, m.mixed_bf, T, cs, m.mixed_h,+ m.mixed_bf_lo);+ if (half == 0 && !core::is_qsa_layer(g, l)) {+ // ======================= GDN (run_impl's) =======================+ const core::WeightRef *wqkv = need(v, "attn_qkv.weight", err), *wg = need(v, "attn_gate.weight", err),+ *wo = need(v, "ssm_out.weight", err), *wa = need(v, "ssm_alpha.weight", err),+ *wb = need(v, "ssm_beta.weight", err), *wc = need(v, "ssm_conv1d.weight", err),+ *wnm = need(v, "ssm_norm.weight", err), *wdt = need(v, "ssm_dt.bias", err),+ *wsa = need(v, "ssm_a", err);+ if (!wqkv || !wg || !wo || !wa || !wb || !wc || !wnm || !wdt || !wsa) return false;+ pt.mark(kPfGdn, cs);+ float* state = ss.gdn_state + (size_t) (gdn_index - ss.gdn_ord0) * gdn_floats;+ float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;+ if (!native_proj(m.gemm, wqkv, m.mixed_h, m.qkv, T, v.name("attn_qkv.weight"), err)) return false;+ if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;+ if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;+ if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;+ pt.mark(kPfGdnConv, cs);+ gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, cs);+ gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, cs);+ pt.mark(kPfGdnRec, cs);+ gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, cs);+ pt.mark(kPfGdnOut, cs);+ if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err)) return false;+ ++gdn_index;+ } else if (half == 0) {+ // ======================= QSA (run_impl's, without its debug dumps) =======================+ const core::QsaState& st = ss.qsa_states[qsa_index];+ const core::WeightRef *wq = need(v, "attn_q.weight", err), *wk = need(v, "attn_k.weight", err),+ *wv = need(v, "attn_v.weight", err), *wo = need(v, "attn_output.weight", err),+ *wik = need(v, "indexer.k_proj.weight", err),+ *wiq = need(v, "indexer.q_proj.weight", err),+ *wqn = need(v, "attn_q_norm.weight", err), *wkn = need(v, "attn_k_norm.weight", err),+ *wiqn = need(v, "indexer.q_norm.weight", err),+ *wikn = need(v, "indexer.k_norm.weight", err);+ if (!wq || !wk || !wv || !wo || !wik || !wiq || !wqn || !wkn || !wiqn || !wikn) return false;+ pt.mark(kPfQsa, cs);+ if (!native_proj(m.gemm, wk, m.mixed_h, m.Kc, T, v.name("attn_k.weight"), err)) return false;+ if (!native_proj(m.gemm, wv, m.mixed_h, m.Vc, T, v.name("attn_v.weight"), err)) return false;+ if (!native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;+ if (!bf16_proj(m.gemm, wik, m.mixed_bf, m.idx_raw, T, v.name("indexer.k_proj.weight"), err, 0, m.mixed_bf_lo)) return false;+ if (!bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;+ rms_rows(m.Kc, (const float*) wkn->data, T * 2, 256, 256, EPS, cs);+ rope(m.Kc, T, 2, 256, 512, p0, strata::kernels::rope_scaling(), cs);+ const bool staged = st.kv_mode == 1;+ if (staged) {+ pt.mark(kPfKvStage, cs);+ strata::kernels::kv_stage_from_host(pools_of(m.stage, m.ident_table), st.host,+ core::qsa_kv_format(st),+ (p0 + s.page_size - 1) / s.page_size, s, cs);+ pt.mark(kPfQsa, cs);+ }+#if defined(STRATA_USE_HIP)+ static const bool kv_host_dma = [] { const char* e = std::getenv("STRATA_KV_HOST_DMA"); return e && e[0] == '1'; }();+#else+ constexpr bool kv_host_dma = false;+#endif+ const bool host_by_dma = staged && kv_host_dma;+ const strata::kernels::KvHostPools* host_w = host_by_dma ? nullptr : &st.host;+ if (st.kv_hybrid) {+ // streamed: each half writes its part of the host copy and of the staging pool+ strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, cs);+ const strata::kernels::KvHostPools hk = strata::kernels::kv_hybrid_k_half(st.host),+ hv = strata::kernels::kv_hybrid_v_half(st.host),+ sk = strata::kernels::kv_hybrid_k_half(m.stage),+ sv = strata::kernels::kv_hybrid_v_half(m.stage);+ const bool mirror = st.host.present();+ kv_append(m.Kc, m.Kc, T, p0, st.page_table, s.page_size, nullptr, nullptr,+ st.k_q, st.k_q, st.k_scale, st.k_scale, cs, mirror ? &hk : nullptr,+ staged ? &sk : nullptr);+ strata::kernels::kv_append_q4(st.v_q4, st.v_q4, st.page_table, p0, T, m.Vc, m.Vc, s, cs,+ mirror ? &hv : nullptr, staged ? &sv : nullptr);+ } else {+ if (st.kv_rot) {+ strata::kernels::fwht256_inplace_cuda(m.Kc, T * 2, cs);+ strata::kernels::fwht256_inplace_cuda(m.Vc, T * 2, cs);+ }+ if (st.kv_q4)+ strata::kernels::kv_append_q4(st.k_q4, st.v_q4, st.page_table, p0, T, m.Kc, m.Vc, s, cs,+ host_w, staged ? &m.stage : nullptr);+ else+ kv_append(m.Kc, m.Vc, T, p0, st.page_table, s.page_size, st.kv_int8 ? nullptr : st.k_pool,+ st.kv_int8 ? nullptr : st.v_pool, st.k_q, st.v_q, st.k_scale, st.v_scale, cs,+ host_w, staged ? &m.stage : nullptr);+ if (host_by_dma)+ strata::kernels::kv_unstage_to_host(pools_of(m.stage, m.ident_table), st.host,+ core::qsa_kv_format(st), p0 / s.page_size,+ (p0 + T + s.page_size - 1) / s.page_size, s, cs);+ }+ split_q(m.Qf, m.q, T, cs);+ rms_rows(m.q, (const float*) wqn->data, T * 24, 256, 256, EPS, cs);+ rope(m.q, T, 24, 256, 6144, p0, strata::kernels::rope_scaling(), cs);+ if (st.kv_rot) strata::kernels::fwht256_inplace_cuda(m.q, T * 24, cs);+ rms_rows(m.q_idx, (const float*) wiqn->data, T * 4, 128, 128, EPS, cs);+ rope(m.q_idx, T, 4, 128, 512, p0, strata::kernels::rope_scaling(), cs);+ const strata::kernels::QsaIndexerBuffers ib{st.idx_tail, st.idx_dead, st.idx_pooled, st.idx_block_pos, st.idx_pooled_i8, st.idx_pooled_scale};+ pt.mark(kPfQsaIdx, cs);+ try {+ static const bool per_token = std::getenv("STRATA_INDEXER_PER_TOKEN") != nullptr;+ if (!per_token) {+ strata::kernels::native_qsa_indexer_append_batch(m.idx_raw, T, p0, 0, (const float*) wikn->data,+ EPS, ib, s, st.max_cells,+ strata::kernels::rope_scaling(), cs);+ }+ for (int64_t t = 0; per_token && t < T; ++t) {+ const int32_t* step_t = m.steps_dev + t * strata::kernels::kStepCount;+ strata::kernels::native_qsa_indexer_append(m.idx_raw + t * 128, step_t + strata::kernels::kStepPos, 0,+ (const float*) wikn->data, EPS, ib, s, st.max_cells,+ strata::kernels::rope_scaling(), cs);+ }+ } catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; }+ pt.mark(kPfQsaSel, cs);+ for (int64_t q0 = 0; q0 < T; q0 += m.sel_batch) {+ const int64_t nb = std::min(m.sel_batch, T - q0);+ const int32_t* steps0 = m.steps_dev + q0 * strata::kernels::kStepCount;+ const int64_t active = (int64_t) m.steps_host[(size_t) ((q0 + nb - 1) * strata::kernels::kStepCount ++ strata::kernels::kStepNBid)] + 1;+ static const bool old_sel = std::getenv("STRATA_SELECT_OLD") != nullptr;+ if (old_sel || !strata::kernels::qsa_block_scores_tc(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512,+ steps0, nb, m.max_blocks, s, m.sel_scores,+ cs, active))+ strata::kernels::qsa_block_scores(st.idx_pooled, st.idx_pooled_i8, st.idx_pooled_scale, st.idx_dead, m.q_idx + q0 * 512, steps0, nb,+ m.max_blocks, s, m.sel_scores, cs, active);+ strata::kernels::qsa_block_topk(m.sel_scores, steps0, nb, m.max_blocks, m.cap, s,+ m.sel_ids + q0 * m.cap, cs, active);+ }+ const strata::kernels::QsaAttnPools pools = staged ? pools_of(m.stage, m.ident_table)+ : core::qsa_attn_pools(st);+ pt.mark(kPfQsaAttn, cs);+ static const bool old_attn = std::getenv("STRATA_PROMPT_ATTN_OLD") != nullptr;+ if (old_attn || !strata::kernels::qsa_prompt_attn_batch(m.q, pools, m.sel_ids, m.steps_dev, m.cap, s,+ m.attn, T, cs))+ for (int64_t q0 = 0; q0 < T; q0 += m.attn_batch) {+ const int64_t nb = std::min(m.attn_batch, T - q0);+ strata::kernels::qsa_decode_attn_batch(m.q + q0 * ZV, pools, m.sel_ids + q0 * m.cap,+ m.steps_dev + q0 * strata::kernels::kStepCount, m.cap,+ s, m.attn_scratch, m.attn + q0 * ZV, nb, cs);+ }+ if (st.kv_rot || st.kv_hybrid) strata::kernels::fwht256_inplace_cuda(m.attn, T * 24, cs);+ pt.mark(kPfQsa, cs);+ gate_attn(m.attn, m.Qf, m.attn_h, T, cs);+ if (!native_proj(m.gemm, wo, m.attn_h, m.bo, T, v.name("attn_output.weight"), err)) return false;+ ++qsa_index;+ } else {+ // ======================= the FFN half: router + shared expert now, the experts later =========+ const core::WeightRef *wr = need(v, "ffn_gate_inp.weight", err),+ *wgi = need(v, "ffn_gate_inp_shexp.weight", err),+ *wsg = need(v, "ffn_gate_shexp.weight", err),+ *wsu = need(v, "ffn_up_shexp.weight", err),+ *wsd = need(v, "ffn_down_shexp.weight", err);+ if (!wr || !wgi || !wsg || !wsu || !wsd) return false;+ pt.mark(kPfRouter, cs);+ if (!bf16_proj(m.gemm, wr, m.mixed_bf, m.logits, T, v.name("ffn_gate_inp.weight"), err, 0, m.mixed_bf_lo)) return false;+ route(m.logits, lb.ids + t0 * K, lb.w + t0 * K, T, m.g->n_expert, cs);+ if (!native_proj(m.gemm, wsg, m.mixed_h, m.sgate, T, v.name("ffn_gate_shexp.weight"), err)) return false;+ if (!native_proj(m.gemm, wsu, m.mixed_h, m.sup, T, v.name("ffn_up_shexp.weight"), err)) return false;+ swiglu_pair(m.sgate, m.sup, m.sh_h, T, cs);+ if (!native_proj(m.gemm, wsd, m.sh_h, m.shared, T, v.name("ffn_down_shexp.weight"), err)) return false;+ if (wgi->kind != core::WeightKind::Bf16InF32) { err = "prefill: shared gate is not BF16"; return false; }+ m.gemm.bf16(m.mixed_bf, (const uint16_t*) wgi->data, m.sg, T, 1, N);+ if (m.mixed_bf_lo) m.gemm.bf16(m.mixed_bf_lo, (const uint16_t*) wgi->data, m.sg, T, 1, N, 0, 1.0f);+ moe_shared_init(m.shared, m.sg, acc, T, cs);+ break; // the FFN half's write waits for the experts (the next layer, or the final pass)+ }+ // ---- the attention half's write, fused with the FFN half's norm (run_impl's F-2)+ const core::WeightRef* wnn = nullptr;+ if (!gr_unfused()) {+ wnn = need(v, "hc_ffn_norm.weight", err);+ if (!wnn) return false;+ }+ if (wnn) {+ gr_write_norm_rs(m.R, m.bo, m.inj, HC, (const float*) wnn->data, EPS, m.grs, m.xn16, T, cs,+ m.xn16_lo);+ normed = true;+ } else {+ gr_write(m.R, m.bo, m.inj, HC, T, cs);+ }+ }+ if (!lb.r_dev) { // down on the copy stream once this sub-chunk's work is done+ cudaEventRecord(lb.ev_done[c & 1], cs);+ cudaStreamWaitEvent(m.copy, lb.ev_done[c & 1], 0);+ if (cudaMemcpyAsync(lb.R_host + (size_t) t0 * D, m.R, (size_t) T * D * 4, cudaMemcpyDeviceToHost,+ m.copy) != cudaSuccess) {+ err = "prefill (layer-major): the residual rows' download failed";+ return false;+ }+ cudaEventRecord(lb.ev_out[c & 1], m.copy);+ }+ }+ ms_attn += ms_since(ta);+ if (l == L) break;+ if (!lm_experts(l, Sn)) return false;+ }+ }+ lm_mid_ = false;+ ss.ple_prev[0] = prev[0];+ ss.ple_prev[1] = prev[1];+ if (ple_next.valid()) ple_next.wait();+ if (cudaStreamSynchronize(cs) != cudaSuccess) {+ err = std::string("prefill: ") + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ if (const cudaError_t cst = cudaStreamSynchronize(m.copy); cst != cudaSuccess) {+ err = std::string("prefill: expert copy stream: ") + cudaGetErrorString(cst);+ return false;+ }+ stats_.ms_total += ms_since(t_start);+ std::fprintf(stderr, "strata prefill: layer-major: %lld tokens in %lld super-chunk(s) of up to %lld (R in %s, %lld MiB VRAM free before), "+ "%lld expert blobs streamed (%.1f MB), %lld resident, %lld MMQ groups; %.0f ms (host: "+ "enqueueing the attention halves %.0f, the experts %.0f: waiting for the attention halves "+ "%.0f, the routing tables %.0f)\n",+ (long long) n, (long long) lm_supers, (long long) S, lb.r_dev ? "VRAM" : "pinned RAM", (long long) lb.free_mib,+ (long long) lm_streamed, (double) lm_bytes / 1e6, (long long) lm_resident, (long long) lm_groups,+ ms_since(t_start), ms_attn, ms_experts, ms_wait_a, ms_host_tables);+ if (pt.on) {+ pt.fold();+ double total = 0.0;+ for (double x : pt.ms) total += x;+ std::string line;+ char b[96];+ for (int i = 0; i < kPfCount; ++i) {+ if (pt.ms[i] <= 0.0) continue;+ std::snprintf(b, sizeof b, " %s %.0f (%.1f%%)", kPfNames[i], pt.ms[i], total > 0 ? 100.0 * pt.ms[i] / total : 0.0);+ line += b;+ }+ std::fprintf(stderr, "strata prefill timing (layer-major): %lld tokens, GPU timeline %.0f ms:%s\n",+ (long long) n, total, line.c_str());+ }+ return true;+}+bool Prefill::drain_pipeline(std::string& err) {bool ok = true;@@ -5585,6 +6576,20 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& ereturn ok;}#endif+ if (lm_env() && hand_in_ == nullptr && impl_) {+ std::string why;+ if (lm_prepare(n, why) > 0) {+ const bool ok = run_layer_major(tokens, n, pos0, err);+ lm_mid_ = false;+ lm_release();+ return ok;+ }+ static std::string said; // once per reason (a short prompt is the common one)+ if (why != said) {+ said = why;+ std::fprintf(stderr, "strata prefill: layer-major off for this prompt: %s\n", why.c_str());+ }+ }const bool body_ok = run_impl(tokens, n, pos0, err);std::string drain_err;diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 54d221cd4..3b6537ce4 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -10127,12 +10127,15 @@ int main(int argc, char** argv) {strata::core::progress_at("reading the prompt (batched), done up to token", done);strata::core::progress_beat();std::fflush(stdout);- const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check;+ // layer-major prefill: a sub-chunk before its super-chunk's end has the super-chunk's state - the+ // checkpoint is taken at that end instead+ const bool periodic_checkpoint = o.prompt_cache_every > 0 && done >= pp_next_check && !sp.layer_major_mid();// Keep the prefill's existing chunk geometry. One extra near-tail checkpoint// can serve a branch whose shared prefix ends before the final cached state.const bool tail_checkpoint = o.prompt_cache_tail && o.prompt_cache > 0 && o.prompt_cache_every > 0 &&!multi_gpu && !pp_tail_saved && pp_total - done > std::max<int64_t>(1, o.short_read) &&- pp_total - done <= T; // this chunk's real size (--prefill auto chunks differ from o.prefill_chunk)+ pp_total - done <= T && // this chunk's real size (--prefill auto chunks differ from o.prefill_chunk)+ !sp.layer_major_mid();if (periodic_checkpoint || tail_checkpoint) {bool saved = false;if (multi_gpu) { // the stages' parts, saved when each of them read this chunk