Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: the per-layer walk warms this layer's and the next layer's file-tier experts (STRATA_PREFILL_WARM=1) (#3976, #1670)
vizdatom162dec341167
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 2d0df32fb..e1f221c76 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -897,6 +897,59 @@ struct PinnedUpload {}};+// STRATA_PREFILL_WARM=1: chunks below stream_all_min() take the per-layer walk, where each layer's file-tier+// experts are page faults of the stager's few threads (128 KiB of read-ahead each). This thread hands the source a+// layer's whole list at once (madvise(WILLNEED) - the drive sees one deep queue), and the next layer's list from+// the previous chunk while this layer computes. Only warms pages: what the stager copies is unchanged.+struct PrefillWarmer {+ core::ExpertSource* src = nullptr;+ struct Job { int64_t layer; std::vector<int64_t> experts; };+ std::vector<Job> queue;+ bool quit = false;+ std::mutex mu;+ std::condition_variable cv;+ std::thread thread;+ std::atomic<int64_t> warmed{0}, dropped{0};++ explicit PrefillWarmer(core::ExpertSource* s) : src(s) {+ thread = std::thread([this] {+ strata::aux_cpus::pin_current_thread();+ for (;;) {+ Job j;+ {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return quit || !queue.empty(); });+ if (quit) return;+ j = std::move(queue.front());+ queue.erase(queue.begin());+ }+ src->warm(j.layer, j.experts.data(), (int64_t) j.experts.size());+ warmed.fetch_add((int64_t) j.experts.size(), std::memory_order_relaxed);+ }+ });+ }+ ~PrefillWarmer() {+ {+ std::lock_guard<std::mutex> lk(mu);+ quit = true;+ }+ cv.notify_all();+ if (thread.joinable()) thread.join();+ }+ void submit(int64_t layer, std::vector<int64_t> experts) {+ if (experts.empty()) return;+ {+ std::lock_guard<std::mutex> lk(mu);+ while (queue.size() >= 4) { // behind: the oldest layer is done or nearly so+ dropped.fetch_add((int64_t) queue.front().experts.size(), std::memory_order_relaxed);+ queue.erase(queue.begin());+ }+ queue.push_back({layer, std::move(experts)});+ }+ cv.notify_one();+ }+};+struct Prefill::Impl {const core::WeightTable* wt = nullptr;const core::ModelGeometry* g = nullptr;@@ -997,6 +1050,8 @@ struct Prefill::Impl {// only write stream_all_min() - 1 of its rows into them; fused_layout() keeps the two in step (#583, #954)bool fused_bufs = false;std::unique_ptr<Stager> stager; // the unpinned experts' host copies (step 4)+ std::unique_ptr<PrefillWarmer> warmer; // STRATA_PREFILL_WARM=1: the per-layer walk's page warming+ std::vector<std::vector<int64_t>> warm_prev; // per layer: the file-tier experts of the previous chunkcudaEvent_t copied[RING_MAX] = {}, used[RING_MAX] = {};bool stage_live[RING_MAX] = {};// the event that releases each ring slot: its own `used`, or - when an MMQ group is gathered in one launch - the@@ -1512,6 +1567,15 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, coif (!m.stager->init((size_t) MAXBLOB(), threads)) ok = false;m.stager->batch = std::min(m.stager->batch, m.stager->kRing);}+ if (!m.warmer && src != nullptr && src->warms()) {+ const char* wv = std::getenv("STRATA_PREFILL_WARM");+ if (wv != nullptr && std::atoi(wv) != 0) {+ m.warmer = std::make_unique<PrefillWarmer>(src);+ m.warm_prev.assign((size_t) g.n_layers, {});+ std::fprintf(stderr, "prefill: STRATA_PREFILL_WARM=1: the per-layer walk warms this layer's and the next "+ "layer's file-tier experts\n");+ }+ }m.steps_host.resize(T * strata::kernels::kStepCount);m.steps_up.init(T * strata::kernels::kStepCount);m.ids_host.resize(T * K); m.slot_host.resize(T * K); m.src_host.resize(T * K); m.cnt.resize(m.g->n_expert); m.off.resize(m.g->n_expert + 1);@@ -4759,6 +4823,23 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (!b) { err = "prefill: expert source has no blob"; return false; }js.push_back({b, (size_t) lay.blob_bytes(l)});}+ if (m.warmer) {+ std::vector<int64_t> cur;+ for (size_t j = 0; j < order.size(); ++j)+ if (job_of[j] >= 0) cur.push_back(order[j]);+ m.warm_prev[(size_t) l] = cur;+ m.warmer->submit(l, std::move(cur));+ if (l + 1 < m.g->n_layers) {+ std::vector<int64_t> nxt;+ for (int64_t e : m.warm_prev[(size_t) (l + 1)]) {+ if (m.host_res && m.cache && m.host_res[(size_t) (l + 1) * m.g->n_expert + e] >= 0)+ continue;+ if (m.src->pinned(l + 1, (int32_t) e)) continue;+ nxt.push_back(e);+ }+ m.warmer->submit(l + 1, std::move(nxt));+ }+ }m.stager->start(std::move(js));}StagerDone stager_done{stream_all ? nullptr : m.stager.get()};