Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill stager: a job waits for the previous job on its buffer; skip() (#3976, #1670)
vizdatomfca1936c2c97
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 8b08b511c..2d0df32fb 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -551,6 +551,8 @@ struct Stager {std::vector<cudaEvent_t> dma_done;std::vector<Job> jobs;std::unique_ptr<std::atomic<int>[]> ready;+ // layer-major prefill: a job the routing turned out not to need (`skip`) is not read once nobody has claimed it+ std::unique_ptr<std::atomic<char>[]> skipped;size_t ready_cap = 0;// gen << 32 | n << 16 | next index: a claim is a CAS on the generation it woke for (a thread late from the// previous layer can never take a job of this one - the expert pool's issue #29 lesson)@@ -630,6 +632,10 @@ struct Stager {else#endifwhile (issued.load(std::memory_order_acquire) <= i - kRing) std::this_thread::yield();+ // ...and its own copy into the buffer is done: `issued` is a watermark, so a job nobody waits for+ // (the layer-major prefetch reads experts the routing may skip) can be passed while its thread still+ // writes the buffer this job is about to fill+ while (!ready[(size_t) (i - kRing)].load(std::memory_order_acquire)) std::this_thread::yield();}// and done - for a generation's first kRing jobs that is the previous generation's last DMA from// the buffer, which nothing else waits for when a chunk ends without a sync (no MTP) or the DMA@@ -641,6 +647,7 @@ struct Stager {size_t nr = 0;core::ExpertSource* from = nullptr;for (int i = j; i < j + k; ++i) {+ if (skipped[(size_t) i].load(std::memory_order_acquire)) continue; // nothing will read it: no copyconst Job& jb = jobs[(size_t) i];if (jb.from == nullptr) { std::memcpy(buf[i % kRing], jb.src, jb.bytes); continue; }from = jb.from;@@ -674,13 +681,31 @@ struct Stager {if (ready_cap < jobs.size()) {ready_cap = jobs.size() * 2;ready.reset(new std::atomic<int>[ready_cap]);+ skipped.reset(new std::atomic<char>[ready_cap]);+ }+ for (size_t i = 0; i < jobs.size(); ++i) {+ ready[i].store(0, std::memory_order_relaxed);+ skipped[i].store(0, std::memory_order_relaxed);}- for (size_t i = 0; i < jobs.size(); ++i) ready[i].store(0, std::memory_order_relaxed);issued.store(0);++gen;head.store((uint64_t) gen << 32 | (uint64_t) jobs.size() << 16, std::memory_order_release);cv.notify_all();}+ /// Layer-major prefill: job j's bytes are not needed after all (the routing skipped the expert) - a thread that has not+ /// started it yet marks it ready without reading. Never waited for, so its buffer's content does not matter.+ void skip(int j) { skipped[(size_t) j].store(1, std::memory_order_release); }+ /// Layer-major prefill, before wait(j): the launching thread stages its jobs in increasing order, so a job below j+ /// it has not issued was skipped and never gets a DMA - the watermark moves to j. A skipped job is never issued,+ /// so without this a routed job after about a ring's depth of skipped ones waits for `issued` forever.+ void pass(int j) {+ if (issued.load(std::memory_order_relaxed) < j) {+ issued.store(j, std::memory_order_release);+#if defined(STRATA_ATOMIC_WAIT)+ issued.notify_all();+#endif+ }+ }/// Job j's bytes, in a pinned buffer (waits for the copy).const uint8_t* wait(int j) {while (!ready[(size_t) j].load(std::memory_order_acquire)) std::this_thread::yield();