Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill: opt-in sparse copied events and deduplicated slot waits in the streamed prompt walk (#4005)
Vop3pa11f8b0aa544
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 6356e27b6..e1aa9ffe8 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -2338,6 +2338,10 @@ if(STRATA_BUILD_TESTS)add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)target_include_directories(warm_policy_test PRIVATE include)add_test(NAME warm_policy_test COMMAND warm_policy_test)+ # #4005: which streamed expert copies get a copied event (header-only, CPU)+ add_executable(copy_events_test src/prefill/copy_events_test.cpp)+ target_include_directories(copy_events_test PRIVATE include)+ add_test(NAME copy_events_test COMMAND copy_events_test)# issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails itadd_executable(pool_stress src/kernels/cpu/pool_stress.cpp)target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)diff --git a/include/strata/prefill/copy_events.hpp b/include/strata/prefill/copy_events.hppnew file mode 100644index 000000000..96f1ea62b--- /dev/null+++ b/include/strata/prefill/copy_events.hpp@@ -0,0 +1,35 @@+// include/strata/prefill/copy_events.hpp - which streamed expert copies get a "copied" event (#4005).+//+// The streamed prompt walk issues one host-to-device copy per expert entry on the copy stream and the compute stream+// waits for it. By default every entry records its own event (every == 1) and the compute stream waits on the event of+// the entry it needs. STRATA_PREFILL_COPY_EVENT_EVERY=N (tester switch) records every N-th entry only, and the+// compute stream waits on the first recorded entry at or after the one it needs: the copy stream is in order, so that+// event covers it. The two rules are pure functions so a host test can check them without a GPU.+#pragma once++#include <cstddef>+#include <cstdint>++namespace strata::prefill {++/// The value of STRATA_PREFILL_COPY_EVENT_EVERY: 1 (every entry) unless the text is a number above 1; at most 64.+inline int copy_event_every_from(int parsed) { return parsed > 1 ? (parsed < 64 ? parsed : 64) : 1; }++/// Entry `idx` of `total` records its copied event right after its copy: always with every == 1; else every N-th entry+/// and the last one.+inline bool copy_event_due(size_t idx, size_t total, int every) {+ return every <= 1 || (idx + 1) % (size_t) every == 0 || idx + 1 == total;+}++/// The entry whose recorded event a consumer waits on for entry `k`, when the entries [0, recorded) have had their+/// events decided (`recorded` = 1 + the last recorded entry) and `is_recorded(r)` says whether entry r has one. The+/// first recorded entry at or after k; the last recorded entry (recorded - 1) when none is found, which covers k as+/// well. Precondition: k < recorded.+template <class IsRecorded>+inline size_t copy_event_entry(size_t k, size_t recorded, IsRecorded is_recorded) {+ for (size_t r = k; r < recorded; ++r)+ if (is_recorded(r)) return r;+ return recorded - 1;+}++} // namespace strata::prefilldiff --git a/src/prefill/copy_events_test.cpp b/src/prefill/copy_events_test.cppnew file mode 100644index 000000000..1ce75e68b--- /dev/null+++ b/src/prefill/copy_events_test.cpp@@ -0,0 +1,76 @@+// Host test for include/strata/prefill/copy_events.hpp (#4005): the copied-event rules of the streamed prompt walk.+#include "strata/prefill/copy_events.hpp"++#include <cstdio>+#include <cstdlib>+#include <vector>++using namespace strata::prefill;++static int g_fail = 0;+static void check(bool ok, const char* what) {+ if (!ok) { std::printf("FAIL: %s\n", what); ++g_fail; }+}++int main() {+ check(copy_event_every_from(0) == 1 && copy_event_every_from(1) == 1 && copy_event_every_from(-5) == 1, "unset / 0 / 1 / negative: every entry");+ check(copy_event_every_from(2) == 2 && copy_event_every_from(16) == 16 && copy_event_every_from(64) == 64, "2..64 as given");+ check(copy_event_every_from(65) == 64 && copy_event_every_from(100000) == 64, "above 64 is 64");++ // default: every entry records (the walk then waits on copied[slot], as before the switch)+ for (size_t total : {1u, 2u, 7u, 100u})+ for (size_t i = 0; i < total; ++i) check(copy_event_due(i, total, 1), "every == 1 records every entry");++ // sparse: the walk's issuer/consumer as a model. The issuer records per copy_event_due (and, when it must wait for+ // the consumer, the entry before the one it is about to issue); the consumer for entry k waits on+ // copy_event_entry(k, recorded, ...). Every wait must name a recorded entry at or after k, and the ring must+ // never need an event that a later entry has overwritten.+ std::srand(4005);+ for (int every : {2, 3, 5, 16, 64})+ for (int ring : {2, 4, 8, 96})+ for (size_t total : {1u, 5u, 33u, 300u}) {+ std::vector<long long> rec_of((size_t) ring, -1);+ size_t recorded = 0, issued = 0, consumed = 0;+ long long last_rec = -1;+ bool ok = true;+ auto record = [&](size_t i) {+ if (rec_of[i % (size_t) ring] >= 0 && (size_t) rec_of[i % (size_t) ring] >= consumed) ok = false; // overwrote a live event+ rec_of[i % (size_t) ring] = (long long) i;+ recorded = i + 1;+ last_rec = (long long) i;+ };+ auto cover = [&](size_t idx) { if (idx > 0 && last_rec < (long long) idx - 1) record(idx - 1); };+ size_t guard = 0;+ while (consumed < total && ok && ++guard < 100000) {+ const bool can_issue = issued < total && issued < consumed + (size_t) ring;+ const bool can_consume = consumed < issued;+ const bool pick_issue = can_issue && (!can_consume || std::rand() % 2);+ if (pick_issue) {+ const size_t idx = issued;+ if (std::rand() % 4 == 0) cover(idx); // the issuer had to wait (ring full or stager)+ if (copy_event_due(idx, total, every)) record(idx);+ ++issued;+ } else if (can_consume) {+ if (issued == total || true) {+ // a consumer needs recorded > k; when the issuer is blocked it has covered what it issued+ if (recorded <= consumed) {+ if (issued < total && !(issued < consumed + (size_t) ring)) cover(issued);+ if (issued == total) cover(issued);+ }+ if (recorded <= consumed) { continue; } // would spin until the issuer records+ const size_t r = copy_event_entry(consumed, recorded, [&](size_t e) { return rec_of[e % (size_t) ring] == (long long) e; });+ if (r < consumed || r >= recorded || rec_of[r % (size_t) ring] != (long long) r) ok = false;+ ++consumed;+ }+ } else {+ break;+ }+ }+ char what[128];+ std::snprintf(what, sizeof what, "sparse every=%d ring=%d total=%zu: every wait names a live recorded entry", every, ring, total);+ check(ok && consumed == total, what);+ }++ if (g_fail == 0) std::printf("copy_events_test: all checks passed\n");+ return g_fail == 0 ? 0 : 1;+}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 08b529e6f..9e71dd696 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1,6 +1,7 @@// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.#include "strata/kernels/fused_gr.hpp"#include "strata/prefill/prefill.hpp"+#include "strata/prefill/copy_events.hpp"#if defined(STRATA_HAVE_DFLASH2)#include "strata/kernels/dflash2.hpp"#endif@@ -3205,13 +3206,54 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri~IssuerJoin() { if (t->joinable()) { stop->store(true); t->join(); } }} issuer_join{&a_stop, &issuer};const bool threaded_issue = stream_all && issuer_on;+ // Opt-in, the threaded issuer only: under WDDM every cudaEventRecord and cross-stream wait is a+ // submission of its own (~80 / ~45 us each on a 6900 XT under Windows 11, ~3 / ~0 us on Linux), and the+ // streamed walk paid one record and one wait per expert although a group gathers after ONE wait on its last+ // copy and is released by ONE event. STRATA_PREFILL_COPY_EVENT_EVERY=N records the copied event every N-th+ // entry (and before the issuer waits, and on the last), and the consumer waits on the first recorded entry at or+ // after the one it needs (the copy stream is in order). STRATA_PREFILL_COPY_WAIT_DEDUPE=1 skips the copy+ // stream's wait on a slot's release event when the previous slot was released by the same event (one flush).+ // The same copies into the same slots: the same bytes.+ static const int copy_event_every = [] {+ const char* v = std::getenv("STRATA_PREFILL_COPY_EVENT_EVERY");+ return strata::prefill::copy_event_every_from(v ? std::atoi(v) : 1);+ }();+ static const bool copy_wait_dedupe = [] {+ const char* v = std::getenv("STRATA_PREFILL_COPY_WAIT_DEDUPE");+ return v != nullptr && std::atoi(v) != 0;+ }();+ const bool sparse_copied = threaded_issue && copy_event_every > 1;+ if (threaded_issue && (sparse_copied || copy_wait_dedupe)) {+ static bool said = false;+ if (!said) {+ said = true;+ std::fprintf(stderr, "strata: prompt copies: a copied event every %d streamed experts "+ "(STRATA_PREFILL_COPY_EVENT_EVERY), %s (STRATA_PREFILL_COPY_WAIT_DEDUPE)\n",+ copy_event_every, copy_wait_dedupe ? "one wait per release event" : "a wait per slot");+ }+ }+ std::vector<std::atomic<int64_t>> rec_of(sparse_copied ? (size_t) m.ring : 0); // the entry a slot's event marks+ for (auto& r : rec_of) r.store(-1, std::memory_order_relaxed);+ std::atomic<size_t> a_rec{0}; // 1 + the last entry whose copied event is recorded+ int64_t iss_waits = 0, iss_dedup = 0;if (threaded_issue) {issuer = std::thread([&] {strata::platform::name_this_thread("strata-issue");strata::aux_cpus::pin_current_thread();const core::OnDevice od(m.device);+ int64_t last_rec = -1; // sparse_copied: the last entry whose copied event is recorded+ int prev_rel = -1; // dedupe: the release event the copy stream waited on for the previous entry+ auto record = [&](size_t i) {+ cudaEventRecord(m.copied[i % (size_t) m.ring], m.copy);+ rec_of[i % (size_t) m.ring].store((int64_t) i, std::memory_order_release);+ a_rec.store(i + 1, std::memory_order_release);+ last_rec = (int64_t) i;+ };+ // before the issuer waits: what it issued is covered, so no consumer waits on an entry with no event+ auto cover = [&](size_t idx) { if (sparse_copied && idx > 0 && last_rec < (int64_t) idx - 1) record(idx - 1); };for (size_t idx = 0; idx < seq.size(); ++idx) {while (idx >= a_consumed.load(std::memory_order_acquire) + (size_t) m.ring) {+ cover(idx);if (a_stop.load(std::memory_order_acquire)) return;std::this_thread::yield();}@@ -3219,16 +3261,31 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striconst int sl = (int) (idx % (size_t) m.ring);const auto th = Clock::now();const size_t bytes = (size_t) lay0.blob_bytes(en.l);- if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);+ if (m.stage_live[sl]) {+ // from the second round on, both slots were released in this walk: the same event means the same+ // flush, and the wait for the previous slot already covers this one+ const int rel = m.used_of[sl];+ if (copy_wait_dedupe && idx >= (size_t) m.ring && rel == prev_rel) {+ ++iss_dedup;+ } else {+ cudaStreamWaitEvent(m.copy, m.used[rel], 0);+ ++iss_waits;+ }+ prev_rel = rel;+ } else {+ prev_rel = -1;+ }if (en.job < 0) {cudaMemcpyAsync(m.stage_dev[sl], en.blob, bytes, cudaMemcpyHostToDevice, m.copy);++iss_dma;} else {+ cover(idx); // the stager may make the issuer waitconst uint8_t* hb = m.stager->wait(en.job);cudaMemcpyAsync(m.stage_dev[sl], hb, bytes, cudaMemcpyHostToDevice, m.copy);m.stager->issued_one(en.job, m.copy);}- cudaEventRecord(m.copied[sl], m.copy);+ if (!sparse_copied) cudaEventRecord(m.copied[sl], m.copy);+ else if (strata::prefill::copy_event_due(idx, seq.size(), copy_event_every)) record(idx);m.stage_live[sl] = true;iss_ms += ms_since(th);++iss_streamed;@@ -3243,6 +3300,22 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (!threaded_issue) return;while (a_issued.load(std::memory_order_acquire) <= k) std::this_thread::yield();};+ // the compute stream waits for entry k's copy: on its slot's event (`slot`, as before this switch existed), or+ // (sparse_copied) on the first recorded entry at or after it. That event's slot cannot be re-recorded before k+ // is given back (the issuer stays a ring ahead).+ auto wait_copied = [&](size_t k, size_t slot) {+ if (!sparse_copied) {+ cudaStreamWaitEvent(m.cs, m.copied[slot], 0);+ return;+ }+ while (a_rec.load(std::memory_order_acquire) <= k) std::this_thread::yield();+ const size_t hi = a_rec.load(std::memory_order_acquire);+ // (hi - 1 is always recorded, so the last entry is only the fallback that covers k all the same)+ const size_t r = strata::prefill::copy_event_entry(k, hi, [&](size_t e) {+ return rec_of[e % (size_t) m.ring].load(std::memory_order_acquire) == (int64_t) e;+ });+ cudaStreamWaitEvent(m.cs, m.copied[r % (size_t) m.ring], 0);+ };auto give_back = [&](size_t upto) {if (threaded_issue) a_consumed.store(upto, std::memory_order_release);else issue_until(upto + (size_t) m.ring);@@ -3913,7 +3986,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri// one copy stream, in order: the batch's last blob covers the otherspt.mark(kPfWaitCopy, cs);wait_issued(k - 1);- cudaStreamWaitEvent(m.cs, m.copied[(k - 1) % (size_t) m.ring], 0);+ wait_copied(k - 1, (k - 1) % (size_t) m.ring);}pt.mark(kPfGemmGU, cs);if (fused_nat) {@@ -4699,7 +4772,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri// STRATA_PREFILL_GROUP_GATHER=0: one gather, one wait and one record per expert.mmq::GatherGroup gg;int gg_slots[MMQ_GROUP];+ size_t gg_ents[MMQ_GROUP]; // the streamed walk: their entries (wait_copied)int gg_nslots = 0; // ring slots gathered by the next flush+ size_t cur_entry = 0; // the streamed walk: the entry compute() is given// STRATA_MMQ_DIRECT=1: a resident expert with guard room in its slot is not gathered - the// group's pointer tables give MMQ each expert's own address (group position q), the others are// gathered at compact positions c = 0..ns-1 of the group buffers, as before (same bytes)@@ -4714,7 +4789,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striconst auto& f = lay.fmt[(size_t) l];if (gg_nslots > 0) { // the copies land in order: the last one covers the otherspt.mark(kPfWaitCopy, cs);- cudaStreamWaitEvent(m.cs, m.copied[gg_slots[gg_nslots - 1]], 0);+ wait_copied(gg_ents[gg_nslots - 1], (size_t) gg_slots[gg_nslots - 1]);pt.mark(kPfDequant, cs);}// NVFP4: each expert's scales go next to its group slot in the same launch@@ -4765,7 +4840,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (gg_now) {gg.blob[c] = blob_dev;gg.n = c + 1;- if (slot >= 0) gg_slots[gg_nslots++] = slot;+ if (slot >= 0) { gg_ents[gg_nslots] = cur_entry; gg_slots[gg_nslots++] = slot; }} else {mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off,mmq_db, m.grp_gu + (size_t) c * mmq_gub,@@ -4782,7 +4857,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (gg_now) {gg.blob[q] = blob_dev;gg.n = (int) q + 1;- if (slot >= 0) gg_slots[gg_nslots++] = slot;+ if (slot >= 0) { gg_ents[gg_nslots] = cur_entry; gg_slots[gg_nslots++] = slot; }if (!last_of_group) return true;flush(MMQ_TAIL);gg = mmq::GatherGroup{};@@ -4946,8 +5021,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striwait_issued(k);if (!group_gather) {pt.mark(kPfWaitCopy, cs);- cudaStreamWaitEvent(m.cs, m.copied[sl], 0);+ wait_copied(k, (size_t) sl);}+ cur_entry = k;if (!compute(j, m.stage_dev[sl], sl, -1)) return false;consumed = ++k;if (!group_gather || gg_nslots == 0) give_back(consumed); // its group was gathered@@ -5106,6 +5182,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stristats_.ms_experts_host += iss_ms;stats_.experts_streamed += iss_streamed;stats_.experts_dma += iss_dma;+ if (copy_wait_dedupe && pt.on)+ std::fprintf(stderr, "strata prefill copies: %lld slot waits, %lld skipped as the same release event\n",+ (long long) iss_waits, (long long) iss_dedup);}stats_.tokens += T;core::progress_at("reading the prompt (batched): finishing the chunk from token", p0);