Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: opt-in sparse copied events and deduplicated slot waits in the streamed prompt walk (#4005)

Vop3pa11f8b0aa544
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6356e27b6..e1aa9ffe8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2338,6 +2338,10 @@ if(STRATA_BUILD_TESTS)
add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)
target_include_directories(warm_policy_test PRIVATE include)
add_test(NAME warm_policy_test COMMAND warm_policy_test)
+ # #4005: which streamed expert copies get a copied event (header-only, CPU)
+ add_executable(copy_events_test src/prefill/copy_events_test.cpp)
+ target_include_directories(copy_events_test PRIVATE include)
+ add_test(NAME copy_events_test COMMAND copy_events_test)
# issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails it
add_executable(pool_stress src/kernels/cpu/pool_stress.cpp)
target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)
diff --git a/include/strata/prefill/copy_events.hpp b/include/strata/prefill/copy_events.hpp
new file mode 100644
index 000000000..96f1ea62b
--- /dev/null
+++ b/include/strata/prefill/copy_events.hpp
@@ -0,0 +1,35 @@
+// include/strata/prefill/copy_events.hpp - which streamed expert copies get a "copied" event (#4005).
+//
+// The streamed prompt walk issues one host-to-device copy per expert entry on the copy stream and the compute stream
+// waits for it. By default every entry records its own event (every == 1) and the compute stream waits on the event of
+// the entry it needs. STRATA_PREFILL_COPY_EVENT_EVERY=N (tester switch) records every N-th entry only, and the
+// compute stream waits on the first recorded entry at or after the one it needs: the copy stream is in order, so that
+// event covers it. The two rules are pure functions so a host test can check them without a GPU.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::prefill {
+
+/// The value of STRATA_PREFILL_COPY_EVENT_EVERY: 1 (every entry) unless the text is a number above 1; at most 64.
+inline int copy_event_every_from(int parsed) { return parsed > 1 ? (parsed < 64 ? parsed : 64) : 1; }
+
+/// Entry `idx` of `total` records its copied event right after its copy: always with every == 1; else every N-th entry
+/// and the last one.
+inline bool copy_event_due(size_t idx, size_t total, int every) {
+ return every <= 1 || (idx + 1) % (size_t) every == 0 || idx + 1 == total;
+}
+
+/// The entry whose recorded event a consumer waits on for entry `k`, when the entries [0, recorded) have had their
+/// events decided (`recorded` = 1 + the last recorded entry) and `is_recorded(r)` says whether entry r has one. The
+/// first recorded entry at or after k; the last recorded entry (recorded - 1) when none is found, which covers k as
+/// well. Precondition: k < recorded.
+template <class IsRecorded>
+inline size_t copy_event_entry(size_t k, size_t recorded, IsRecorded is_recorded) {
+ for (size_t r = k; r < recorded; ++r)
+ if (is_recorded(r)) return r;
+ return recorded - 1;
+}
+
+} // namespace strata::prefill
diff --git a/src/prefill/copy_events_test.cpp b/src/prefill/copy_events_test.cpp
new file mode 100644
index 000000000..1ce75e68b
--- /dev/null
+++ b/src/prefill/copy_events_test.cpp
@@ -0,0 +1,76 @@
+// Host test for include/strata/prefill/copy_events.hpp (#4005): the copied-event rules of the streamed prompt walk.
+#include "strata/prefill/copy_events.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+using namespace strata::prefill;
+
+static int g_fail = 0;
+static void check(bool ok, const char* what) {
+ if (!ok) { std::printf("FAIL: %s\n", what); ++g_fail; }
+}
+
+int main() {
+ check(copy_event_every_from(0) == 1 && copy_event_every_from(1) == 1 && copy_event_every_from(-5) == 1, "unset / 0 / 1 / negative: every entry");
+ check(copy_event_every_from(2) == 2 && copy_event_every_from(16) == 16 && copy_event_every_from(64) == 64, "2..64 as given");
+ check(copy_event_every_from(65) == 64 && copy_event_every_from(100000) == 64, "above 64 is 64");
+
+ // default: every entry records (the walk then waits on copied[slot], as before the switch)
+ for (size_t total : {1u, 2u, 7u, 100u})
+ for (size_t i = 0; i < total; ++i) check(copy_event_due(i, total, 1), "every == 1 records every entry");
+
+ // sparse: the walk's issuer/consumer as a model. The issuer records per copy_event_due (and, when it must wait for
+ // the consumer, the entry before the one it is about to issue); the consumer for entry k waits on
+ // copy_event_entry(k, recorded, ...). Every wait must name a recorded entry at or after k, and the ring must
+ // never need an event that a later entry has overwritten.
+ std::srand(4005);
+ for (int every : {2, 3, 5, 16, 64})
+ for (int ring : {2, 4, 8, 96})
+ for (size_t total : {1u, 5u, 33u, 300u}) {
+ std::vector<long long> rec_of((size_t) ring, -1);
+ size_t recorded = 0, issued = 0, consumed = 0;
+ long long last_rec = -1;
+ bool ok = true;
+ auto record = [&](size_t i) {
+ if (rec_of[i % (size_t) ring] >= 0 && (size_t) rec_of[i % (size_t) ring] >= consumed) ok = false; // overwrote a live event
+ rec_of[i % (size_t) ring] = (long long) i;
+ recorded = i + 1;
+ last_rec = (long long) i;
+ };
+ auto cover = [&](size_t idx) { if (idx > 0 && last_rec < (long long) idx - 1) record(idx - 1); };
+ size_t guard = 0;
+ while (consumed < total && ok && ++guard < 100000) {
+ const bool can_issue = issued < total && issued < consumed + (size_t) ring;
+ const bool can_consume = consumed < issued;
+ const bool pick_issue = can_issue && (!can_consume || std::rand() % 2);
+ if (pick_issue) {
+ const size_t idx = issued;
+ if (std::rand() % 4 == 0) cover(idx); // the issuer had to wait (ring full or stager)
+ if (copy_event_due(idx, total, every)) record(idx);
+ ++issued;
+ } else if (can_consume) {
+ if (issued == total || true) {
+ // a consumer needs recorded > k; when the issuer is blocked it has covered what it issued
+ if (recorded <= consumed) {
+ if (issued < total && !(issued < consumed + (size_t) ring)) cover(issued);
+ if (issued == total) cover(issued);
+ }
+ if (recorded <= consumed) { continue; } // would spin until the issuer records
+ const size_t r = copy_event_entry(consumed, recorded, [&](size_t e) { return rec_of[e % (size_t) ring] == (long long) e; });
+ if (r < consumed || r >= recorded || rec_of[r % (size_t) ring] != (long long) r) ok = false;
+ ++consumed;
+ }
+ } else {
+ break;
+ }
+ }
+ char what[128];
+ std::snprintf(what, sizeof what, "sparse every=%d ring=%d total=%zu: every wait names a live recorded entry", every, ring, total);
+ check(ok && consumed == total, what);
+ }
+
+ if (g_fail == 0) std::printf("copy_events_test: all checks passed\n");
+ return g_fail == 0 ? 0 : 1;
+}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 08b529e6f..9e71dd696 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1,6 +1,7 @@
// src/prefill/prefill.cpp - see include/strata/prefill/prefill.hpp.
#include "strata/kernels/fused_gr.hpp"
#include "strata/prefill/prefill.hpp"
+#include "strata/prefill/copy_events.hpp"
#if defined(STRATA_HAVE_DFLASH2)
#include "strata/kernels/dflash2.hpp"
#endif
@@ -3205,13 +3206,54 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
~IssuerJoin() { if (t->joinable()) { stop->store(true); t->join(); } }
} issuer_join{&a_stop, &issuer};
const bool threaded_issue = stream_all && issuer_on;
+ // Opt-in, the threaded issuer only: under WDDM every cudaEventRecord and cross-stream wait is a
+ // submission of its own (~80 / ~45 us each on a 6900 XT under Windows 11, ~3 / ~0 us on Linux), and the
+ // streamed walk paid one record and one wait per expert although a group gathers after ONE wait on its last
+ // copy and is released by ONE event. STRATA_PREFILL_COPY_EVENT_EVERY=N records the copied event every N-th
+ // entry (and before the issuer waits, and on the last), and the consumer waits on the first recorded entry at or
+ // after the one it needs (the copy stream is in order). STRATA_PREFILL_COPY_WAIT_DEDUPE=1 skips the copy
+ // stream's wait on a slot's release event when the previous slot was released by the same event (one flush).
+ // The same copies into the same slots: the same bytes.
+ static const int copy_event_every = [] {
+ const char* v = std::getenv("STRATA_PREFILL_COPY_EVENT_EVERY");
+ return strata::prefill::copy_event_every_from(v ? std::atoi(v) : 1);
+ }();
+ static const bool copy_wait_dedupe = [] {
+ const char* v = std::getenv("STRATA_PREFILL_COPY_WAIT_DEDUPE");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ const bool sparse_copied = threaded_issue && copy_event_every > 1;
+ if (threaded_issue && (sparse_copied || copy_wait_dedupe)) {
+ static bool said = false;
+ if (!said) {
+ said = true;
+ std::fprintf(stderr, "strata: prompt copies: a copied event every %d streamed experts "
+ "(STRATA_PREFILL_COPY_EVENT_EVERY), %s (STRATA_PREFILL_COPY_WAIT_DEDUPE)\n",
+ copy_event_every, copy_wait_dedupe ? "one wait per release event" : "a wait per slot");
+ }
+ }
+ std::vector<std::atomic<int64_t>> rec_of(sparse_copied ? (size_t) m.ring : 0); // the entry a slot's event marks
+ for (auto& r : rec_of) r.store(-1, std::memory_order_relaxed);
+ std::atomic<size_t> a_rec{0}; // 1 + the last entry whose copied event is recorded
+ int64_t iss_waits = 0, iss_dedup = 0;
if (threaded_issue) {
issuer = std::thread([&] {
strata::platform::name_this_thread("strata-issue");
strata::aux_cpus::pin_current_thread();
const core::OnDevice od(m.device);
+ int64_t last_rec = -1; // sparse_copied: the last entry whose copied event is recorded
+ int prev_rel = -1; // dedupe: the release event the copy stream waited on for the previous entry
+ auto record = [&](size_t i) {
+ cudaEventRecord(m.copied[i % (size_t) m.ring], m.copy);
+ rec_of[i % (size_t) m.ring].store((int64_t) i, std::memory_order_release);
+ a_rec.store(i + 1, std::memory_order_release);
+ last_rec = (int64_t) i;
+ };
+ // before the issuer waits: what it issued is covered, so no consumer waits on an entry with no event
+ auto cover = [&](size_t idx) { if (sparse_copied && idx > 0 && last_rec < (int64_t) idx - 1) record(idx - 1); };
for (size_t idx = 0; idx < seq.size(); ++idx) {
while (idx >= a_consumed.load(std::memory_order_acquire) + (size_t) m.ring) {
+ cover(idx);
if (a_stop.load(std::memory_order_acquire)) return;
std::this_thread::yield();
}
@@ -3219,16 +3261,31 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
const int sl = (int) (idx % (size_t) m.ring);
const auto th = Clock::now();
const size_t bytes = (size_t) lay0.blob_bytes(en.l);
- if (m.stage_live[sl]) cudaStreamWaitEvent(m.copy, m.used[m.used_of[sl]], 0);
+ if (m.stage_live[sl]) {
+ // from the second round on, both slots were released in this walk: the same event means the same
+ // flush, and the wait for the previous slot already covers this one
+ const int rel = m.used_of[sl];
+ if (copy_wait_dedupe && idx >= (size_t) m.ring && rel == prev_rel) {
+ ++iss_dedup;
+ } else {
+ cudaStreamWaitEvent(m.copy, m.used[rel], 0);
+ ++iss_waits;
+ }
+ prev_rel = rel;
+ } else {
+ prev_rel = -1;
+ }
if (en.job < 0) {
cudaMemcpyAsync(m.stage_dev[sl], en.blob, bytes, cudaMemcpyHostToDevice, m.copy);
++iss_dma;
} else {
+ cover(idx); // the stager may make the issuer wait
const uint8_t* hb = m.stager->wait(en.job);
cudaMemcpyAsync(m.stage_dev[sl], hb, bytes, cudaMemcpyHostToDevice, m.copy);
m.stager->issued_one(en.job, m.copy);
}
- cudaEventRecord(m.copied[sl], m.copy);
+ if (!sparse_copied) cudaEventRecord(m.copied[sl], m.copy);
+ else if (strata::prefill::copy_event_due(idx, seq.size(), copy_event_every)) record(idx);
m.stage_live[sl] = true;
iss_ms += ms_since(th);
++iss_streamed;
@@ -3243,6 +3300,22 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (!threaded_issue) return;
while (a_issued.load(std::memory_order_acquire) <= k) std::this_thread::yield();
};
+ // the compute stream waits for entry k's copy: on its slot's event (`slot`, as before this switch existed), or
+ // (sparse_copied) on the first recorded entry at or after it. That event's slot cannot be re-recorded before k
+ // is given back (the issuer stays a ring ahead).
+ auto wait_copied = [&](size_t k, size_t slot) {
+ if (!sparse_copied) {
+ cudaStreamWaitEvent(m.cs, m.copied[slot], 0);
+ return;
+ }
+ while (a_rec.load(std::memory_order_acquire) <= k) std::this_thread::yield();
+ const size_t hi = a_rec.load(std::memory_order_acquire);
+ // (hi - 1 is always recorded, so the last entry is only the fallback that covers k all the same)
+ const size_t r = strata::prefill::copy_event_entry(k, hi, [&](size_t e) {
+ return rec_of[e % (size_t) m.ring].load(std::memory_order_acquire) == (int64_t) e;
+ });
+ cudaStreamWaitEvent(m.cs, m.copied[r % (size_t) m.ring], 0);
+ };
auto give_back = [&](size_t upto) {
if (threaded_issue) a_consumed.store(upto, std::memory_order_release);
else issue_until(upto + (size_t) m.ring);
@@ -3913,7 +3986,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// one copy stream, in order: the batch's last blob covers the others
pt.mark(kPfWaitCopy, cs);
wait_issued(k - 1);
- cudaStreamWaitEvent(m.cs, m.copied[(k - 1) % (size_t) m.ring], 0);
+ wait_copied(k - 1, (k - 1) % (size_t) m.ring);
}
pt.mark(kPfGemmGU, cs);
if (fused_nat) {
@@ -4699,7 +4772,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// STRATA_PREFILL_GROUP_GATHER=0: one gather, one wait and one record per expert.
mmq::GatherGroup gg;
int gg_slots[MMQ_GROUP];
+ size_t gg_ents[MMQ_GROUP]; // the streamed walk: their entries (wait_copied)
int gg_nslots = 0; // ring slots gathered by the next flush
+ size_t cur_entry = 0; // the streamed walk: the entry compute() is given
// STRATA_MMQ_DIRECT=1: a resident expert with guard room in its slot is not gathered - the
// group's pointer tables give MMQ each expert's own address (group position q), the others are
// gathered at compact positions c = 0..ns-1 of the group buffers, as before (same bytes)
@@ -4714,7 +4789,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
const auto& f = lay.fmt[(size_t) l];
if (gg_nslots > 0) { // the copies land in order: the last one covers the others
pt.mark(kPfWaitCopy, cs);
- cudaStreamWaitEvent(m.cs, m.copied[gg_slots[gg_nslots - 1]], 0);
+ wait_copied(gg_ents[gg_nslots - 1], (size_t) gg_slots[gg_nslots - 1]);
pt.mark(kPfDequant, cs);
}
// NVFP4: each expert's scales go next to its group slot in the same launch
@@ -4765,7 +4840,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (gg_now) {
gg.blob[c] = blob_dev;
gg.n = c + 1;
- if (slot >= 0) gg_slots[gg_nslots++] = slot;
+ if (slot >= 0) { gg_ents[gg_nslots] = cur_entry; gg_slots[gg_nslots++] = slot; }
} else {
mmq::gather_native(blob_dev, blob_dev + f.up_off, mmq_gub / 2, blob_dev + f.down_off,
mmq_db, m.grp_gu + (size_t) c * mmq_gub,
@@ -4782,7 +4857,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (gg_now) {
gg.blob[q] = blob_dev;
gg.n = (int) q + 1;
- if (slot >= 0) gg_slots[gg_nslots++] = slot;
+ if (slot >= 0) { gg_ents[gg_nslots] = cur_entry; gg_slots[gg_nslots++] = slot; }
if (!last_of_group) return true;
flush(MMQ_TAIL);
gg = mmq::GatherGroup{};
@@ -4946,8 +5021,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
wait_issued(k);
if (!group_gather) {
pt.mark(kPfWaitCopy, cs);
- cudaStreamWaitEvent(m.cs, m.copied[sl], 0);
+ wait_copied(k, (size_t) sl);
}
+ cur_entry = k;
if (!compute(j, m.stage_dev[sl], sl, -1)) return false;
consumed = ++k;
if (!group_gather || gg_nslots == 0) give_back(consumed); // its group was gathered
@@ -5106,6 +5182,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
stats_.ms_experts_host += iss_ms;
stats_.experts_streamed += iss_streamed;
stats_.experts_dma += iss_dma;
+ if (copy_wait_dedupe && pt.on)
+ std::fprintf(stderr, "strata prefill copies: %lld slot waits, %lld skipped as the same release event\n",
+ (long long) iss_waits, (long long) iss_dedup);
}
stats_.tokens += T;
core::progress_at("reading the prompt (batched): finishing the chunk from token", p0);