Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill: the copy-wait dedupe skips a wait only when both slots were released in the current walk and the ring is longer than one group (#4005)

Niko1221f217b23bd35a
diff --git a/include/strata/prefill/copy_events.hpp b/include/strata/prefill/copy_events.hpp
index 96f1ea62b..b8e93e405 100644
--- a/include/strata/prefill/copy_events.hpp
+++ b/include/strata/prefill/copy_events.hpp
@@ -32,4 +32,13 @@ inline size_t copy_event_entry(size_t k, size_t recorded, IsRecorded is_recorded
return recorded - 1;
}
+/// STRATA_PREFILL_COPY_WAIT_DEDUPE: the copy stream may skip its wait on the release event of entry `idx`'s slot (`rel`)
+/// when the wait it made for the previous entry was on the same event (`prev_rel`). That holds only when both slots were
+/// released in the current walk (idx - 1 >= ring), because the event of an earlier walk can be re-recorded by this walk's
+/// first flush, and when the ring is longer than one MMQ group of `group` experts, because a flush of a whole ring would
+/// re-record the event right after the previous slot's wait.
+inline bool copy_wait_dedupe_ok(size_t idx, int ring, int group, int rel, int prev_rel) {
+ return ring > group && idx > (size_t) ring && rel == prev_rel;
+}
+
} // namespace strata::prefill
diff --git a/src/prefill/copy_events_test.cpp b/src/prefill/copy_events_test.cpp
index 1ce75e68b..fca467e13 100644
--- a/src/prefill/copy_events_test.cpp
+++ b/src/prefill/copy_events_test.cpp
@@ -71,6 +71,83 @@ int main() {
check(ok && consumed == total, what);
}
+ // copy_wait_dedupe_ok: skipping the copy stream's wait on a slot's release event must never leave a slot's refill
+ // without a wait on a record at or after the flush that released it. Two walks of random length with random group
+ // sizes (1..group), the issuer and the consumer interleaved at random. A "record" is a counter tick of the event.
+ auto simulate = [&](int ring, int group, bool original_rule, unsigned seed) -> bool {
+ std::srand(seed);
+ std::vector<int> used_of((size_t) ring);
+ std::vector<long long> rec_time((size_t) ring, 0), last_flush((size_t) ring, 0);
+ for (int i = 0; i < ring; ++i) used_of[(size_t) i] = i;
+ long long tick = 0;
+ for (int walk = 0; walk < 3; ++walk) {
+ const size_t n = (size_t) ring * (size_t) (2 + std::rand() % 3) + (size_t) (std::rand() % (ring + 1));
+ std::vector<size_t> group_end; // the last entry of each group, in order
+ for (size_t s = 0; s < n;) {
+ size_t len = (size_t) (1 + std::rand() % group);
+ if (s + len > n) len = n - s;
+ s += len;
+ group_end.push_back(s - 1);
+ }
+ size_t issued = 0, consumed = 0, g = 0;
+ const int issue_pct = std::vector<int>{50, 90, 99}[(size_t) (std::rand() % 3)]; // how far ahead the issuer runs
+ int prev_rel = -1;
+ long long prev_cap = 0;
+ std::vector<bool> live((size_t) ring, walk > 0); // the first walk starts with empty slots
+ size_t guard = 0;
+ while (consumed < n && ++guard < 1000000) {
+ const bool can_issue = issued < n && issued < consumed + (size_t) ring;
+ const bool can_flush = g < group_end.size() && group_end[g] < issued;
+ if (can_issue && (!can_flush || std::rand() % 100 < issue_pct)) {
+ const size_t idx = issued++;
+ const int sl = (int) (idx % (size_t) ring);
+ if (live[(size_t) sl]) {
+ const int rel = used_of[(size_t) sl];
+ const bool skip = original_rule ? (idx >= (size_t) ring && rel == prev_rel)
+ : copy_wait_dedupe_ok(idx, ring, group, rel, prev_rel);
+ const long long cap = skip ? prev_cap : rec_time[(size_t) rel];
+ if (cap < last_flush[(size_t) sl]) return false; // the refill would not wait for its release
+ prev_cap = cap;
+ prev_rel = rel;
+ } else {
+ prev_rel = -1;
+ }
+ live[(size_t) sl] = true;
+ } else if (can_flush) {
+ size_t first = consumed;
+ const size_t last = group_end[g++];
+ const int rel = (int) (last % (size_t) ring);
+ rec_time[(size_t) rel] = ++tick;
+ for (size_t e = first; e <= last; ++e) {
+ used_of[e % (size_t) ring] = rel;
+ last_flush[e % (size_t) ring] = tick;
+ }
+ consumed = last + 1;
+ } else {
+ return false; // stuck: the model is wrong
+ }
+ }
+ }
+ return true;
+ };
+ for (int ring : {17, 18, 24, 32, 96, 1024})
+ for (unsigned seed = 1; seed <= 300; ++seed) {
+ char what[96];
+ std::snprintf(what, sizeof what, "dedupe never skips a needed wait (ring %d, seed %u)", ring, seed);
+ if (!simulate(ring, 16, false, seed)) { check(false, what); break; }
+ }
+ {
+ bool rule_ok = true; // the rule refuses a ring that a group can fill
+ for (size_t idx : {20u, 1000u}) rule_ok = rule_ok && !copy_wait_dedupe_ok(idx, 16, 16, 3, 3);
+ check(rule_ok, "a ring of one group never dedupes");
+ check(!copy_wait_dedupe_ok(32, 32, 16, 3, 3) && copy_wait_dedupe_ok(33, 32, 16, 3, 3), "the first round never dedupes");
+ check(!copy_wait_dedupe_ok(40, 32, 16, 3, 4) && !copy_wait_dedupe_ok(40, 32, 16, 3, -1), "different events or none: wait");
+ // the control: the PR's first rule (idx >= ring) is caught by this model
+ bool caught = false;
+ for (int ring : {17, 18, 20, 24, 32}) for (unsigned seed = 1; seed <= 600 && !caught; ++seed) caught = !simulate(ring, 16, true, seed);
+ check(caught, "the model finds the stale-event race in the rule 'idx >= ring'");
+ }
+
if (g_fail == 0) std::printf("copy_events_test: all checks passed\n");
return g_fail == 0 ? 0 : 1;
}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 9e71dd696..8b08b511c 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -3262,10 +3262,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
const auto th = Clock::now();
const size_t bytes = (size_t) lay0.blob_bytes(en.l);
if (m.stage_live[sl]) {
- // from the second round on, both slots were released in this walk: the same event means the same
- // flush, and the wait for the previous slot already covers this one
+ // Both this slot and the previous one must have been released in THIS walk (idx - 1 >= ring: the
+ // previous slot's event of an earlier walk could be re-recorded by this walk's first flush), and
+ // the ring must be longer than one MMQ group (a flush of a whole ring would re-record the event
+ // after the previous slot's wait). Then the same event means the same flush, and the wait for
+ // the previous slot already covers this one.
const int rel = m.used_of[sl];
- if (copy_wait_dedupe && idx >= (size_t) m.ring && rel == prev_rel) {
+ if (copy_wait_dedupe && strata::prefill::copy_wait_dedupe_ok(idx, m.ring, MMQ_GROUP, rel, prev_rel)) {
++iss_dedup;
} else {
cudaStreamWaitEvent(m.copy, m.used[rel], 0);