Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek CPU lane: compute the store hits while the misses are read, each missed expert as its read lands (DS4_MISS_SPLIT=0 restores the wait-for-all order); ExpertStore::ensure split into begin / wait_slots

Niko12216fcde7e305d8
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index fa051a9f3..c77728257 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -5600,26 +5600,82 @@ struct Model::Impl {
struct StoreHold {
Impl* m = nullptr;
int layer = 0;
- std::vector<int> es;
+ std::vector<int> es, sl;
std::vector<char> ok;
+ // lazy: the reads are started here and waited for by wait_all() / wait_expert(); the destructor releases
StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
if (!m_->store) return;
m = m_;
layer = l.il;
for (const Ent& en : ents)
if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
- m->store->ensure(layer, es.data(), (int) es.size(), ok);
+ m->store->begin(layer, es.data(), (int) es.size(), ok, sl);
+ }
+ int idx(int e) const {
+ for (size_t i = 0; i < es.size(); ++i)
+ if (es[i] == e) return (int) i;
+ return -1;
+ }
+ // expert e needs no waiting: not a store expert (RAM copy) or its reads are done
+ bool ready(int e) const {
+ if (!m) return true;
+ const int i = idx(e);
+ return i < 0 || m->store->loaded(sl[(size_t) i]);
+ }
+ void wait_expert(int e) {
+ if (!m) return;
+ const int i = idx(e);
+ if (i >= 0) m->store->wait_slots(sl, ok, i, i + 1);
+ }
+ void wait_all() {
+ if (m) m->store->wait_slots(sl, ok, 0, (int) es.size());
}
~StoreHold() {
if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
}
};
+ // DS4_MISS_SPLIT=0: the old order (wait for every read, then compute); default: the entries whose reads are done
+ // computed first, the others each as its read lands (the same bits: every entry's rows are independent)
void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
const int* ord = nullptr) {
if (ldx == 0) ldx = (size_t) g.dim;
if (l.lay.fmt == gg::kFmtMXFP4) {
StoreHold hold(this, l, ents);
- return cpu_experts_mx(l, x, ents, ldx, ord);
+ static const bool split = std::getenv("DS4_MISS_SPLIT") == nullptr || std::atoi(std::getenv("DS4_MISS_SPLIT")) != 0;
+ if (!split || ord != nullptr || !hold.m || ents.size() < 2) {
+ hold.wait_all();
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
+ std::vector<Ent> hot, cold;
+ for (const Ent& en : ents) (hold.ready(en.e) ? hot : cold).push_back(en);
+ if (cold.empty()) {
+ hold.wait_all();
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
+ for (const Ent& en : hot) hold.wait_expert(en.e); // (done reading: only the bookkeeping)
+ if (!hot.empty()) cpu_experts_mx(l, x, hot, ldx, nullptr);
+ // the ones that came in meanwhile first, then the rest in the order they land
+ std::vector<char> done(cold.size(), 0);
+ for (size_t left = cold.size(); left > 0;) {
+ size_t pick = cold.size();
+ for (size_t i = 0; i < cold.size(); ++i)
+ if (!done[i] && hold.ready(cold[i].e)) {
+ pick = i;
+ break;
+ }
+ if (pick == cold.size())
+ for (size_t i = 0; i < cold.size(); ++i)
+ if (!done[i]) {
+ pick = i;
+ break;
+ }
+ hold.wait_expert(cold[pick].e);
+ std::vector<Ent> one(1, cold[pick]);
+ cpu_experts_mx(l, x, one, ldx, nullptr);
+ done[pick] = 1;
+ --left;
+ }
+ return;
}
const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hpp
index 0fb77af7c..4a5c030eb 100644
--- a/src/dsv41/ds4_store.hpp
+++ b/src/dsv41/ds4_store.hpp
@@ -143,34 +143,43 @@ public:
/// caller releases them. False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller
/// reads the others another way).
bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {
+ std::vector<int> sl;
+ begin(layer, es, n, ok, sl);
+ return wait_slots(sl, ok, 0, n);
+ }
+ /// The first half of ensure(): the experts' slots found or their reads started (pinned), no waiting. sl[i] is the slot or -1.
+ void begin(int layer, const int* es, int n, std::vector<char>& ok, std::vector<int>& sl) {
ok.assign((size_t) n, 0);
- if (layer < 0 || layer >= (int) geo_.size()) return false;
- std::vector<int> sl((size_t) n, -1);
- {
- std::lock_guard<std::mutex> lk(mu_);
- for (int i = 0; i < n; ++i) {
- const int e = es[i];
- touch_heat(layer, e);
- int s = where_[(size_t) layer][(size_t) e];
- if (s >= 0) {
- Slot& sp = slots_[(size_t) s];
- try_ready(sp);
- if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);
- if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);
- sp.prefetch = false;
- sp.pin++;
- sp.tick = ++clock_;
- sl[(size_t) i] = s;
- continue;
- }
- st_.misses.fetch_add(1, std::memory_order_relaxed);
- s = start_fill(layer, e, true);
+ sl.assign((size_t) n, -1);
+ if (layer < 0 || layer >= (int) geo_.size()) return;
+ std::lock_guard<std::mutex> lk(mu_);
+ for (int i = 0; i < n; ++i) {
+ const int e = es[i];
+ touch_heat(layer, e);
+ int s = where_[(size_t) layer][(size_t) e];
+ if (s >= 0) {
+ Slot& sp = slots_[(size_t) s];
+ try_ready(sp);
+ if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);
+ if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);
+ sp.prefetch = false;
+ sp.pin++;
+ sp.tick = ++clock_;
sl[(size_t) i] = s;
+ continue;
}
+ st_.misses.fetch_add(1, std::memory_order_relaxed);
+ s = start_fill(layer, e, true);
+ sl[(size_t) i] = s;
}
+ }
+ /// True when the reads of slot sl (from begin) are all done.
+ bool loaded(int s) const { return s >= 0 && slots_[(size_t) s].left.load(std::memory_order_acquire) <= 0; }
+ /// The second half: waits for the slots sl[a..b) and marks ok[i] for the good ones. False if any failed or had no slot.
+ bool wait_slots(std::vector<int>& sl, std::vector<char>& ok, int a, int b) {
const auto t0 = std::chrono::steady_clock::now();
bool all = true;
- for (int i = 0; i < n; ++i) {
+ for (int i = a; i < b; ++i) {
const int s = sl[(size_t) i];
if (s < 0) {
all = false;