Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek CPU lane: compute the store hits while the misses are read, each missed expert as its read lands (DS4_MISS_SPLIT=0 restores the wait-for-all order); ExpertStore::ensure split into begin / wait_slots
Niko12216fcde7e305d8
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex fa051a9f3..c77728257 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -5600,26 +5600,82 @@ struct Model::Impl {struct StoreHold {Impl* m = nullptr;int layer = 0;- std::vector<int> es;+ std::vector<int> es, sl;std::vector<char> ok;+ // lazy: the reads are started here and waited for by wait_all() / wait_expert(); the destructor releasesStoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {if (!m_->store) return;m = m_;layer = l.il;for (const Ent& en : ents)if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);- m->store->ensure(layer, es.data(), (int) es.size(), ok);+ m->store->begin(layer, es.data(), (int) es.size(), ok, sl);+ }+ int idx(int e) const {+ for (size_t i = 0; i < es.size(); ++i)+ if (es[i] == e) return (int) i;+ return -1;+ }+ // expert e needs no waiting: not a store expert (RAM copy) or its reads are done+ bool ready(int e) const {+ if (!m) return true;+ const int i = idx(e);+ return i < 0 || m->store->loaded(sl[(size_t) i]);+ }+ void wait_expert(int e) {+ if (!m) return;+ const int i = idx(e);+ if (i >= 0) m->store->wait_slots(sl, ok, i, i + 1);+ }+ void wait_all() {+ if (m) m->store->wait_slots(sl, ok, 0, (int) es.size());}~StoreHold() {if (m) m->store->release(layer, es.data(), (int) es.size(), ok);}};+ // DS4_MISS_SPLIT=0: the old order (wait for every read, then compute); default: the entries whose reads are done+ // computed first, the others each as its read lands (the same bits: every entry's rows are independent)void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,const int* ord = nullptr) {if (ldx == 0) ldx = (size_t) g.dim;if (l.lay.fmt == gg::kFmtMXFP4) {StoreHold hold(this, l, ents);- return cpu_experts_mx(l, x, ents, ldx, ord);+ static const bool split = std::getenv("DS4_MISS_SPLIT") == nullptr || std::atoi(std::getenv("DS4_MISS_SPLIT")) != 0;+ if (!split || ord != nullptr || !hold.m || ents.size() < 2) {+ hold.wait_all();+ return cpu_experts_mx(l, x, ents, ldx, ord);+ }+ std::vector<Ent> hot, cold;+ for (const Ent& en : ents) (hold.ready(en.e) ? hot : cold).push_back(en);+ if (cold.empty()) {+ hold.wait_all();+ return cpu_experts_mx(l, x, ents, ldx, ord);+ }+ for (const Ent& en : hot) hold.wait_expert(en.e); // (done reading: only the bookkeeping)+ if (!hot.empty()) cpu_experts_mx(l, x, hot, ldx, nullptr);+ // the ones that came in meanwhile first, then the rest in the order they land+ std::vector<char> done(cold.size(), 0);+ for (size_t left = cold.size(); left > 0;) {+ size_t pick = cold.size();+ for (size_t i = 0; i < cold.size(); ++i)+ if (!done[i] && hold.ready(cold[i].e)) {+ pick = i;+ break;+ }+ if (pick == cold.size())+ for (size_t i = 0; i < cold.size(); ++i)+ if (!done[i]) {+ pick = i;+ break;+ }+ hold.wait_expert(cold[pick].e);+ std::vector<Ent> one(1, cold[pick]);+ cpu_experts_mx(l, x, one, ldx, nullptr);+ done[pick] = 1;+ --left;+ }+ return;}const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex 0fb77af7c..4a5c030eb 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -143,34 +143,43 @@ public:/// caller releases them. False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller/// reads the others another way).bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {+ std::vector<int> sl;+ begin(layer, es, n, ok, sl);+ return wait_slots(sl, ok, 0, n);+ }+ /// The first half of ensure(): the experts' slots found or their reads started (pinned), no waiting. sl[i] is the slot or -1.+ void begin(int layer, const int* es, int n, std::vector<char>& ok, std::vector<int>& sl) {ok.assign((size_t) n, 0);- if (layer < 0 || layer >= (int) geo_.size()) return false;- std::vector<int> sl((size_t) n, -1);- {- std::lock_guard<std::mutex> lk(mu_);- for (int i = 0; i < n; ++i) {- const int e = es[i];- touch_heat(layer, e);- int s = where_[(size_t) layer][(size_t) e];- if (s >= 0) {- Slot& sp = slots_[(size_t) s];- try_ready(sp);- if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);- if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);- sp.prefetch = false;- sp.pin++;- sp.tick = ++clock_;- sl[(size_t) i] = s;- continue;- }- st_.misses.fetch_add(1, std::memory_order_relaxed);- s = start_fill(layer, e, true);+ sl.assign((size_t) n, -1);+ if (layer < 0 || layer >= (int) geo_.size()) return;+ std::lock_guard<std::mutex> lk(mu_);+ for (int i = 0; i < n; ++i) {+ const int e = es[i];+ touch_heat(layer, e);+ int s = where_[(size_t) layer][(size_t) e];+ if (s >= 0) {+ Slot& sp = slots_[(size_t) s];+ try_ready(sp);+ if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);+ if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);+ sp.prefetch = false;+ sp.pin++;+ sp.tick = ++clock_;sl[(size_t) i] = s;+ continue;}+ st_.misses.fetch_add(1, std::memory_order_relaxed);+ s = start_fill(layer, e, true);+ sl[(size_t) i] = s;}+ }+ /// True when the reads of slot sl (from begin) are all done.+ bool loaded(int s) const { return s >= 0 && slots_[(size_t) s].left.load(std::memory_order_acquire) <= 0; }+ /// The second half: waits for the slots sl[a..b) and marks ok[i] for the good ones. False if any failed or had no slot.+ bool wait_slots(std::vector<int>& sl, std::vector<char>& ok, int a, int b) {const auto t0 = std::chrono::steady_clock::now();bool all = true;- for (int i = 0; i < n; ++i) {+ for (int i = a; i < b; ++i) {const int s = sl[(size_t) i];if (s < 0) {all = false;