Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the RAM store no longer misses the experts the VRAM cache gives back to the CPU lane: the preload fills the slots the lane leaves over (and the 256 reserved ones, evictable first) with the VRAM experts the profile counts least and the lane experts it never saw (DS4_STORE_SPARE=0 off), and an expert leaving VRAM is copied into a store slot from its VRAM blob (put_begin/put_done, DS4_STORE_PUT=0 off) instead of being read from the SSD while the lane waits. r9700 1 card 2.00, store forced (--ssd-cache 140): decode code 23.7 -> 26.2, agent 20.7 -> 24.6 tok/s, misses 1.5/2.6 -> 0.5/0.6 a token; P2003 RTX 3060: no loss, ids identical
Niko1221dfaae03b009e
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex b8d3991d8..650b3ab4e 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4767,8 +4767,28 @@ struct Model::Impl {std::vector<std::pair<int, int>> list;list.reserve(cand.size());for (const auto& c : cand) list.push_back(c.second);+ // The slots the CPU lane's experts leave over: the experts the VRAM cache holds that the profile counts least (the cache gives+ // up its coldest to the CPU lane, and a store without them reads each from the SSD while the CPU lane waits - 1.8 misses a+ // token on the 251 GB R9700 box, 2.9 ms of a 49 ms token), then the lane's experts the profile never saw. Last in the list: they+ // take only the slots the likely ones left. DS4_STORE_SPARE=0: off.+ if (!(std::getenv("DS4_STORE_SPARE") && std::atoi(std::getenv("DS4_STORE_SPARE")) == 0)) {+ std::vector<std::pair<double, std::pair<int, int>>> vr, zero;+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0) vr.push_back({(double) p[e] / (double) l.blob, {l.il, e}});+ else if (p[e] == 0) zero.push_back({0.0, {l.il, e}});+ }+ }+ std::stable_sort(vr.begin(), vr.end(), [](const auto& a, const auto& b) { return a.first < b.first; });+ for (const auto& c : vr) list.push_back(c.second);+ for (const auto& c : zero) list.push_back(c.second);+ }const double t0 = now_ms();- const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;+ // (with the spare experts in the list the reserve is theirs: last in the list, so the oldest ticks, so the first a demand read replaces)+ const bool spare_on = !(std::getenv("DS4_STORE_SPARE") && std::atoi(std::getenv("DS4_STORE_SPARE")) == 0);+ const size_t leave = spare_on ? 0 : (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;// The first 48 GB (what a store held before it could take every expert) are read before the first token; the rest of the// hottest-first list is read after it, in pieces of 128 experts on a thread of its own, behind the model's own reads (a// guess's priority) - the same bytes. Opt-in (DS4_STORE_BG=1): "ready" comes 9 s sooner on the 251 GB R9700 box (24 s against 34), but a prompt sent at@@ -6155,6 +6175,19 @@ struct Model::Impl {}if (got) blob_to_ram(l, sw_b, sw.ram);else sw.ram = -1; // (back to the file's pages then)+ } else if (ok && store && l.nm == 1 && l.lay.fmt == gg::kFmtMXFP4 && !(std::getenv("DS4_STORE_PUT") && std::atoi(std::getenv("DS4_STORE_PUT")) == 0)) {+ // `out` goes back to the CPU lane: its blob is in VRAM, so it is copied into a store slot now - the lane would otherwise read it from+ // the SSD and wait for it (1-2 misses a token on the 251 GB R9700 box). DS4_STORE_PUT=0: off.+ uint8_t* dst = store->put_begin(l.il, sw.out);+ if (dst != nullptr) {+ const Layer& x = member(l, 0);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ const bool got = cudaMemcpyAsync(dst, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l), l.blob, cudaMemcpyDeviceToHost, cs) ==+ cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ store->put_done(l.il, sw.out, got);+ }}if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for bytestd::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex ea6b1779e..23c01ebf9 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -48,7 +48,7 @@ public:enum Policy { kLru = 0, kLfu = 1 };struct Stats {std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};- std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0};+ std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0}, put{0};};struct Vis { // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)int layer = 0, e = 0;@@ -255,6 +255,52 @@ public:}return n;}+ /// A slot for (layer, e) that the caller fills with the expert's blob itself (an expert the VRAM cache gives back to the CPU lane: its bytes+ /// are in memory already, so no read from the SSD while the lane waits). Returns the blob's address, or nullptr (it is here already, or+ /// no slot can be had). The slot is pinned and not findable until put_done().+ uint8_t* put_begin(int layer, int e) {+ if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return nullptr;+ std::lock_guard<std::mutex> lk(mu_);+ if (where_[(size_t) layer][(size_t) e] >= 0) return nullptr;+ const int s = take_slot(false);+ if (s < 0) return nullptr;+ Slot& sp = slots_[(size_t) s];+ if (sp.layer >= 0 && sp.expert >= 0) {+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;+ if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);+ st_.evictions.fetch_add(1, std::memory_order_relaxed);+ }+ sp.layer = layer;+ sp.expert = e;+ sp.pin = 1;+ sp.prefetch = false;+ sp.visible = false;+ sp.tick = ++clock_;+ sp.bad.store(false);+ sp.cancel.store(false);+ sp.left.store(1, std::memory_order_release);+ sp.state.store(kLoading, std::memory_order_release);+ where_[(size_t) layer][(size_t) e] = s;+ st_.put.fetch_add(1, std::memory_order_relaxed);+ return sp.base;+ }+ void put_done(int layer, int e, bool ok) {+ std::lock_guard<std::mutex> lk(mu_);+ const int s = where_[(size_t) layer][(size_t) e];+ if (s < 0) return;+ Slot& sp = slots_[(size_t) s];+ if (sp.pin > 0) sp.pin--;+ if (ok) {+ sp.left.store(0, std::memory_order_release);+ sp.state.store(kReady, std::memory_order_release);+ } else {+ where_[(size_t) layer][(size_t) e] = -1;+ sp.layer = sp.expert = -1;+ sp.left.store(0, std::memory_order_release);+ sp.state.store(kFree, std::memory_order_release);+ free_hint_ = std::min<size_t>(free_hint_, (size_t) s);+ }+ }/// Guesses still waiting in the queue for a layer the token is past: dropped unread (their slots free once the queue has gone by)void cancel_stale(int layer_now) {std::lock_guard<std::mutex> lk(mu_);