Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: store auto size, profile preload, decode staging zc_tab (T=1), release plumbing
Niko12212e61fef4d193
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex e0b5bdc45..fe13bec85 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2434,6 +2434,7 @@ struct Model::Impl {if (!load_experts(err)) return false;if (pk.gguf) make_pools();init_store();+ preload_store();if (chunk > 0) {// the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slotsconst double tr = now_ms();@@ -4190,9 +4191,17 @@ struct Model::Impl {}double store_gb() const {- if (!pk.gguf) return 0.0;- if (const char* v = std::getenv("DS4_STORE_GB")) return std::atof(v);- return o.ssd_cache_gb;+ static double cached = -2.0; // (decided once: the layout of the blobs depends on it)+ if (cached > -1.5) return cached;+ double gb = o.ssd_cache_gb;+ if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);+ if (!pk.gguf) gb = 0.0;+ else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most+ const uint64_t avail = ram_avail();+ const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;+ gb = room >= 4.0 ? std::min(48.0, room) : 0.0;+ }+ return cached = gb;}void init_store() {const double gb = store_gb();@@ -4279,6 +4288,34 @@ struct Model::Impl {zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);}}+ // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)+ void preload_store() {+ if (!store) return;+ if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;+ std::vector<std::pair<double, std::pair<int, int>>> cand;+ double tot = 0;+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) tot += p[e];+ }+ if (tot <= 0) return; // (no profile yet: the first tokens fill it)+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });+ std::vector<std::pair<int, int>> list;+ list.reserve(cand.size());+ for (const auto& c : cand) list.push_back(c.second);+ const double t0 = now_ms();+ const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;+ const size_t n = store->preload(list, leave);+ std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,+ (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);+ }// at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;// each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next windowvoid store_sync() {diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex df203f123..c291d13ac 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -35,7 +35,7 @@ struct Options {// the experts the VRAM leaves over are cached in RAM by a store of their own (ds4_store.hpp): whole experts read unbuffered into// slots, a replacement policy (0 lru, 1 lfu, 2 lfu then lru), filled on demand and by the lookahead's guesses. GB of RAM at// most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before- double ssd_cache_gb = 0.0;+ double ssd_cache_gb = -1.0; // < 0: auto (the RAM the process can get less a floor, at most 48 GB; off below 4 GB), 0: offint ssd_policy = 0;bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decodingfloat dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least thisdiff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex 3c7b923a7..fa0f86443 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -196,6 +196,44 @@ public:if (where_[(size_t) layer][(size_t) e] >= 0) return;if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);}+ /// Reads the experts of `list` (layer, expert; hottest first) into free slots and returns when they are in (a start that+ /// is already warm: the profile's hottest, so the first tokens find them). The first `list.size()` fit the arena less+ /// `leave` slots; returns how many were read.+ size_t preload(const std::vector<std::pair<int, int>>& list, size_t leave) {+ size_t n = 0;+ {+ std::lock_guard<std::mutex> lk(mu_);+ const size_t room = nmax_ > leave ? nmax_ - leave : 0;+ const size_t take = std::min(list.size(), room);+ for (size_t k = take; k-- > 0;) { // (coldest first: the hottest get the newest ticks)+ const int layer = list[k].first, e = list[k].second;+ if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) continue;+ if (where_[(size_t) layer][(size_t) e] >= 0) continue;+ const int s = start_fill(layer, e, false);+ if (s < 0) break;+ slots_[(size_t) s].prefetch = false; // (warm, not a guess)+ ++n;+ }+ }+ for (size_t i = 0;; ++i) { // until every read is done+ bool busy = false;+ {+ std::lock_guard<std::mutex> lk(mu_);+ for (size_t j = 0; j < nalloc_; ++j) {+ Slot& sp = slots_[j];+ if (sp.state.load(std::memory_order_acquire) != kLoading) continue;+ if (sp.left.load(std::memory_order_acquire) > 0) busy = true;+ else {+ try_ready(sp);+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);+ }+ }+ }+ if (!busy) break;+ io_->wait_any(5000);+ }+ return n;+ }/// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).void touch(int layer, int e) {if (layer < 0 || layer >= (int) where_.size()) return;