Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: store auto size, profile preload, decode staging zc_tab (T=1), release plumbing

Niko12212e61fef4d193
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index e0b5bdc45..fe13bec85 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2434,6 +2434,7 @@ struct Model::Impl {
if (!load_experts(err)) return false;
if (pk.gguf) make_pools();
init_store();
+ preload_store();
if (chunk > 0) {
// the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
const double tr = now_ms();
@@ -4190,9 +4191,17 @@ struct Model::Impl {
}
double store_gb() const {
- if (!pk.gguf) return 0.0;
- if (const char* v = std::getenv("DS4_STORE_GB")) return std::atof(v);
- return o.ssd_cache_gb;
+ static double cached = -2.0; // (decided once: the layout of the blobs depends on it)
+ if (cached > -1.5) return cached;
+ double gb = o.ssd_cache_gb;
+ if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);
+ if (!pk.gguf) gb = 0.0;
+ else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most
+ const uint64_t avail = ram_avail();
+ const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;
+ gb = room >= 4.0 ? std::min(48.0, room) : 0.0;
+ }
+ return cached = gb;
}
void init_store() {
const double gb = store_gb();
@@ -4279,6 +4288,34 @@ struct Model::Impl {
zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
}
}
+ // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)
+ void preload_store() {
+ if (!store) return;
+ if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;
+ std::vector<std::pair<double, std::pair<int, int>>> cand;
+ double tot = 0;
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) tot += p[e];
+ }
+ if (tot <= 0) return; // (no profile yet: the first tokens fill it)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ std::vector<std::pair<int, int>> list;
+ list.reserve(cand.size());
+ for (const auto& c : cand) list.push_back(c.second);
+ const double t0 = now_ms();
+ const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;
+ const size_t n = store->preload(list, leave);
+ std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,
+ (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
+ }
// at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
// each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
void store_sync() {
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index df203f123..c291d13ac 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -35,7 +35,7 @@ struct Options {
// the experts the VRAM leaves over are cached in RAM by a store of their own (ds4_store.hpp): whole experts read unbuffered into
// slots, a replacement policy (0 lru, 1 lfu, 2 lfu then lru), filled on demand and by the lookahead's guesses. GB of RAM at
// most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before
- double ssd_cache_gb = 0.0;
+ double ssd_cache_gb = -1.0; // < 0: auto (the RAM the process can get less a floor, at most 48 GB; off below 4 GB), 0: off
int ssd_policy = 0;
bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this
diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hpp
index 3c7b923a7..fa0f86443 100644
--- a/src/dsv41/ds4_store.hpp
+++ b/src/dsv41/ds4_store.hpp
@@ -196,6 +196,44 @@ public:
if (where_[(size_t) layer][(size_t) e] >= 0) return;
if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);
}
+ /// Reads the experts of `list` (layer, expert; hottest first) into free slots and returns when they are in (a start that
+ /// is already warm: the profile's hottest, so the first tokens find them). The first `list.size()` fit the arena less
+ /// `leave` slots; returns how many were read.
+ size_t preload(const std::vector<std::pair<int, int>>& list, size_t leave) {
+ size_t n = 0;
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ const size_t room = nmax_ > leave ? nmax_ - leave : 0;
+ const size_t take = std::min(list.size(), room);
+ for (size_t k = take; k-- > 0;) { // (coldest first: the hottest get the newest ticks)
+ const int layer = list[k].first, e = list[k].second;
+ if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) continue;
+ if (where_[(size_t) layer][(size_t) e] >= 0) continue;
+ const int s = start_fill(layer, e, false);
+ if (s < 0) break;
+ slots_[(size_t) s].prefetch = false; // (warm, not a guess)
+ ++n;
+ }
+ }
+ for (size_t i = 0;; ++i) { // until every read is done
+ bool busy = false;
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ for (size_t j = 0; j < nalloc_; ++j) {
+ Slot& sp = slots_[j];
+ if (sp.state.load(std::memory_order_acquire) != kLoading) continue;
+ if (sp.left.load(std::memory_order_acquire) > 0) busy = true;
+ else {
+ try_ready(sp);
+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+ }
+ }
+ }
+ if (!busy) break;
+ io_->wait_any(5000);
+ }
+ return n;
+ }
/// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).
void touch(int layer, int e) {
if (layer < 0 || layer >= (int) where_.size()) return;