Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: priority SSD queue with cancellation (demand before guesses), partial GPU visibility, store shrink on RAM pressure, predsim
Niko122108224f228f40
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 6c1a7b1e6..bd23b83c0 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4236,6 +4236,7 @@ struct Model::Impl {cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);cfg.ram_avail = [] { return ram_avail(); };cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);+ cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;// (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a// 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane// took which expert, the CPU lane's are the mmap path's byte for byte)@@ -5244,6 +5245,19 @@ struct Model::Impl {void cache_tick(int T) {cache_tick_swaps(T);store_sync();+ store_watch();+ }+ // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time+ // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free+ void store_watch() {+ if (!store) return;+ static int tick = 0;+ if (++tick % 16 != 0) return;+ const uint64_t avail = ram_avail();+ if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;+ const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));+ const size_t n = store->shrink_chunks(chunks);+ if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());}void cache_tick_swaps(int T) {if (!o.expert_cache || cache_ths.empty()) return;@@ -5467,6 +5481,7 @@ struct Model::Impl {Layer& l = L[(size_t) li];uint32_t* pr = prof.data() + (size_t) li * g.n_exp;ents.clear();+ if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)// lookahead: the guesses made for this layer (when they are for this request), counted on decodeconst uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next? ahead_fl.data() + (size_t) li * 1024 : nullptr;diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex e37e44223..50f6ee5af 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -392,6 +392,15 @@ inline bool range_commit(void* p, size_t bytes) {return ::mprotect(p, bytes, PROT_READ | PROT_WRITE) == 0;#endif}+/// A committed piece of a reserved range given back to the system (its address range stays reserved)+inline void range_decommit(void* p, size_t bytes) {+#if defined(_WIN32)+ VirtualFree(p, bytes, MEM_DECOMMIT);+#else+ ::madvise(p, bytes, MADV_DONTNEED);+ ::mprotect(p, bytes, PROT_NONE);+#endif+}inline void range_release(void* p, size_t bytes) {if (p == nullptr) return;#if defined(_WIN32)diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hppindex 6a6a2dd4d..767cf9d65 100644--- a/src/dsv41/ds4_pool.hpp+++ b/src/dsv41/ds4_pool.hpp@@ -145,6 +145,7 @@ public:const uint8_t* mem = nullptr; // non-null: a copy of len bytes from memory to dst instead of a readconst void* map = nullptr; // the range in the file's mapping: all in the page cache, it is read buffereduint8_t* keep = nullptr; // non-null (a read): the len bytes read copied here as well+ std::atomic<bool>* cancel = nullptr; // set before the job starts: it is dropped unread (bad is set, left counted down)};explicit IoQueue(int n) {for (int i = 0; i < n; ++i) th_.emplace_back([this] { loop(); });@@ -157,10 +158,11 @@ public:cv_.notify_all();for (auto& t : th_) t.join();}- void push(const Job* jobs, int n) {+ /// low: a read that can wait (a guess) - the jobs of the first queue are all taken before any of this one's+ void push(const Job* jobs, int n, bool low = false) {{std::lock_guard<std::mutex> lk(mu_);- for (int i = 0; i < n; ++i) q_.push_back(jobs[i]);+ for (int i = 0; i < n; ++i) (low ? ql_ : q_).push_back(jobs[i]);}if (n == 1) cv_.notify_one();else cv_.notify_all();@@ -180,10 +182,21 @@ private:Job j;{std::unique_lock<std::mutex> lk(mu_);- cv_.wait(lk, [&] { return quit_ || !q_.empty(); });- if (quit_ && q_.empty()) return;- j = q_.front();- q_.pop_front();+ cv_.wait(lk, [&] { return quit_ || !q_.empty() || !ql_.empty(); });+ if (quit_ && q_.empty() && ql_.empty()) return;+ std::deque<Job>& qq = q_.empty() ? ql_ : q_;+ j = qq.front();+ qq.pop_front();+ }+ if (j.cancel != nullptr && j.cancel->load(std::memory_order_acquire)) { // (dropped unread)+ if (j.bad != nullptr) j.bad->store(true, std::memory_order_release);+ if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);+ {+ std::lock_guard<std::mutex> lk(done_mu_);+ ++done_;+ }+ done_cv_.notify_all();+ continue;}const auto t0 = std::chrono::steady_clock::now();const uint64_t lead = j.off % os::kDirectAlign;@@ -215,7 +228,7 @@ private:std::vector<std::thread> th_;std::mutex mu_, done_mu_;std::condition_variable cv_, done_cv_;- std::deque<Job> q_;+ std::deque<Job> q_, ql_;uint64_t done_ = 0;bool quit_ = false;std::atomic<uint64_t> busy_us_{0}, direct_{0}, buffered_{0};diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex fa0f86443..c7b35ec7c 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -48,7 +48,7 @@ public:enum Policy { kLru = 0, kLfu = 1 };struct Stats {std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};- std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0};+ std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0};};struct Vis { // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)int layer = 0, e = 0;@@ -67,6 +67,7 @@ public:std::function<void*(void* host, size_t bytes)> map_chunk;std::function<void(void* host)> unmap_chunk;size_t reserve = 192; // slots kept evictable (not visible) for the window's demand and guesses+ int vis_percent = 100; // the share of experts the GPU may take (by a fixed hash): the rest stay the CPU lane's, so both lanes work};bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {@@ -234,6 +235,15 @@ public:}return n;}+ /// Guesses still waiting in the queue for a layer the token is past: dropped unread (their slots free once the queue has gone by)+ void cancel_stale(int layer_now) {+ std::lock_guard<std::mutex> lk(mu_);+ for (size_t i = 0; i < nalloc_; ++i) {+ Slot& sp = slots_[i];+ if (sp.prefetch && sp.pin == 0 && sp.layer >= 0 && sp.layer <= layer_now && sp.state.load(std::memory_order_acquire) == kLoading)+ sp.cancel.store(true, std::memory_order_release);+ }+ }/// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).void touch(int layer, int e) {if (layer < 0 || layer >= (int) where_.size()) return;@@ -276,7 +286,7 @@ public:continue;}if (sp.pin == 0) ++nfree;- if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch) {+ if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch && vis_hash(sp.layer, sp.expert) < cfg_.vis_percent) {sp.visible = true;++nvis;if (sp.pin == 0) --nfree;@@ -299,6 +309,42 @@ public:}return ev.size();}+ /// RAM is short: gives back whole chunks from the end of the arena (their experts forgotten) - up to `want`, stopping at one+ /// with a slot in use (pinned, loading, or GPU-visible). Returns the chunks given back.+ size_t shrink_chunks(size_t want) {+ std::lock_guard<std::mutex> lk(mu_);+ size_t done = 0;+ while (done < want && !chunks_.empty()) {+ const size_t c = chunks_.size() - 1, first = c * kChunk;+ bool busy = false;+ for (size_t i = first; i < nalloc_ && !busy; ++i) {+ const Slot& sp = slots_[i];+ busy = sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) == kLoading;+ }+ if (busy) break;+ for (size_t i = first; i < nalloc_; ++i) {+ Slot& sp = slots_[i];+ if (sp.layer >= 0 && sp.expert >= 0) {+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;+ st_.evictions.fetch_add(1, std::memory_order_relaxed);+ }+ sp.layer = sp.expert = -1;+ sp.prefetch = false;+ sp.state.store(kFree, std::memory_order_release);+ sp.base = nullptr;+ }+ if (cfg_.unmap_chunk && chunk_dev_[c] != nullptr) cfg_.unmap_chunk(chunks_[c].first);+ os::range_decommit(chunks_[c].first, chunks_[c].second);+ chunks_.pop_back();+ chunk_dev_.pop_back();+ nalloc_ = first;+ free_hint_ = std::min(free_hint_, first);+ ++done;+ ++shrunk_;+ }+ return done;+ }+ size_t shrunk() const { return shrunk_; }/// An expert VRAM took over: no longer a visible slotvoid unmark(int layer, int e) {if (layer < 0 || layer >= (int) where_.size()) return;@@ -320,10 +366,11 @@ private:int layer = -1, expert = -1, pin = 0;bool prefetch = false, visible = false;std::atomic<int> state{kFree}, left{0};- std::atomic<bool> bad{false};+ std::atomic<bool> bad{false}, cancel{false};uint64_t tick = 0;};+ static int vis_hash(int layer, int e) { return (int) (((unsigned) layer * 2654435761u + (unsigned) e * 40503u) >> 7) % 100; }double score(const Slot& sp) const {if (cfg_.policy == kLru) return (double) sp.tick;return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;@@ -354,6 +401,7 @@ private:sp.tick = ++clock_;sp.state.store(kLoading, std::memory_order_release);sp.bad.store(false);+ sp.cancel.store(false);sp.left.store(3, std::memory_order_release);where_[(size_t) layer][(size_t) e] = s;const LayerGeo& g = geo_[(size_t) layer];@@ -374,9 +422,10 @@ private:j.keep = fin;j.left = &sp.left;j.bad = &sp.bad;+ j.cancel = demand ? nullptr : &sp.cancel;}st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);- io_->push(jobs, 3);+ io_->push(jobs, 3, !demand); // (a guess waits behind every demand read)return s;}// loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)@@ -392,7 +441,8 @@ private:sp.layer = sp.expert = -1;sp.state.store(kFree, std::memory_order_release);sp.prefetch = false;- st_.fails.fetch_add(1, std::memory_order_relaxed);+ if (sp.cancel.load()) st_.cancelled.fetch_add(1, std::memory_order_relaxed);+ else st_.fails.fetch_add(1, std::memory_order_relaxed);return;}sp.state.store(kReady, std::memory_order_release);@@ -449,6 +499,7 @@ private:std::vector<os::fd_t> dfds_, pfds_;size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;uint8_t* arena_ = nullptr;+ size_t shrunk_ = 0;std::unique_ptr<Slot[]> slots_;std::vector<std::pair<void*, size_t>> chunks_;std::vector<void*> chunk_dev_;diff --git a/tools/dsv41/win/predsim.py b/tools/dsv41/win/predsim.pynew file mode 100644index 000000000..42b59a94b--- /dev/null+++ b/tools/dsv41/win/predsim.py@@ -0,0 +1,107 @@+"""predsim.py - token-level prediction of a decode step's experts from the token history (n-gram match), as a prefetch into an LRU+cache, on DS4_TRACE decode traces.++ predsim.py <trace> <prompt ids file> <output ids file> [--cap GB] [--warm N] [--budget MB] [--k 2]++Before step i (its token known: the one the last step sampled) the experts of the step after the previous occurrence of the same+k-gram are the guess for every layer. They are read ahead (up to --budget MB a step, hottest layers first), so a miss at use time+is only the ones the guess missed. Reports demand hit rate, SSD MB a step (demand + wasted guesses) and the guess's recall /+precision. The engine's lead time for these reads is a whole step (the SSD is idle while the CPU lane computes)."""+import argparse+import collections+import sys+sys.path.insert(0, r'C:\Users\AI-Server\Desktop\Strata\GitHub\wt-dsv41-win\tools\dsv41\win')+import cachesim as cs+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument('trace')+ ap.add_argument('prompt')+ ap.add_argument('out')+ ap.add_argument('--cap', type=float, default=36.0)+ ap.add_argument('--warm', type=int, default=150)+ ap.add_argument('--budget', type=float, default=400.0, help='MB of reads ahead per step')+ ap.add_argument('--k', type=int, default=2)+ ap.add_argument('--sizes', default=r'C:\ds-ista-work\sizes.json')+ a = ap.parse_args()+ size = cs.Sizes(a.sizes)+ st = cs.read_trace(a.trace)+ p = [int(x) for x in open(a.prompt).read().strip().split(',')]+ o = [int(x) for x in open(a.out).read().strip().strip(',').split(',')]+ S = p + o+ cap = a.cap * 1e9+ routes = {}+ for i, s in enumerate(st):+ if i == 0:+ continue+ routes[i] = {layer: list(dict.fromkeys(rows[0])) for layer, rows, _ in s}+ pos = lambda i: len(p) + i - 1 # noqa: E731+ for mode in ('lru', 'lru+hist'):+ od = collections.OrderedDict()+ used = 0+ last = {}+ n = hit = 0+ demand_b = pref_b = wasted_b = 0.0+ rec = tot = prec = ptot = 0+ steps = 0+ for i in range(1, len(st)):+ key = tuple(S[pos(i) - a.k + 1:pos(i) + 1])+ pred = None+ if mode == 'lru+hist' and key in last:+ pred = routes[last[key]]+ last[key] = i+ counted = i >= a.warm+ pend = {}+ if pred is not None:+ budget = a.budget * 1e6+ for l in range(40):+ for e in pred[l]:+ kk = (l, e)+ if kk in od or kk in pend:+ continue+ if budget < size(l):+ continue+ budget -= size(l)+ pend[kk] = size(l)+ if counted:+ pref_b += sum(pend.values())+ for l in range(40):+ a_ = set(routes[i][l])+ b_ = set(pred[l])+ rec += len(a_ & b_)+ tot += len(a_)+ prec += len(a_ & b_)+ ptot += len(b_)+ steps += counted+ for l in range(40):+ sz = size(l)+ for e in routes[i][l]:+ kk = (l, e)+ if counted:+ n += 1+ if kk in od:+ od.move_to_end(kk)+ if counted:+ hit += 1+ continue+ if kk in pend: # read ahead: arrived in time+ del pend[kk]+ if counted:+ hit += 1+ elif counted:+ demand_b += sz+ od[kk] = sz+ used += sz+ while used > cap:+ _, s0 = od.popitem(last=False)+ used -= s0+ if counted:+ wasted_b += sum(pend.values()) # read ahead, not used this step (they stay in the cache only if inserted: not here)+ print('%-9s cap %.0f GB k=%d: demand hit %.1f%%; SSD per step: demand %.0f MB + guesses %.0f MB (wasted %.0f); guess recall %.1f%% precision %.1f%%' % (+ mode, a.cap, a.k, 100 * hit / max(1, n), demand_b / steps / 1e6, pref_b / steps / 1e6, wasted_b / steps / 1e6,+ 100 * rec / max(1, tot), 100 * prec / max(1, ptot)))+++if __name__ == '__main__':+ main()