Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: priority SSD queue with cancellation (demand before guesses), partial GPU visibility, store shrink on RAM pressure, predsim

Niko122108224f228f40
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 6c1a7b1e6..bd23b83c0 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -4236,6 +4236,7 @@ struct Model::Impl {
cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
cfg.ram_avail = [] { return ram_avail(); };
cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
+ cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;
// (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a
// 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane
// took which expert, the CPU lane's are the mmap path's byte for byte)
@@ -5244,6 +5245,19 @@ struct Model::Impl {
void cache_tick(int T) {
cache_tick_swaps(T);
store_sync();
+ store_watch();
+ }
+ // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time
+ // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free
+ void store_watch() {
+ if (!store) return;
+ static int tick = 0;
+ if (++tick % 16 != 0) return;
+ const uint64_t avail = ram_avail();
+ if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;
+ const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));
+ const size_t n = store->shrink_chunks(chunks);
+ if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());
}
void cache_tick_swaps(int T) {
if (!o.expert_cache || cache_ths.empty()) return;
@@ -5467,6 +5481,7 @@ struct Model::Impl {
Layer& l = L[(size_t) li];
uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
ents.clear();
+ if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)
// lookahead: the guesses made for this layer (when they are for this request), counted on decode
const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
? ahead_fl.data() + (size_t) li * 1024 : nullptr;
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index e37e44223..50f6ee5af 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -392,6 +392,15 @@ inline bool range_commit(void* p, size_t bytes) {
return ::mprotect(p, bytes, PROT_READ | PROT_WRITE) == 0;
#endif
}
+/// A committed piece of a reserved range given back to the system (its address range stays reserved)
+inline void range_decommit(void* p, size_t bytes) {
+#if defined(_WIN32)
+ VirtualFree(p, bytes, MEM_DECOMMIT);
+#else
+ ::madvise(p, bytes, MADV_DONTNEED);
+ ::mprotect(p, bytes, PROT_NONE);
+#endif
+}
inline void range_release(void* p, size_t bytes) {
if (p == nullptr) return;
#if defined(_WIN32)
diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hpp
index 6a6a2dd4d..767cf9d65 100644
--- a/src/dsv41/ds4_pool.hpp
+++ b/src/dsv41/ds4_pool.hpp
@@ -145,6 +145,7 @@ public:
const uint8_t* mem = nullptr; // non-null: a copy of len bytes from memory to dst instead of a read
const void* map = nullptr; // the range in the file's mapping: all in the page cache, it is read buffered
uint8_t* keep = nullptr; // non-null (a read): the len bytes read copied here as well
+ std::atomic<bool>* cancel = nullptr; // set before the job starts: it is dropped unread (bad is set, left counted down)
};
explicit IoQueue(int n) {
for (int i = 0; i < n; ++i) th_.emplace_back([this] { loop(); });
@@ -157,10 +158,11 @@ public:
cv_.notify_all();
for (auto& t : th_) t.join();
}
- void push(const Job* jobs, int n) {
+ /// low: a read that can wait (a guess) - the jobs of the first queue are all taken before any of this one's
+ void push(const Job* jobs, int n, bool low = false) {
{
std::lock_guard<std::mutex> lk(mu_);
- for (int i = 0; i < n; ++i) q_.push_back(jobs[i]);
+ for (int i = 0; i < n; ++i) (low ? ql_ : q_).push_back(jobs[i]);
}
if (n == 1) cv_.notify_one();
else cv_.notify_all();
@@ -180,10 +182,21 @@ private:
Job j;
{
std::unique_lock<std::mutex> lk(mu_);
- cv_.wait(lk, [&] { return quit_ || !q_.empty(); });
- if (quit_ && q_.empty()) return;
- j = q_.front();
- q_.pop_front();
+ cv_.wait(lk, [&] { return quit_ || !q_.empty() || !ql_.empty(); });
+ if (quit_ && q_.empty() && ql_.empty()) return;
+ std::deque<Job>& qq = q_.empty() ? ql_ : q_;
+ j = qq.front();
+ qq.pop_front();
+ }
+ if (j.cancel != nullptr && j.cancel->load(std::memory_order_acquire)) { // (dropped unread)
+ if (j.bad != nullptr) j.bad->store(true, std::memory_order_release);
+ if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);
+ {
+ std::lock_guard<std::mutex> lk(done_mu_);
+ ++done_;
+ }
+ done_cv_.notify_all();
+ continue;
}
const auto t0 = std::chrono::steady_clock::now();
const uint64_t lead = j.off % os::kDirectAlign;
@@ -215,7 +228,7 @@ private:
std::vector<std::thread> th_;
std::mutex mu_, done_mu_;
std::condition_variable cv_, done_cv_;
- std::deque<Job> q_;
+ std::deque<Job> q_, ql_;
uint64_t done_ = 0;
bool quit_ = false;
std::atomic<uint64_t> busy_us_{0}, direct_{0}, buffered_{0};
diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hpp
index fa0f86443..c7b35ec7c 100644
--- a/src/dsv41/ds4_store.hpp
+++ b/src/dsv41/ds4_store.hpp
@@ -48,7 +48,7 @@ public:
enum Policy { kLru = 0, kLfu = 1 };
struct Stats {
std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};
- std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0};
+ std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0};
};
struct Vis { // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)
int layer = 0, e = 0;
@@ -67,6 +67,7 @@ public:
std::function<void*(void* host, size_t bytes)> map_chunk;
std::function<void(void* host)> unmap_chunk;
size_t reserve = 192; // slots kept evictable (not visible) for the window's demand and guesses
+ int vis_percent = 100; // the share of experts the GPU may take (by a fixed hash): the rest stay the CPU lane's, so both lanes work
};
bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {
@@ -234,6 +235,15 @@ public:
}
return n;
}
+ /// Guesses still waiting in the queue for a layer the token is past: dropped unread (their slots free once the queue has gone by)
+ void cancel_stale(int layer_now) {
+ std::lock_guard<std::mutex> lk(mu_);
+ for (size_t i = 0; i < nalloc_; ++i) {
+ Slot& sp = slots_[i];
+ if (sp.prefetch && sp.pin == 0 && sp.layer >= 0 && sp.layer <= layer_now && sp.state.load(std::memory_order_acquire) == kLoading)
+ sp.cancel.store(true, std::memory_order_release);
+ }
+ }
/// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).
void touch(int layer, int e) {
if (layer < 0 || layer >= (int) where_.size()) return;
@@ -276,7 +286,7 @@ public:
continue;
}
if (sp.pin == 0) ++nfree;
- if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch) {
+ if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch && vis_hash(sp.layer, sp.expert) < cfg_.vis_percent) {
sp.visible = true;
++nvis;
if (sp.pin == 0) --nfree;
@@ -299,6 +309,42 @@ public:
}
return ev.size();
}
+ /// RAM is short: gives back whole chunks from the end of the arena (their experts forgotten) - up to `want`, stopping at one
+ /// with a slot in use (pinned, loading, or GPU-visible). Returns the chunks given back.
+ size_t shrink_chunks(size_t want) {
+ std::lock_guard<std::mutex> lk(mu_);
+ size_t done = 0;
+ while (done < want && !chunks_.empty()) {
+ const size_t c = chunks_.size() - 1, first = c * kChunk;
+ bool busy = false;
+ for (size_t i = first; i < nalloc_ && !busy; ++i) {
+ const Slot& sp = slots_[i];
+ busy = sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) == kLoading;
+ }
+ if (busy) break;
+ for (size_t i = first; i < nalloc_; ++i) {
+ Slot& sp = slots_[i];
+ if (sp.layer >= 0 && sp.expert >= 0) {
+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+ st_.evictions.fetch_add(1, std::memory_order_relaxed);
+ }
+ sp.layer = sp.expert = -1;
+ sp.prefetch = false;
+ sp.state.store(kFree, std::memory_order_release);
+ sp.base = nullptr;
+ }
+ if (cfg_.unmap_chunk && chunk_dev_[c] != nullptr) cfg_.unmap_chunk(chunks_[c].first);
+ os::range_decommit(chunks_[c].first, chunks_[c].second);
+ chunks_.pop_back();
+ chunk_dev_.pop_back();
+ nalloc_ = first;
+ free_hint_ = std::min(free_hint_, first);
+ ++done;
+ ++shrunk_;
+ }
+ return done;
+ }
+ size_t shrunk() const { return shrunk_; }
/// An expert VRAM took over: no longer a visible slot
void unmark(int layer, int e) {
if (layer < 0 || layer >= (int) where_.size()) return;
@@ -320,10 +366,11 @@ private:
int layer = -1, expert = -1, pin = 0;
bool prefetch = false, visible = false;
std::atomic<int> state{kFree}, left{0};
- std::atomic<bool> bad{false};
+ std::atomic<bool> bad{false}, cancel{false};
uint64_t tick = 0;
};
+ static int vis_hash(int layer, int e) { return (int) (((unsigned) layer * 2654435761u + (unsigned) e * 40503u) >> 7) % 100; }
double score(const Slot& sp) const {
if (cfg_.policy == kLru) return (double) sp.tick;
return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;
@@ -354,6 +401,7 @@ private:
sp.tick = ++clock_;
sp.state.store(kLoading, std::memory_order_release);
sp.bad.store(false);
+ sp.cancel.store(false);
sp.left.store(3, std::memory_order_release);
where_[(size_t) layer][(size_t) e] = s;
const LayerGeo& g = geo_[(size_t) layer];
@@ -374,9 +422,10 @@ private:
j.keep = fin;
j.left = &sp.left;
j.bad = &sp.bad;
+ j.cancel = demand ? nullptr : &sp.cancel;
}
st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);
- io_->push(jobs, 3);
+ io_->push(jobs, 3, !demand); // (a guess waits behind every demand read)
return s;
}
// loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)
@@ -392,7 +441,8 @@ private:
sp.layer = sp.expert = -1;
sp.state.store(kFree, std::memory_order_release);
sp.prefetch = false;
- st_.fails.fetch_add(1, std::memory_order_relaxed);
+ if (sp.cancel.load()) st_.cancelled.fetch_add(1, std::memory_order_relaxed);
+ else st_.fails.fetch_add(1, std::memory_order_relaxed);
return;
}
sp.state.store(kReady, std::memory_order_release);
@@ -449,6 +499,7 @@ private:
std::vector<os::fd_t> dfds_, pfds_;
size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;
uint8_t* arena_ = nullptr;
+ size_t shrunk_ = 0;
std::unique_ptr<Slot[]> slots_;
std::vector<std::pair<void*, size_t>> chunks_;
std::vector<void*> chunk_dev_;
diff --git a/tools/dsv41/win/predsim.py b/tools/dsv41/win/predsim.py
new file mode 100644
index 000000000..42b59a94b
--- /dev/null
+++ b/tools/dsv41/win/predsim.py
@@ -0,0 +1,107 @@
+"""predsim.py - token-level prediction of a decode step's experts from the token history (n-gram match), as a prefetch into an LRU
+cache, on DS4_TRACE decode traces.
+
+ predsim.py <trace> <prompt ids file> <output ids file> [--cap GB] [--warm N] [--budget MB] [--k 2]
+
+Before step i (its token known: the one the last step sampled) the experts of the step after the previous occurrence of the same
+k-gram are the guess for every layer. They are read ahead (up to --budget MB a step, hottest layers first), so a miss at use time
+is only the ones the guess missed. Reports demand hit rate, SSD MB a step (demand + wasted guesses) and the guess's recall /
+precision. The engine's lead time for these reads is a whole step (the SSD is idle while the CPU lane computes)."""
+import argparse
+import collections
+import sys
+sys.path.insert(0, r'C:\Users\AI-Server\Desktop\Strata\GitHub\wt-dsv41-win\tools\dsv41\win')
+import cachesim as cs
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('trace')
+ ap.add_argument('prompt')
+ ap.add_argument('out')
+ ap.add_argument('--cap', type=float, default=36.0)
+ ap.add_argument('--warm', type=int, default=150)
+ ap.add_argument('--budget', type=float, default=400.0, help='MB of reads ahead per step')
+ ap.add_argument('--k', type=int, default=2)
+ ap.add_argument('--sizes', default=r'C:\ds-ista-work\sizes.json')
+ a = ap.parse_args()
+ size = cs.Sizes(a.sizes)
+ st = cs.read_trace(a.trace)
+ p = [int(x) for x in open(a.prompt).read().strip().split(',')]
+ o = [int(x) for x in open(a.out).read().strip().strip(',').split(',')]
+ S = p + o
+ cap = a.cap * 1e9
+ routes = {}
+ for i, s in enumerate(st):
+ if i == 0:
+ continue
+ routes[i] = {layer: list(dict.fromkeys(rows[0])) for layer, rows, _ in s}
+ pos = lambda i: len(p) + i - 1 # noqa: E731
+ for mode in ('lru', 'lru+hist'):
+ od = collections.OrderedDict()
+ used = 0
+ last = {}
+ n = hit = 0
+ demand_b = pref_b = wasted_b = 0.0
+ rec = tot = prec = ptot = 0
+ steps = 0
+ for i in range(1, len(st)):
+ key = tuple(S[pos(i) - a.k + 1:pos(i) + 1])
+ pred = None
+ if mode == 'lru+hist' and key in last:
+ pred = routes[last[key]]
+ last[key] = i
+ counted = i >= a.warm
+ pend = {}
+ if pred is not None:
+ budget = a.budget * 1e6
+ for l in range(40):
+ for e in pred[l]:
+ kk = (l, e)
+ if kk in od or kk in pend:
+ continue
+ if budget < size(l):
+ continue
+ budget -= size(l)
+ pend[kk] = size(l)
+ if counted:
+ pref_b += sum(pend.values())
+ for l in range(40):
+ a_ = set(routes[i][l])
+ b_ = set(pred[l])
+ rec += len(a_ & b_)
+ tot += len(a_)
+ prec += len(a_ & b_)
+ ptot += len(b_)
+ steps += counted
+ for l in range(40):
+ sz = size(l)
+ for e in routes[i][l]:
+ kk = (l, e)
+ if counted:
+ n += 1
+ if kk in od:
+ od.move_to_end(kk)
+ if counted:
+ hit += 1
+ continue
+ if kk in pend: # read ahead: arrived in time
+ del pend[kk]
+ if counted:
+ hit += 1
+ elif counted:
+ demand_b += sz
+ od[kk] = sz
+ used += sz
+ while used > cap:
+ _, s0 = od.popitem(last=False)
+ used -= s0
+ if counted:
+ wasted_b += sum(pend.values()) # read ahead, not used this step (they stay in the cache only if inserted: not here)
+ print('%-9s cap %.0f GB k=%d: demand hit %.1f%%; SSD per step: demand %.0f MB + guesses %.0f MB (wasted %.0f); guess recall %.1f%% precision %.1f%%' % (
+ mode, a.cap, a.k, 100 * hit / max(1, n), demand_b / steps / 1e6, pref_b / steps / 1e6, wasted_b / steps / 1e6,
+ 100 * rec / max(1, tot), 100 * prec / max(1, ptot)))
+
+
+if __name__ == '__main__':
+ main()