Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: the store's slots in blob layout, GPU-visible (zero-copy mapped) with a PCIe staging copy for decode picks, node-free reserved arena
Niko12217d2307deb0a0
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 9f9a1d484..10ab1edab 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1831,6 +1831,33 @@ void route_ahead(const float* logits, const float* bias, int n, int m, int T, Reroute_ahead_kernel<<<T, 512, 0, s>>>(logits, bias, n, m, req, off);}void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { wait_seq_kernel<<<1, 32, 0, s>>>(seq_ptr, seq); }++// the RAM store's experts for the device (a decode token, window of one): a pick the VRAM table left at 0 whose expert the store holds+// (zc_tab[expert] = the slot's device address over PCIe, 0: not) is copied to staging slot i with wide coalesced loads (the PCIe+// rate; the expert kernels' own small loads over PCIe ran at a tenth of it), then the plan points at the copy+__global__ void zc_copy_kernel(const unsigned long long* __restrict__ plan, const int* __restrict__ idx,+ const unsigned long long* __restrict__ zc_tab, uint8_t* __restrict__ stage, size_t stride, size_t bytes) {+ const int i = blockIdx.y;+ if (plan[i] != 0) return;+ const unsigned long long p = zc_tab[idx[i]];+ if (p == 0) return;+ const uint4* src = (const uint4*) p;+ uint4* dst = (uint4*) (stage + (size_t) i * stride);+ const size_t n = bytes / 16;+ for (size_t o = (size_t) blockIdx.x * blockDim.x + threadIdx.x; o < n; o += (size_t) gridDim.x * blockDim.x) dst[o] = src[o];+}+__global__ void zc_fix_kernel(unsigned long long* __restrict__ plan, const int* __restrict__ idx, const unsigned long long* __restrict__ zc_tab,+ int k, uint8_t* __restrict__ stage, size_t stride) {+ const int i = threadIdx.x;+ if (i >= k || plan[i] != 0) return;+ if (zc_tab[idx[i]] != 0) plan[i] = (unsigned long long) (stage + (size_t) i * stride);+}+void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,+ cudaStream_t s) {+ zc_copy_kernel<<<dim3(64, k), 256, 0, s>>>(plan, idx, zc_tab, stage, stride, bytes);+ zc_fix_kernel<<<1, 32, 0, s>>>(plan, idx, zc_tab, k, stage, stride);+}+void init_pre(float* pre, int T, cudaStream_t s) { init_pre_kernel<<<cdiv(T * 8, 128), 128, 0, s>>>(pre, T); }void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhindex 94a13997e..829255f1d 100644--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@ -232,6 +232,10 @@ void route_publish(const float* logits, const float* bias, int n, int k, int T,/// Lookahead: the m picks a later layer will likely make (its gate's logits on this layer's FFN input, the routing's/// score + bias, the m best of n) into req->ahead[t][off ..) for the T tokens - before route_publish, which publishes them.void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s);+/// The RAM store's picks for the device (a decode token's k plan pointers): those the VRAM table left at 0 whose expert zc_tab has (the+/// slot's device address) are copied into stage slot i (stride apart, `bytes` each) and the plan then points there.+void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,+ cudaStream_t s);/// Waits (one thread, on the device) until *seq_ptr == seq.void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s);/// A chunk's routed experts as sorted rows: the picks (t, i < k) of T tokens (idx rows kPicks apart) by the virtualdiff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex be716c935..608923b04 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -880,12 +880,14 @@ int main(int argc, char** argv) {std::fprintf(stderr,"store: %.1f%% of the CPU lane's experts were in RAM (%.1f hits, %.1f misses a token); read %.1f MB a token ""(demand and guesses), the CPU lane waited %.1f ms a token for them; %lld prefetched, %lld used, %lld given up "- "unused; slots %lld of %lld, %lld evictions, %lld failed reads\n",+ "unused; slots %lld of %lld, %lld evictions, %lld failed reads; the GPU computed %.1f experts a token from the store "+ "(%lld slots visible)\n",100.0 * h / std::max(1.0, h + m), h / nt, m / nt, (double) (s.store_bytes - s0.store_bytes) / 1e6 / nt,(s.store_wait_ms - s0.store_wait_ms) / nt, (long long) (s.store_prefetched - s0.store_prefetched),(long long) (s.store_pf_used - s0.store_pf_used), (long long) (s.store_pf_wasted - s0.store_pf_wasted),(long long) s.store_slots, (long long) s.store_slots_max, (long long) (s.store_evictions - s0.store_evictions),- (long long) (s.store_fails - s0.store_fails));+ (long long) (s.store_fails - s0.store_fails), (double) (s.store_zc - s0.store_zc) / nt,+ (long long) s.store_zc_visible);}if (s.ahead_picks > s0.ahead_picks) { // lookaheadconst double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 8829c8594..e0b5bdc45 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -659,6 +659,8 @@ struct Layer {uint8_t* ring_bak = nullptr;float *c_snap = nullptr, *c_rows = nullptr;unsigned long long* vram_tab = nullptr; // device [n_exp]+ std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe+ unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads itstd::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blobstd::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)std::vector<uint64_t> region_bytes;@@ -988,6 +990,9 @@ struct Model::Impl {}();// the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)std::unique_ptr<ExpertStore> store;+ uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token+ size_t zc_stride = 0;+ bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselvesbool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM nowif (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;if (l.lay.fmt == gg::kFmtMXFP4) {@@ -1568,7 +1573,7 @@ struct Model::Impl {l.lay.trellis[r] = off;l.lay.suh[r] = l.lay.svh[r] = off;l.lay.K[r] = ty;- off += align256(bytes);+ off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)}l.lay.fmt = gg::kFmtMXFP4;if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||@@ -4202,26 +4207,105 @@ struct Model::Impl {gl.off[r] = l.mx_off[r];gl.stride[r] = l.mx_stride[r];gl.bytes[r] = mx_part_bytes(l, r);+ gl.tr[r] = l.lay.trellis[r];}+ gl.blob = l.blob;}- std::vector<std::string> paths;ExpertStore::Cfg cfg;cfg.budget_bytes = (size_t) (gb * 1e9);cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);cfg.ram_avail = [] { return ram_avail(); };+ cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);+ const bool want_zc = !(std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) == 0) && devs.size() == 1 &&+ L[0].nm == 1;+ if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)+ const int dev0 = devs[0].id;+ cfg.map_chunk = [dev0](void* h, size_t n) -> void* {+ cudaSetDevice(dev0);+ if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {+ (void) cudaGetLastError();+ return nullptr;+ }+ void* d = nullptr;+ if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {+ (void) cudaGetLastError();+ cudaHostUnregister(h);+ return nullptr;+ }+ if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);+ return d;+ };+ cfg.unmap_chunk = [dev0](void* h) {+ cudaSetDevice(dev0);+ cudaHostUnregister(h);+ };+ }bool direct = true;for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;auto st = std::make_unique<ExpertStore>();- if (!st->init(cfg, geo, pk.fds, pk.dfds, pk.pfds)) {+ if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");return;}store = std::move(st);+ for (Layer& l : L)+ if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,(int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");+ if (want_zc) {+ if (!store->gpu_mapped()) {+ std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");+ for (Layer& l : L) l.zc.clear();+ } else {+ zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;+ cudaSetDevice(devs[0].id);+ if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {+ (void) cudaGetLastError();+ zc_stage_buf = nullptr;+ for (Layer& l : L) l.zc.clear();+ } else {+ for (Layer& l : L)+ if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);+ else if (!l.zc.empty()) {+ (void) cudaGetLastError();+ l.zc.clear();+ }+ }+ }+ std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",+ zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);+ }+ }+ // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;+ // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window+ void store_sync() {+ if (!store || !store->gpu_mapped()) return;+ std::vector<ExpertStore::Vis> ev;+ store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });+ if (ev.empty()) return;+ std::vector<char> dirty(L.size(), 0);+ for (const auto& v : ev) {+ Layer& l = L[(size_t) v.layer];+ if (l.zc.empty()) continue;+ l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;+ dirty[(size_t) v.layer] = 1;+ }+ for (Layer& l : L) {+ if (!dirty[(size_t) l.il]) continue;+ if (l.zc_tab == nullptr) continue;+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.zc[(size_t) e]) {+ const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;+ tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;+ if (s < 0) l.zc[(size_t) e] = 0;+ }+ cudaSetDevice(devs[0].id);+ cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);+ }}int cpu_threads() const {@@ -5109,6 +5193,10 @@ struct Model::Impl {}void cache_tick(int T) {+ cache_tick_swaps(T);+ store_sync();+ }+ void cache_tick_swaps(int T) {if (!o.expert_cache || cache_ths.empty()) return;std::vector<Swap> ready;std::vector<RamSwap> rready;@@ -5150,6 +5238,10 @@ struct Model::Impl {l.ram[(size_t) sw.in] = -1;}l.slot[(size_t) sw.in] = l.spare_vram;+ if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)+ l.zc[(size_t) sw.in] = 0;+ store->unmark(l.il, sw.in);+ }if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};l.host[(size_t) sw.in] = nullptr;@@ -5380,6 +5472,11 @@ struct Model::Impl {}if (l.slot[(size_t) e] < 0) {l.cpu_hits.push_back(e);+ if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot+ store->touch(li, e);+ ++st.store_zc;+ continue;+ }if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {++st.skipped;continue;@@ -7682,6 +7779,8 @@ struct Model::Impl {ma.n_ff = FF;ma.limit = g.lim;const bool mx = lay.fmt == gg::kFmtMXFP4;+ if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)+ zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);else x3::moe_gate_up(ma, d.x, d.xg, s);prof_mark(d, 3);@@ -8020,6 +8119,7 @@ const Stats& Model::stats() const {st.store_fails = x.stats().fails.load();st.store_wait_ms = x.stats().wait_ns.load() / 1e6;st.store_busy_ms = x.busy_ms();+ st.store_zc_visible = (int64_t) x.visible_count();st.store_slots = (int64_t) x.slots_allocated();st.store_slots_max = (int64_t) x.slots_max();}diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex 41b87a847..df203f123 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -82,6 +82,7 @@ struct Stats {// of use (prefetched) of which used / given up unused, slots given up, bytes read, ms the CPU lane waited for the readsint64_t store_hits = 0, store_misses = 0, store_prefetched = 0, store_pf_used = 0, store_pf_wasted = 0, store_evictions = 0;int64_t store_bytes = 0, store_fails = 0;+ int64_t store_zc = 0, store_zc_visible = 0; // experts the GPU computed from the store (zero-copy over PCIe); slots it may readdouble store_wait_ms = 0.0, store_busy_ms = 0.0;int64_t store_slots = 0, store_slots_max = 0;int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex d93d9632b..e37e44223 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -375,6 +375,32 @@ inline void* anon_alloc(size_t bytes, int node, int nodes) {return m;#endif}+/// An address range of `bytes` reserved (no memory behind it yet), and pieces of it made real: the store's arena is one range so a+/// kernel can tell its pointers by [lo, hi). nullptr: it could not be had.+inline void* range_reserve(size_t bytes) {+#if defined(_WIN32)+ return VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_READWRITE);+#else+ void* m = ::mmap(nullptr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);+ return m == MAP_FAILED ? nullptr : m;+#endif+}+inline bool range_commit(void* p, size_t bytes) {+#if defined(_WIN32)+ return VirtualAlloc(p, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr;+#else+ return ::mprotect(p, bytes, PROT_READ | PROT_WRITE) == 0;+#endif+}+inline void range_release(void* p, size_t bytes) {+ if (p == nullptr) return;+#if defined(_WIN32)+ (void) bytes;+ VirtualFree(p, 0, MEM_RELEASE);+#else+ ::munmap(p, bytes);+#endif+}inline void anon_free(void* p, size_t bytes) {if (p == nullptr) return;#if defined(_WIN32)diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hppindex 167ecde63..6a6a2dd4d 100644--- a/src/dsv41/ds4_pool.hpp+++ b/src/dsv41/ds4_pool.hpp@@ -199,7 +199,7 @@ private:ok = true;buffered_.fetch_add(j.len, std::memory_order_relaxed);}- if (ok && j.keep != nullptr && j.mem == nullptr) std::memcpy(j.keep, j.dst + lead, (size_t) j.len);+ if (ok && j.keep != nullptr && j.mem == nullptr) std::memmove(j.keep, j.dst + lead, (size_t) j.len); // (keep may overlap dst: the store reads in place and moves up)busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - t0).count(),std::memory_order_relaxed);diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppindex 1a407ba83..3c7b923a7 100644--- a/src/dsv41/ds4_store.hpp+++ b/src/dsv41/ds4_store.hpp@@ -2,16 +2,18 @@// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).//-// An expert (layer, e) is three matrices at three file offsets (the GGUF's one tensor a role and layer). A slot holds their-// bytes at 4096-aligned places (unbuffered reads land the file range rounded out to 4096 at an aligned address; the matrix's-// first byte is `lead` bytes into it), so mat(slot, r) is the pointer the CPU lane's rows read - the same bytes as the file's-// mapping, so every answer is the same. Nothing here touches a device.+// A slot holds an expert in the engine's blob layout (gate | up | down at the layer's offsets tr[r], the same as a VRAM slot), so+// the CPU lane's rows read it in place and - when the arena is registered with the GPU - the device kernels read it over PCIe+// (zero-copy, 24 GB/s on a 5070) through the same plan pointers as a VRAM expert. An expert is three matrices at three file+// offsets (the GGUF's one tensor a role and layer): each is read aligned (4096) into the slot and moved into place (the read's+// lead of up to 4095 bytes), the blob layout leaving 8192 bytes between matrices (and around the blob) for the aligned spans.//-// Memory: slots are allocated in chunks as they are first needed (VirtualAlloc / mmap, private memory, not pinned), never past-// `max_slots` nor past the RAM this process can still get less `floor_bytes` (the owner's session and the system keep theirs).+// Memory: slots are allocated in chunks as they are first needed (private memory), never past the budget nor past the RAM this+// process can still get less `floor_bytes` (the owner's session and the system keep theirs).//-// Policy (pick the slot to give up, among the unpinned ready ones): lru (oldest touch), lfu (lowest decayed hit count), or-// lfu with an LRU tie-break. A slot whose expert a window is computing is pinned from ensure() to release().+// Policy (the slot to give up, among the unpinned, not GPU-visible ones): lru (oldest touch), lfu (lowest decayed hit count).+// A slot whose expert a window is computing is pinned from ensure() to release(); a slot the GPU may read is `visible` until+// sync() takes it back at a window boundary (the GPU idle), which also publishes the newly ready ones.#pragma once#include "ds4_os.hpp"@@ -40,15 +42,20 @@ public:int file[3] = {-1, -1, -1};uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0}; // matrix r of expert e: file[r], off[r] + e stride[r]uint64_t bytes[3] = {0, 0, 0};+ uint64_t tr[3] = {0, 0, 0}; // matrix r's place in the blob+ uint64_t blob = 0; // the blob's bytes (gaps of 8192 between matrices)};- enum Policy { kLru = 0, kLfu = 1, kLfuLru = 2 };+ enum Policy { kLru = 0, kLfu = 1 };struct Stats {std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};- std::atomic<int64_t> bytes_read{0}, wait_ns{0};+ std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0};+ };+ struct Vis { // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)+ int layer = 0, e = 0;+ unsigned long long dev = 0;};struct Cfg {- const std::vector<std::string>* paths = nullptr; // the files, by the layers' file indicesint threads = 8;size_t budget_bytes = 0; // the arena at most (grown in chunks as slots are first needed)size_t floor_bytes = 8ull << 30; // RAM left alone@@ -56,24 +63,26 @@ public:float decay = 0.97f; // lfu: heat x decay every `decay_every` accesses of its layerint decay_every = 240;std::function<uint64_t()> ram_avail; // the RAM this process can still get (~0: unknown)+ // a new chunk of the arena for the GPU to read in place: returns the device address of `host` (null: not visible)+ std::function<void*(void* host, size_t bytes)> map_chunk;+ std::function<void(void* host)> unmap_chunk;+ size_t reserve = 192; // slots kept evictable (not visible) for the window's demand and guesses};- bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& fds, const std::vector<os::fd_t>& dfds,- const std::vector<os::fd_t>& pfds) {+ bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {cfg_ = c;geo_ = layers;- fds_ = fds;dfds_ = dfds;pfds_ = pfds;- slot_bytes_ = 0;- for (const LayerGeo& g : geo_) {- size_t need = 0;- for (int r = 0; r < 3; ++r) need += (size_t) g.bytes[r] + 2 * os::kDirectAlign;- slot_bytes_ = std::max(slot_bytes_, need);- }- slot_bytes_ = (slot_bytes_ + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);- if (slot_bytes_ == 0 || c.budget_bytes < slot_bytes_) return false;+ size_t blob = 0;+ for (const LayerGeo& g : geo_) blob = std::max<size_t>(blob, g.blob);+ slot_bytes_ = (blob + 2 * kSlack + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);+ if (blob == 0 || c.budget_bytes < slot_bytes_) return false;nmax_ = c.budget_bytes / slot_bytes_;+ // one reserved range for the whole arena (chunks committed as they are needed), 64 KB aligned chunks+ arena_bytes_ = ((nmax_ + kChunk - 1) / kChunk) * kChunk * slot_bytes_;+ arena_ = (uint8_t*) os::range_reserve(arena_bytes_);+ if (arena_ == nullptr) return false;slots_.reset(new Slot[nmax_]);where_.resize(geo_.size());heat_.resize(geo_.size());@@ -83,11 +92,14 @@ public:heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);}io_.reset(new IoQueue(std::max(1, c.threads)));+ grow(); // (the first chunk now: whether the GPU can map it is known from the start)return true;}~ExpertStore() {io_.reset();- for (auto& ch : chunks_) os::anon_free(ch.first, ch.second);+ for (size_t i = 0; i < chunks_.size(); ++i)+ if (cfg_.unmap_chunk && chunk_dev_[i] != nullptr) cfg_.unmap_chunk(chunks_[i].first);+ os::range_release(arena_, arena_bytes_);}size_t slot_bytes() const { return slot_bytes_; }@@ -95,8 +107,9 @@ public:size_t slots_max() const { return nmax_; }const Stats& stats() const { return st_; }double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }+ static constexpr size_t kSlack = 8192;- /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned.)+ /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned or visible.)int find(int layer, int e) const {if (layer < 0 || layer >= (int) where_.size()) return -1; // (a layer the store does not cover: the DSpark drafter's)const int s = where_[(size_t) layer][(size_t) e];@@ -105,11 +118,18 @@ public:try_ready(sp);return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;}- const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + slots_[(size_t) slot].lead[r]; }+ const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + geo_[(size_t) slots_[(size_t) slot].layer].tr[r]; }+ const uint8_t* blob(int slot) const { return slots_[(size_t) slot].base; }+ /// The device address of the slot's blob (0: the arena is not mapped for the GPU)+ unsigned long long dev_blob(int slot) const {+ const size_t c = (size_t) slot / kChunk;+ if (c >= chunk_dev_.size() || chunk_dev_[c] == nullptr) return 0;+ return (unsigned long long) ((uint8_t*) chunk_dev_[c] + (slots_[(size_t) slot].base - (uint8_t*) chunks_[c].first));+ }/// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the- /// caller releases them. False if a read failed or no slot could be had for one (then none of the failed ones is- /// pinned: ok[i] says which are; the caller reads those another way).+ /// caller releases them. False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller+ /// reads the others another way).bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {ok.assign((size_t) n, 0);if (layer < 0 || layer >= (int) geo_.size()) return false;@@ -176,33 +196,110 @@ public:if (where_[(size_t) layer][(size_t) e] >= 0) return;if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);}- /// Heat of an expert as the policy sees it (tests / the report).- float heat(int layer, int e) const { return heat_[(size_t) layer][(size_t) e]; }+ /// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).+ void touch(int layer, int e) {+ if (layer < 0 || layer >= (int) where_.size()) return;+ std::lock_guard<std::mutex> lk(mu_);+ touch_heat(layer, e);+ const int s = where_[(size_t) layer][(size_t) e];+ if (s >= 0) slots_[(size_t) s].tick = ++clock_;+ }+ bool gpu_mapped() const { return !chunk_dev_.empty() && chunk_dev_[0] != nullptr; }+ /// The arena's address range (the device sees its host pointers as they are when the mapping is the identity)+ unsigned long long arena_lo() const { return (unsigned long long) (uintptr_t) arena_; }+ unsigned long long arena_hi() const { return (unsigned long long) (uintptr_t) (arena_ + arena_bytes_); }+ bool identity_mapped() const {+ for (size_t i = 0; i < chunks_.size(); ++i)+ if (chunk_dev_[i] != nullptr && chunk_dev_[i] != chunks_[i].first) return false;+ return true;+ }+ size_t max_blob() const { return slot_bytes_ - 2 * kSlack; }++ /// At a window boundary (nothing reading the slots; the caller applies `ev` to the device's table before the next window):+ /// every ready slot not visible yet becomes visible (`gpu_ok(layer, e)`: false for an expert VRAM holds), and visible+ /// slots of the lowest score are taken back until `cfg.reserve` slots can be given up. Returns the number of events.+ size_t sync(std::vector<Vis>& ev, const std::function<bool(int, int)>& gpu_ok) {+ std::lock_guard<std::mutex> lk(mu_);+ size_t nvis = 0, nfree = 0;+ for (size_t i = 0; i < nalloc_; ++i) {+ Slot& sp = slots_[i];+ if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {+ try_ready(sp);+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);+ }+ const int st = sp.state.load(std::memory_order_acquire);+ if (st == kFree) {+ ++nfree;+ continue;+ }+ if (st != kReady) continue;+ if (sp.visible) {+ ++nvis;+ continue;+ }+ if (sp.pin == 0) ++nfree;+ if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch) {+ sp.visible = true;+ ++nvis;+ if (sp.pin == 0) --nfree;+ ev.push_back({sp.layer, sp.expert, dev_blob((int) i)});+ }+ }+ // give slots back until the reserve is there: the visible ones with the lowest score+ if (nfree < cfg_.reserve && nvis > 0) {+ std::vector<std::pair<double, size_t>> cand;+ cand.reserve(nvis);+ for (size_t i = 0; i < nalloc_; ++i)+ if (slots_[i].visible && slots_[i].state.load(std::memory_order_acquire) == kReady) cand.push_back({score(slots_[i]), i});+ const size_t want = std::min(cand.size(), cfg_.reserve - nfree);+ std::partial_sort(cand.begin(), cand.begin() + (long) want, cand.end());+ for (size_t k = 0; k < want; ++k) {+ Slot& sp = slots_[cand[k].second];+ sp.visible = false;+ ev.push_back({sp.layer, sp.expert, 0ull});+ }+ }+ return ev.size();+ }+ /// An expert VRAM took over: no longer a visible slot+ void unmark(int layer, int e) {+ if (layer < 0 || layer >= (int) where_.size()) return;+ std::lock_guard<std::mutex> lk(mu_);+ const int s = where_[(size_t) layer][(size_t) e];+ if (s >= 0) slots_[(size_t) s].visible = false;+ }+ size_t visible_count() const {+ size_t n = 0;+ for (size_t i = 0; i < nalloc_; ++i) n += slots_[i].visible ? 1 : 0;+ return n;+ }private:static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;+ static constexpr size_t kChunk = 64; // slots a chunk of the arenastruct Slot {- uint8_t* base = nullptr;- uint32_t lead[3] = {0, 0, 0};+ uint8_t* base = nullptr; // the blob's first byteint layer = -1, expert = -1, pin = 0;- bool prefetch = false;+ bool prefetch = false, visible = false;std::atomic<int> state{kFree}, left{0};std::atomic<bool> bad{false};uint64_t tick = 0;};+ double score(const Slot& sp) const {+ if (cfg_.policy == kLru) return (double) sp.tick;+ return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;+ }void touch_heat(int layer, int e) {heat_[(size_t) layer][(size_t) e] += 1.0f;- if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0) {- // (the layer's heat x decay: a linear pass over 384 floats every 240 accesses of that layer)- for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;- }+ if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0)+ for (float& h : heat_[(size_t) layer]) h *= cfg_.decay; // (a linear pass over 384 floats every 240 accesses of that layer)}// a slot for (layer, e) and its reads started; -1 if there is none to be had. mu_ held.int start_fill(int layer, int e, bool demand) {int s = take_slot(demand);if (s < 0) {- st_.fails.fetch_add(1, std::memory_order_relaxed);+ st_.no_slot.fetch_add(1, std::memory_order_relaxed);return -1;}Slot& sp = slots_[(size_t) s];@@ -215,6 +312,7 @@ private:sp.expert = e;sp.pin = demand ? 1 : 0;sp.prefetch = !demand;+ sp.visible = false;sp.tick = ++clock_;sp.state.store(kLoading, std::memory_order_release);sp.bad.store(false);@@ -222,20 +320,22 @@ private:where_[(size_t) layer][(size_t) e] = s;const LayerGeo& g = geo_[(size_t) layer];IoQueue::Job jobs[3];- size_t at = 0;for (int r = 0; r < 3; ++r) {const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];const uint64_t lead = off % os::kDirectAlign;- sp.lead[r] = (uint32_t) (at + lead);+ uint8_t* fin = sp.base + g.tr[r]; // where the matrix belongs+ // the aligned read lands the file's byte `off` at dst + lead: dst is fin - lead rounded down to 4096 (its span spills+ // under 4096 bytes either way into the blob's gaps), the matrix is then moved up to fin+ uint8_t* dst = (uint8_t*) ((uintptr_t) (fin - lead) & ~(uintptr_t) (os::kDirectAlign - 1));IoQueue::Job& j = jobs[r];j.fd = pfds_[(size_t) g.file[r]];j.dfd = dfds_[(size_t) g.file[r]];j.off = off;j.len = g.bytes[r];- j.dst = sp.base + at;+ j.dst = dst;+ j.keep = fin;j.left = &sp.left;j.bad = &sp.bad;- at += os::direct_span(off, g.bytes[r]);}st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);io_->push(jobs, 3);@@ -261,19 +361,20 @@ private:}bool grow() {if (nalloc_ >= nmax_) return false;- constexpr size_t kChunk = 64; // slotsconst size_t n = std::min(kChunk, nmax_ - nalloc_);const size_t bytes = n * slot_bytes_;const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;- void* p = os::anon_alloc(bytes, -1, 1);- if (p == nullptr) return false;+ void* p = arena_ + nalloc_ * slot_bytes_;+ if (!os::range_commit(p, bytes)) return false;chunks_.push_back({p, bytes});- for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_;+ void* dev = cfg_.map_chunk ? cfg_.map_chunk(p, bytes) : nullptr;+ chunk_dev_.push_back(dev);+ for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_ + kSlack;nalloc_ += n;return true;}- // the slot to give up (or a free one): the policy's lowest among the unpinned ready ones. mu_ held.+ // the slot to give up (or a free one): the policy's lowest among the unpinned, not visible, ready ones. mu_ held.int take_slot(bool demand) {for (size_t i = free_hint_; i < nalloc_; ++i)if (slots_[i].state.load(std::memory_order_acquire) == kFree) {@@ -287,20 +388,14 @@ private:Slot& sp = slots_[i];if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {try_ready(sp);- if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp); // (failed: freed)+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);if (sp.state.load(std::memory_order_acquire) == kFree) {free_hint_ = std::min(free_hint_, i);return (int) i;}}- if (sp.pin > 0 || sp.state.load(std::memory_order_acquire) != kReady) continue;- double v;- if (cfg_.policy == kLru) v = (double) sp.tick;- else {- const double h = heat_[(size_t) sp.layer][(size_t) sp.expert];- v = cfg_.policy == kLfu ? h : h * 1e12 + (double) sp.tick; // lfu with the age as tie-break- if (cfg_.policy == kLfuLru) v = h + 1e-9 * (double) sp.tick;- }+ if (sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) != kReady) continue;+ const double v = score(sp);if (best < 0 || v < bv) {best = (int) i;bv = v;@@ -313,10 +408,12 @@ private:Cfg cfg_;std::vector<LayerGeo> geo_;- std::vector<os::fd_t> fds_, dfds_, pfds_;- size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0;+ std::vector<os::fd_t> dfds_, pfds_;+ size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;+ uint8_t* arena_ = nullptr;std::unique_ptr<Slot[]> slots_;std::vector<std::pair<void*, size_t>> chunks_;+ std::vector<void*> chunk_dev_;std::vector<std::vector<int>> where_;std::vector<std::vector<float>> heat_;std::vector<uint64_t> hits_in_layer_;diff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cuindex 0d03d3bf3..d351dade4 100644--- a/tools/dsv41/win/hwbench.cu+++ b/tools/dsv41/win/hwbench.cu@@ -95,10 +95,49 @@ static void bench_nvme(const char* path, size_t block, int qd, double secs, boolCloseHandle(f);}+// GPU kernels reading host memory over PCIe in place (zero-copy): the pinned + mapped (or registered) arena as a weights source+__global__ void zc_sum(const int4* __restrict__ p, size_t n, unsigned long long* out) {+ unsigned long long acc = 0;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (size_t) gridDim.x * blockDim.x) {+ const int4 v = p[i];+ acc += (unsigned) v.x ^ (unsigned) v.y ^ (unsigned) v.z ^ (unsigned) v.w;+ }+ if (acc == 0x123456789ull) atomicAdd(out, acc);+}+static void bench_zc(size_t mib) {+ const size_t n = mib << 20;+ for (int mode = 0; mode < 2; ++mode) {+ void *h = nullptr, *dp = nullptr;+ if (mode == 0) {+ if (cudaHostAlloc(&h, n, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) { printf("hostalloc failed\n"); continue; }+ } else {+ h = VirtualAlloc(nullptr, n, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);+ memset(h, 1, n);+ if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) { printf("hostregister failed\n"); continue; }+ }+ memset(h, 1, n);+ cudaHostGetDevicePointer(&dp, h, 0);+ unsigned long long* out; cudaMalloc(&out, 8);+ for (int blocks : {64, 256, 1024}) {+ zc_sum<<<blocks, 256>>>((const int4*) dp, n / 16, out);+ cudaDeviceSynchronize();+ cudaEvent_t a, b; cudaEventCreate(&a); cudaEventCreate(&b);+ cudaEventRecord(a);+ for (int r = 0; r < 4; ++r) zc_sum<<<blocks, 256>>>((const int4*) dp, n / 16, out);+ cudaEventRecord(b); cudaEventSynchronize(b);+ float ms; cudaEventElapsedTime(&ms, a, b);+ printf("zero-copy %s %zu MiB, %d blocks: %.2f GB/s\n", mode == 0 ? "cudaHostAlloc(mapped)" : "VirtualAlloc+cudaHostRegister(mapped)", mib, blocks, 4.0 * n / 1e9 / (ms / 1e3));+ }+ if (mode == 0) cudaFreeHost(h); else { cudaHostUnregister(h); VirtualFree(h, 0, MEM_RELEASE); }+ cudaFree(out);+ }+}+int main(int argc, char** argv) {std::string m = argc > 1 ? argv[1] : "";if (m == "ram") bench_ram(argc > 2 ? atoi(argv[2]) : 6, argc > 3 ? atoi(argv[3]) : 512);else if (m == "pcie") bench_pcie();+ else if (m == "zc") bench_zc(argc > 2 ? atoi(argv[2]) : 1024);else if (m == "nvme" && argc > 5) bench_nvme(argv[2], (size_t)atoi(argv[3]) << 10, atoi(argv[4]), atof(argv[5]), argc > 6 && std::string(argv[6]) == "seq");else printf("usage: hwbench ram|pcie|nvme ...\n");}