Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: the store's slots in blob layout, GPU-visible (zero-copy mapped) with a PCIe staging copy for decode picks, node-free reserved arena

Niko12217d2307deb0a0
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index 9f9a1d484..10ab1edab 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1831,6 +1831,33 @@ void route_ahead(const float* logits, const float* bias, int n, int m, int T, Re
route_ahead_kernel<<<T, 512, 0, s>>>(logits, bias, n, m, req, off);
}
void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { wait_seq_kernel<<<1, 32, 0, s>>>(seq_ptr, seq); }
+
+// the RAM store's experts for the device (a decode token, window of one): a pick the VRAM table left at 0 whose expert the store holds
+// (zc_tab[expert] = the slot's device address over PCIe, 0: not) is copied to staging slot i with wide coalesced loads (the PCIe
+// rate; the expert kernels' own small loads over PCIe ran at a tenth of it), then the plan points at the copy
+__global__ void zc_copy_kernel(const unsigned long long* __restrict__ plan, const int* __restrict__ idx,
+ const unsigned long long* __restrict__ zc_tab, uint8_t* __restrict__ stage, size_t stride, size_t bytes) {
+ const int i = blockIdx.y;
+ if (plan[i] != 0) return;
+ const unsigned long long p = zc_tab[idx[i]];
+ if (p == 0) return;
+ const uint4* src = (const uint4*) p;
+ uint4* dst = (uint4*) (stage + (size_t) i * stride);
+ const size_t n = bytes / 16;
+ for (size_t o = (size_t) blockIdx.x * blockDim.x + threadIdx.x; o < n; o += (size_t) gridDim.x * blockDim.x) dst[o] = src[o];
+}
+__global__ void zc_fix_kernel(unsigned long long* __restrict__ plan, const int* __restrict__ idx, const unsigned long long* __restrict__ zc_tab,
+ int k, uint8_t* __restrict__ stage, size_t stride) {
+ const int i = threadIdx.x;
+ if (i >= k || plan[i] != 0) return;
+ if (zc_tab[idx[i]] != 0) plan[i] = (unsigned long long) (stage + (size_t) i * stride);
+}
+void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,
+ cudaStream_t s) {
+ zc_copy_kernel<<<dim3(64, k), 256, 0, s>>>(plan, idx, zc_tab, stage, stride, bytes);
+ zc_fix_kernel<<<1, 32, 0, s>>>(plan, idx, zc_tab, k, stage, stride);
+}
+
void init_pre(float* pre, int T, cudaStream_t s) { init_pre_kernel<<<cdiv(T * 8, 128), 128, 0, s>>>(pre, T); }
void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
index 94a13997e..829255f1d 100644
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@ -232,6 +232,10 @@ void route_publish(const float* logits, const float* bias, int n, int k, int T,
/// Lookahead: the m picks a later layer will likely make (its gate's logits on this layer's FFN input, the routing's
/// score + bias, the m best of n) into req->ahead[t][off ..) for the T tokens - before route_publish, which publishes them.
void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s);
+/// The RAM store's picks for the device (a decode token's k plan pointers): those the VRAM table left at 0 whose expert zc_tab has (the
+/// slot's device address) are copied into stage slot i (stride apart, `bytes` each) and the plan then points there.
+void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,
+ cudaStream_t s);
/// Waits (one thread, on the device) until *seq_ptr == seq.
void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s);
/// A chunk's routed experts as sorted rows: the picks (t, i < k) of T tokens (idx rows kPicks apart) by the virtual
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index be716c935..608923b04 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -880,12 +880,14 @@ int main(int argc, char** argv) {
std::fprintf(stderr,
"store: %.1f%% of the CPU lane's experts were in RAM (%.1f hits, %.1f misses a token); read %.1f MB a token "
"(demand and guesses), the CPU lane waited %.1f ms a token for them; %lld prefetched, %lld used, %lld given up "
- "unused; slots %lld of %lld, %lld evictions, %lld failed reads\n",
+ "unused; slots %lld of %lld, %lld evictions, %lld failed reads; the GPU computed %.1f experts a token from the store "
+ "(%lld slots visible)\n",
100.0 * h / std::max(1.0, h + m), h / nt, m / nt, (double) (s.store_bytes - s0.store_bytes) / 1e6 / nt,
(s.store_wait_ms - s0.store_wait_ms) / nt, (long long) (s.store_prefetched - s0.store_prefetched),
(long long) (s.store_pf_used - s0.store_pf_used), (long long) (s.store_pf_wasted - s0.store_pf_wasted),
(long long) s.store_slots, (long long) s.store_slots_max, (long long) (s.store_evictions - s0.store_evictions),
- (long long) (s.store_fails - s0.store_fails));
+ (long long) (s.store_fails - s0.store_fails), (double) (s.store_zc - s0.store_zc) / nt,
+ (long long) s.store_zc_visible);
}
if (s.ahead_picks > s0.ahead_picks) { // lookahead
const double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 8829c8594..e0b5bdc45 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -659,6 +659,8 @@ struct Layer {
uint8_t* ring_bak = nullptr;
float *c_snap = nullptr, *c_rows = nullptr;
unsigned long long* vram_tab = nullptr; // device [n_exp]
+ std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe
+ unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it
std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
std::vector<uint64_t> region_bytes;
@@ -988,6 +990,9 @@ struct Model::Impl {
}();
// the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
std::unique_ptr<ExpertStore> store;
+ uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token
+ size_t zc_stride = 0;
+ bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves
bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
if (l.lay.fmt == gg::kFmtMXFP4) {
@@ -1568,7 +1573,7 @@ struct Model::Impl {
l.lay.trellis[r] = off;
l.lay.suh[r] = l.lay.svh[r] = off;
l.lay.K[r] = ty;
- off += align256(bytes);
+ off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)
}
l.lay.fmt = gg::kFmtMXFP4;
if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
@@ -4202,26 +4207,105 @@ struct Model::Impl {
gl.off[r] = l.mx_off[r];
gl.stride[r] = l.mx_stride[r];
gl.bytes[r] = mx_part_bytes(l, r);
+ gl.tr[r] = l.lay.trellis[r];
}
+ gl.blob = l.blob;
}
- std::vector<std::string> paths;
ExpertStore::Cfg cfg;
cfg.budget_bytes = (size_t) (gb * 1e9);
cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
cfg.ram_avail = [] { return ram_avail(); };
+ cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
+ const bool want_zc = !(std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) == 0) && devs.size() == 1 &&
+ L[0].nm == 1;
+ if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)
+ const int dev0 = devs[0].id;
+ cfg.map_chunk = [dev0](void* h, size_t n) -> void* {
+ cudaSetDevice(dev0);
+ if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return nullptr;
+ }
+ void* d = nullptr;
+ if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {
+ (void) cudaGetLastError();
+ cudaHostUnregister(h);
+ return nullptr;
+ }
+ if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);
+ return d;
+ };
+ cfg.unmap_chunk = [dev0](void* h) {
+ cudaSetDevice(dev0);
+ cudaHostUnregister(h);
+ };
+ }
bool direct = true;
for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
auto st = std::make_unique<ExpertStore>();
- if (!st->init(cfg, geo, pk.fds, pk.dfds, pk.pfds)) {
+ if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {
std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
return;
}
store = std::move(st);
+ for (Layer& l : L)
+ if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);
std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
(int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
+ if (want_zc) {
+ if (!store->gpu_mapped()) {
+ std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;
+ cudaSetDevice(devs[0].id);
+ if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {
+ (void) cudaGetLastError();
+ zc_stage_buf = nullptr;
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ for (Layer& l : L)
+ if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);
+ else if (!l.zc.empty()) {
+ (void) cudaGetLastError();
+ l.zc.clear();
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",
+ zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
+ }
+ }
+ // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
+ // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
+ void store_sync() {
+ if (!store || !store->gpu_mapped()) return;
+ std::vector<ExpertStore::Vis> ev;
+ store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });
+ if (ev.empty()) return;
+ std::vector<char> dirty(L.size(), 0);
+ for (const auto& v : ev) {
+ Layer& l = L[(size_t) v.layer];
+ if (l.zc.empty()) continue;
+ l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;
+ dirty[(size_t) v.layer] = 1;
+ }
+ for (Layer& l : L) {
+ if (!dirty[(size_t) l.il]) continue;
+ if (l.zc_tab == nullptr) continue;
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.zc[(size_t) e]) {
+ const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;
+ tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;
+ if (s < 0) l.zc[(size_t) e] = 0;
+ }
+ cudaSetDevice(devs[0].id);
+ cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);
+ }
}
int cpu_threads() const {
@@ -5109,6 +5193,10 @@ struct Model::Impl {
}
void cache_tick(int T) {
+ cache_tick_swaps(T);
+ store_sync();
+ }
+ void cache_tick_swaps(int T) {
if (!o.expert_cache || cache_ths.empty()) return;
std::vector<Swap> ready;
std::vector<RamSwap> rready;
@@ -5150,6 +5238,10 @@ struct Model::Impl {
l.ram[(size_t) sw.in] = -1;
}
l.slot[(size_t) sw.in] = l.spare_vram;
+ if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)
+ l.zc[(size_t) sw.in] = 0;
+ store->unmark(l.il, sw.in);
+ }
if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
l.host[(size_t) sw.in] = nullptr;
@@ -5380,6 +5472,11 @@ struct Model::Impl {
}
if (l.slot[(size_t) e] < 0) {
l.cpu_hits.push_back(e);
+ if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot
+ store->touch(li, e);
+ ++st.store_zc;
+ continue;
+ }
if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
++st.skipped;
continue;
@@ -7682,6 +7779,8 @@ struct Model::Impl {
ma.n_ff = FF;
ma.limit = g.lim;
const bool mx = lay.fmt == gg::kFmtMXFP4;
+ if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)
+ zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);
if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
else x3::moe_gate_up(ma, d.x, d.xg, s);
prof_mark(d, 3);
@@ -8020,6 +8119,7 @@ const Stats& Model::stats() const {
st.store_fails = x.stats().fails.load();
st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
st.store_busy_ms = x.busy_ms();
+ st.store_zc_visible = (int64_t) x.visible_count();
st.store_slots = (int64_t) x.slots_allocated();
st.store_slots_max = (int64_t) x.slots_max();
}
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index 41b87a847..df203f123 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -82,6 +82,7 @@ struct Stats {
// of use (prefetched) of which used / given up unused, slots given up, bytes read, ms the CPU lane waited for the reads
int64_t store_hits = 0, store_misses = 0, store_prefetched = 0, store_pf_used = 0, store_pf_wasted = 0, store_evictions = 0;
int64_t store_bytes = 0, store_fails = 0;
+ int64_t store_zc = 0, store_zc_visible = 0; // experts the GPU computed from the store (zero-copy over PCIe); slots it may read
double store_wait_ms = 0.0, store_busy_ms = 0.0;
int64_t store_slots = 0, store_slots_max = 0;
int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index d93d9632b..e37e44223 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -375,6 +375,32 @@ inline void* anon_alloc(size_t bytes, int node, int nodes) {
return m;
#endif
}
+/// An address range of `bytes` reserved (no memory behind it yet), and pieces of it made real: the store's arena is one range so a
+/// kernel can tell its pointers by [lo, hi). nullptr: it could not be had.
+inline void* range_reserve(size_t bytes) {
+#if defined(_WIN32)
+ return VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_READWRITE);
+#else
+ void* m = ::mmap(nullptr, bytes, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+ return m == MAP_FAILED ? nullptr : m;
+#endif
+}
+inline bool range_commit(void* p, size_t bytes) {
+#if defined(_WIN32)
+ return VirtualAlloc(p, bytes, MEM_COMMIT, PAGE_READWRITE) != nullptr;
+#else
+ return ::mprotect(p, bytes, PROT_READ | PROT_WRITE) == 0;
+#endif
+}
+inline void range_release(void* p, size_t bytes) {
+ if (p == nullptr) return;
+#if defined(_WIN32)
+ (void) bytes;
+ VirtualFree(p, 0, MEM_RELEASE);
+#else
+ ::munmap(p, bytes);
+#endif
+}
inline void anon_free(void* p, size_t bytes) {
if (p == nullptr) return;
#if defined(_WIN32)
diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hpp
index 167ecde63..6a6a2dd4d 100644
--- a/src/dsv41/ds4_pool.hpp
+++ b/src/dsv41/ds4_pool.hpp
@@ -199,7 +199,7 @@ private:
ok = true;
buffered_.fetch_add(j.len, std::memory_order_relaxed);
}
- if (ok && j.keep != nullptr && j.mem == nullptr) std::memcpy(j.keep, j.dst + lead, (size_t) j.len);
+ if (ok && j.keep != nullptr && j.mem == nullptr) std::memmove(j.keep, j.dst + lead, (size_t) j.len); // (keep may overlap dst: the store reads in place and moves up)
busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - t0).count(),
std::memory_order_relaxed);
diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hpp
index 1a407ba83..3c7b923a7 100644
--- a/src/dsv41/ds4_store.hpp
+++ b/src/dsv41/ds4_store.hpp
@@ -2,16 +2,18 @@
// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots
// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).
//
-// An expert (layer, e) is three matrices at three file offsets (the GGUF's one tensor a role and layer). A slot holds their
-// bytes at 4096-aligned places (unbuffered reads land the file range rounded out to 4096 at an aligned address; the matrix's
-// first byte is `lead` bytes into it), so mat(slot, r) is the pointer the CPU lane's rows read - the same bytes as the file's
-// mapping, so every answer is the same. Nothing here touches a device.
+// A slot holds an expert in the engine's blob layout (gate | up | down at the layer's offsets tr[r], the same as a VRAM slot), so
+// the CPU lane's rows read it in place and - when the arena is registered with the GPU - the device kernels read it over PCIe
+// (zero-copy, 24 GB/s on a 5070) through the same plan pointers as a VRAM expert. An expert is three matrices at three file
+// offsets (the GGUF's one tensor a role and layer): each is read aligned (4096) into the slot and moved into place (the read's
+// lead of up to 4095 bytes), the blob layout leaving 8192 bytes between matrices (and around the blob) for the aligned spans.
//
-// Memory: slots are allocated in chunks as they are first needed (VirtualAlloc / mmap, private memory, not pinned), never past
-// `max_slots` nor past the RAM this process can still get less `floor_bytes` (the owner's session and the system keep theirs).
+// Memory: slots are allocated in chunks as they are first needed (private memory), never past the budget nor past the RAM this
+// process can still get less `floor_bytes` (the owner's session and the system keep theirs).
//
-// Policy (pick the slot to give up, among the unpinned ready ones): lru (oldest touch), lfu (lowest decayed hit count), or
-// lfu with an LRU tie-break. A slot whose expert a window is computing is pinned from ensure() to release().
+// Policy (the slot to give up, among the unpinned, not GPU-visible ones): lru (oldest touch), lfu (lowest decayed hit count).
+// A slot whose expert a window is computing is pinned from ensure() to release(); a slot the GPU may read is `visible` until
+// sync() takes it back at a window boundary (the GPU idle), which also publishes the newly ready ones.
#pragma once
#include "ds4_os.hpp"
@@ -40,15 +42,20 @@ public:
int file[3] = {-1, -1, -1};
uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0}; // matrix r of expert e: file[r], off[r] + e stride[r]
uint64_t bytes[3] = {0, 0, 0};
+ uint64_t tr[3] = {0, 0, 0}; // matrix r's place in the blob
+ uint64_t blob = 0; // the blob's bytes (gaps of 8192 between matrices)
};
- enum Policy { kLru = 0, kLfu = 1, kLfuLru = 2 };
+ enum Policy { kLru = 0, kLfu = 1 };
struct Stats {
std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};
- std::atomic<int64_t> bytes_read{0}, wait_ns{0};
+ std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0};
+ };
+ struct Vis { // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)
+ int layer = 0, e = 0;
+ unsigned long long dev = 0;
};
struct Cfg {
- const std::vector<std::string>* paths = nullptr; // the files, by the layers' file indices
int threads = 8;
size_t budget_bytes = 0; // the arena at most (grown in chunks as slots are first needed)
size_t floor_bytes = 8ull << 30; // RAM left alone
@@ -56,24 +63,26 @@ public:
float decay = 0.97f; // lfu: heat x decay every `decay_every` accesses of its layer
int decay_every = 240;
std::function<uint64_t()> ram_avail; // the RAM this process can still get (~0: unknown)
+ // a new chunk of the arena for the GPU to read in place: returns the device address of `host` (null: not visible)
+ std::function<void*(void* host, size_t bytes)> map_chunk;
+ std::function<void(void* host)> unmap_chunk;
+ size_t reserve = 192; // slots kept evictable (not visible) for the window's demand and guesses
};
- bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& fds, const std::vector<os::fd_t>& dfds,
- const std::vector<os::fd_t>& pfds) {
+ bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {
cfg_ = c;
geo_ = layers;
- fds_ = fds;
dfds_ = dfds;
pfds_ = pfds;
- slot_bytes_ = 0;
- for (const LayerGeo& g : geo_) {
- size_t need = 0;
- for (int r = 0; r < 3; ++r) need += (size_t) g.bytes[r] + 2 * os::kDirectAlign;
- slot_bytes_ = std::max(slot_bytes_, need);
- }
- slot_bytes_ = (slot_bytes_ + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);
- if (slot_bytes_ == 0 || c.budget_bytes < slot_bytes_) return false;
+ size_t blob = 0;
+ for (const LayerGeo& g : geo_) blob = std::max<size_t>(blob, g.blob);
+ slot_bytes_ = (blob + 2 * kSlack + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);
+ if (blob == 0 || c.budget_bytes < slot_bytes_) return false;
nmax_ = c.budget_bytes / slot_bytes_;
+ // one reserved range for the whole arena (chunks committed as they are needed), 64 KB aligned chunks
+ arena_bytes_ = ((nmax_ + kChunk - 1) / kChunk) * kChunk * slot_bytes_;
+ arena_ = (uint8_t*) os::range_reserve(arena_bytes_);
+ if (arena_ == nullptr) return false;
slots_.reset(new Slot[nmax_]);
where_.resize(geo_.size());
heat_.resize(geo_.size());
@@ -83,11 +92,14 @@ public:
heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);
}
io_.reset(new IoQueue(std::max(1, c.threads)));
+ grow(); // (the first chunk now: whether the GPU can map it is known from the start)
return true;
}
~ExpertStore() {
io_.reset();
- for (auto& ch : chunks_) os::anon_free(ch.first, ch.second);
+ for (size_t i = 0; i < chunks_.size(); ++i)
+ if (cfg_.unmap_chunk && chunk_dev_[i] != nullptr) cfg_.unmap_chunk(chunks_[i].first);
+ os::range_release(arena_, arena_bytes_);
}
size_t slot_bytes() const { return slot_bytes_; }
@@ -95,8 +107,9 @@ public:
size_t slots_max() const { return nmax_; }
const Stats& stats() const { return st_; }
double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }
+ static constexpr size_t kSlack = 8192;
- /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned.)
+ /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned or visible.)
int find(int layer, int e) const {
if (layer < 0 || layer >= (int) where_.size()) return -1; // (a layer the store does not cover: the DSpark drafter's)
const int s = where_[(size_t) layer][(size_t) e];
@@ -105,11 +118,18 @@ public:
try_ready(sp);
return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;
}
- const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + slots_[(size_t) slot].lead[r]; }
+ const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + geo_[(size_t) slots_[(size_t) slot].layer].tr[r]; }
+ const uint8_t* blob(int slot) const { return slots_[(size_t) slot].base; }
+ /// The device address of the slot's blob (0: the arena is not mapped for the GPU)
+ unsigned long long dev_blob(int slot) const {
+ const size_t c = (size_t) slot / kChunk;
+ if (c >= chunk_dev_.size() || chunk_dev_[c] == nullptr) return 0;
+ return (unsigned long long) ((uint8_t*) chunk_dev_[c] + (slots_[(size_t) slot].base - (uint8_t*) chunks_[c].first));
+ }
/// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the
- /// caller releases them. False if a read failed or no slot could be had for one (then none of the failed ones is
- /// pinned: ok[i] says which are; the caller reads those another way).
+ /// caller releases them. False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller
+ /// reads the others another way).
bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {
ok.assign((size_t) n, 0);
if (layer < 0 || layer >= (int) geo_.size()) return false;
@@ -176,33 +196,110 @@ public:
if (where_[(size_t) layer][(size_t) e] >= 0) return;
if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);
}
- /// Heat of an expert as the policy sees it (tests / the report).
- float heat(int layer, int e) const { return heat_[(size_t) layer][(size_t) e]; }
+ /// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).
+ void touch(int layer, int e) {
+ if (layer < 0 || layer >= (int) where_.size()) return;
+ std::lock_guard<std::mutex> lk(mu_);
+ touch_heat(layer, e);
+ const int s = where_[(size_t) layer][(size_t) e];
+ if (s >= 0) slots_[(size_t) s].tick = ++clock_;
+ }
+ bool gpu_mapped() const { return !chunk_dev_.empty() && chunk_dev_[0] != nullptr; }
+ /// The arena's address range (the device sees its host pointers as they are when the mapping is the identity)
+ unsigned long long arena_lo() const { return (unsigned long long) (uintptr_t) arena_; }
+ unsigned long long arena_hi() const { return (unsigned long long) (uintptr_t) (arena_ + arena_bytes_); }
+ bool identity_mapped() const {
+ for (size_t i = 0; i < chunks_.size(); ++i)
+ if (chunk_dev_[i] != nullptr && chunk_dev_[i] != chunks_[i].first) return false;
+ return true;
+ }
+ size_t max_blob() const { return slot_bytes_ - 2 * kSlack; }
+
+ /// At a window boundary (nothing reading the slots; the caller applies `ev` to the device's table before the next window):
+ /// every ready slot not visible yet becomes visible (`gpu_ok(layer, e)`: false for an expert VRAM holds), and visible
+ /// slots of the lowest score are taken back until `cfg.reserve` slots can be given up. Returns the number of events.
+ size_t sync(std::vector<Vis>& ev, const std::function<bool(int, int)>& gpu_ok) {
+ std::lock_guard<std::mutex> lk(mu_);
+ size_t nvis = 0, nfree = 0;
+ for (size_t i = 0; i < nalloc_; ++i) {
+ Slot& sp = slots_[i];
+ if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
+ try_ready(sp);
+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
+ }
+ const int st = sp.state.load(std::memory_order_acquire);
+ if (st == kFree) {
+ ++nfree;
+ continue;
+ }
+ if (st != kReady) continue;
+ if (sp.visible) {
+ ++nvis;
+ continue;
+ }
+ if (sp.pin == 0) ++nfree;
+ if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch) {
+ sp.visible = true;
+ ++nvis;
+ if (sp.pin == 0) --nfree;
+ ev.push_back({sp.layer, sp.expert, dev_blob((int) i)});
+ }
+ }
+ // give slots back until the reserve is there: the visible ones with the lowest score
+ if (nfree < cfg_.reserve && nvis > 0) {
+ std::vector<std::pair<double, size_t>> cand;
+ cand.reserve(nvis);
+ for (size_t i = 0; i < nalloc_; ++i)
+ if (slots_[i].visible && slots_[i].state.load(std::memory_order_acquire) == kReady) cand.push_back({score(slots_[i]), i});
+ const size_t want = std::min(cand.size(), cfg_.reserve - nfree);
+ std::partial_sort(cand.begin(), cand.begin() + (long) want, cand.end());
+ for (size_t k = 0; k < want; ++k) {
+ Slot& sp = slots_[cand[k].second];
+ sp.visible = false;
+ ev.push_back({sp.layer, sp.expert, 0ull});
+ }
+ }
+ return ev.size();
+ }
+ /// An expert VRAM took over: no longer a visible slot
+ void unmark(int layer, int e) {
+ if (layer < 0 || layer >= (int) where_.size()) return;
+ std::lock_guard<std::mutex> lk(mu_);
+ const int s = where_[(size_t) layer][(size_t) e];
+ if (s >= 0) slots_[(size_t) s].visible = false;
+ }
+ size_t visible_count() const {
+ size_t n = 0;
+ for (size_t i = 0; i < nalloc_; ++i) n += slots_[i].visible ? 1 : 0;
+ return n;
+ }
private:
static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;
+ static constexpr size_t kChunk = 64; // slots a chunk of the arena
struct Slot {
- uint8_t* base = nullptr;
- uint32_t lead[3] = {0, 0, 0};
+ uint8_t* base = nullptr; // the blob's first byte
int layer = -1, expert = -1, pin = 0;
- bool prefetch = false;
+ bool prefetch = false, visible = false;
std::atomic<int> state{kFree}, left{0};
std::atomic<bool> bad{false};
uint64_t tick = 0;
};
+ double score(const Slot& sp) const {
+ if (cfg_.policy == kLru) return (double) sp.tick;
+ return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;
+ }
void touch_heat(int layer, int e) {
heat_[(size_t) layer][(size_t) e] += 1.0f;
- if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0) {
- // (the layer's heat x decay: a linear pass over 384 floats every 240 accesses of that layer)
- for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;
- }
+ if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0)
+ for (float& h : heat_[(size_t) layer]) h *= cfg_.decay; // (a linear pass over 384 floats every 240 accesses of that layer)
}
// a slot for (layer, e) and its reads started; -1 if there is none to be had. mu_ held.
int start_fill(int layer, int e, bool demand) {
int s = take_slot(demand);
if (s < 0) {
- st_.fails.fetch_add(1, std::memory_order_relaxed);
+ st_.no_slot.fetch_add(1, std::memory_order_relaxed);
return -1;
}
Slot& sp = slots_[(size_t) s];
@@ -215,6 +312,7 @@ private:
sp.expert = e;
sp.pin = demand ? 1 : 0;
sp.prefetch = !demand;
+ sp.visible = false;
sp.tick = ++clock_;
sp.state.store(kLoading, std::memory_order_release);
sp.bad.store(false);
@@ -222,20 +320,22 @@ private:
where_[(size_t) layer][(size_t) e] = s;
const LayerGeo& g = geo_[(size_t) layer];
IoQueue::Job jobs[3];
- size_t at = 0;
for (int r = 0; r < 3; ++r) {
const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];
const uint64_t lead = off % os::kDirectAlign;
- sp.lead[r] = (uint32_t) (at + lead);
+ uint8_t* fin = sp.base + g.tr[r]; // where the matrix belongs
+ // the aligned read lands the file's byte `off` at dst + lead: dst is fin - lead rounded down to 4096 (its span spills
+ // under 4096 bytes either way into the blob's gaps), the matrix is then moved up to fin
+ uint8_t* dst = (uint8_t*) ((uintptr_t) (fin - lead) & ~(uintptr_t) (os::kDirectAlign - 1));
IoQueue::Job& j = jobs[r];
j.fd = pfds_[(size_t) g.file[r]];
j.dfd = dfds_[(size_t) g.file[r]];
j.off = off;
j.len = g.bytes[r];
- j.dst = sp.base + at;
+ j.dst = dst;
+ j.keep = fin;
j.left = &sp.left;
j.bad = &sp.bad;
- at += os::direct_span(off, g.bytes[r]);
}
st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);
io_->push(jobs, 3);
@@ -261,19 +361,20 @@ private:
}
bool grow() {
if (nalloc_ >= nmax_) return false;
- constexpr size_t kChunk = 64; // slots
const size_t n = std::min(kChunk, nmax_ - nalloc_);
const size_t bytes = n * slot_bytes_;
const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};
if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;
- void* p = os::anon_alloc(bytes, -1, 1);
- if (p == nullptr) return false;
+ void* p = arena_ + nalloc_ * slot_bytes_;
+ if (!os::range_commit(p, bytes)) return false;
chunks_.push_back({p, bytes});
- for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_;
+ void* dev = cfg_.map_chunk ? cfg_.map_chunk(p, bytes) : nullptr;
+ chunk_dev_.push_back(dev);
+ for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_ + kSlack;
nalloc_ += n;
return true;
}
- // the slot to give up (or a free one): the policy's lowest among the unpinned ready ones. mu_ held.
+ // the slot to give up (or a free one): the policy's lowest among the unpinned, not visible, ready ones. mu_ held.
int take_slot(bool demand) {
for (size_t i = free_hint_; i < nalloc_; ++i)
if (slots_[i].state.load(std::memory_order_acquire) == kFree) {
@@ -287,20 +388,14 @@ private:
Slot& sp = slots_[i];
if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
try_ready(sp);
- if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp); // (failed: freed)
+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
if (sp.state.load(std::memory_order_acquire) == kFree) {
free_hint_ = std::min(free_hint_, i);
return (int) i;
}
}
- if (sp.pin > 0 || sp.state.load(std::memory_order_acquire) != kReady) continue;
- double v;
- if (cfg_.policy == kLru) v = (double) sp.tick;
- else {
- const double h = heat_[(size_t) sp.layer][(size_t) sp.expert];
- v = cfg_.policy == kLfu ? h : h * 1e12 + (double) sp.tick; // lfu with the age as tie-break
- if (cfg_.policy == kLfuLru) v = h + 1e-9 * (double) sp.tick;
- }
+ if (sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) != kReady) continue;
+ const double v = score(sp);
if (best < 0 || v < bv) {
best = (int) i;
bv = v;
@@ -313,10 +408,12 @@ private:
Cfg cfg_;
std::vector<LayerGeo> geo_;
- std::vector<os::fd_t> fds_, dfds_, pfds_;
- size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0;
+ std::vector<os::fd_t> dfds_, pfds_;
+ size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;
+ uint8_t* arena_ = nullptr;
std::unique_ptr<Slot[]> slots_;
std::vector<std::pair<void*, size_t>> chunks_;
+ std::vector<void*> chunk_dev_;
std::vector<std::vector<int>> where_;
std::vector<std::vector<float>> heat_;
std::vector<uint64_t> hits_in_layer_;
diff --git a/tools/dsv41/win/hwbench.cu b/tools/dsv41/win/hwbench.cu
index 0d03d3bf3..d351dade4 100644
--- a/tools/dsv41/win/hwbench.cu
+++ b/tools/dsv41/win/hwbench.cu
@@ -95,10 +95,49 @@ static void bench_nvme(const char* path, size_t block, int qd, double secs, bool
CloseHandle(f);
}
+// GPU kernels reading host memory over PCIe in place (zero-copy): the pinned + mapped (or registered) arena as a weights source
+__global__ void zc_sum(const int4* __restrict__ p, size_t n, unsigned long long* out) {
+ unsigned long long acc = 0;
+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (size_t) gridDim.x * blockDim.x) {
+ const int4 v = p[i];
+ acc += (unsigned) v.x ^ (unsigned) v.y ^ (unsigned) v.z ^ (unsigned) v.w;
+ }
+ if (acc == 0x123456789ull) atomicAdd(out, acc);
+}
+static void bench_zc(size_t mib) {
+ const size_t n = mib << 20;
+ for (int mode = 0; mode < 2; ++mode) {
+ void *h = nullptr, *dp = nullptr;
+ if (mode == 0) {
+ if (cudaHostAlloc(&h, n, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) { printf("hostalloc failed\n"); continue; }
+ } else {
+ h = VirtualAlloc(nullptr, n, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+ memset(h, 1, n);
+ if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) { printf("hostregister failed\n"); continue; }
+ }
+ memset(h, 1, n);
+ cudaHostGetDevicePointer(&dp, h, 0);
+ unsigned long long* out; cudaMalloc(&out, 8);
+ for (int blocks : {64, 256, 1024}) {
+ zc_sum<<<blocks, 256>>>((const int4*) dp, n / 16, out);
+ cudaDeviceSynchronize();
+ cudaEvent_t a, b; cudaEventCreate(&a); cudaEventCreate(&b);
+ cudaEventRecord(a);
+ for (int r = 0; r < 4; ++r) zc_sum<<<blocks, 256>>>((const int4*) dp, n / 16, out);
+ cudaEventRecord(b); cudaEventSynchronize(b);
+ float ms; cudaEventElapsedTime(&ms, a, b);
+ printf("zero-copy %s %zu MiB, %d blocks: %.2f GB/s\n", mode == 0 ? "cudaHostAlloc(mapped)" : "VirtualAlloc+cudaHostRegister(mapped)", mib, blocks, 4.0 * n / 1e9 / (ms / 1e3));
+ }
+ if (mode == 0) cudaFreeHost(h); else { cudaHostUnregister(h); VirtualFree(h, 0, MEM_RELEASE); }
+ cudaFree(out);
+ }
+}
+
int main(int argc, char** argv) {
std::string m = argc > 1 ? argv[1] : "";
if (m == "ram") bench_ram(argc > 2 ? atoi(argv[2]) : 6, argc > 3 ? atoi(argv[3]) : 512);
else if (m == "pcie") bench_pcie();
+ else if (m == "zc") bench_zc(argc > 2 ? atoi(argv[2]) : 1024);
else if (m == "nvme" && argc > 5) bench_nvme(argv[2], (size_t)atoi(argv[3]) << 10, atoi(argv[4]), atof(argv[5]), argc > 6 && std::string(argv[6]) == "seq");
else printf("usage: hwbench ram|pcie|nvme ...\n");
}