Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 win: RAM expert store (unbuffered reads, policy, lookahead fill), Windows direct reads, dense dequant for the chunked prompt, SMT option, store test

Niko1221e7b3f1524044
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 55d849448..5bee58804 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1154,6 +1154,10 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
}
void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
+ if (m.fmt == kFmtGG && m.raw != nullptr) { // a K-quant / i-quant dense matrix (the IST-DAS sets' Q5_K): its own dequantizer
+ if (!iq::dequant_rows(m.K, m.raw, row_stride(m), m.k, m.n, dst, bf16, s)) fail("rows: a low-bit type not covered");
+ return;
+ }
if ((m.fmt != kFmtBF16 && m.fmt != kFmtQ8_0) || m.raw == nullptr) {
fail("rows: not a BF16 or Q8_0 matrix");
return;
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index a49e2cffb..ed251059d 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -3,6 +3,7 @@
// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are
// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.
#include "ds4_iq.cuh"
+#include "ds4_numerics.cuh"
#include <cuda_fp16.h>
#if !defined(STRATA_USE_HIP)
@@ -1007,8 +1008,60 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
else mv_kernel<T, 16><<<grid, kThreads, 0, s>>>(J);
}
+
+// ---------------------------------------------------------------- whole matrices as 16-bit rows (the chunked prompt's GEMMs)
+// n rows of k values from `raw` (ld bytes a row): one thread a 32-value sub-block, the sub-block's codes and scales from decode<>
+// (value = d s q - m of its half, ggml's dequantize_row_*), rounded to BF16 (bf16) or F16
+template <int T>
+__global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __restrict__ raw, size_t ld, int k, int n,
+ uint16_t* __restrict__ dst, bool bf16) {
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ load_tab<T>(tab);
+ __syncthreads();
+ const int nsb = k / 32;
+ const size_t total = (size_t) n * (size_t) nsb;
+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
+ const size_t r = i / (size_t) nsb;
+ const int kb = (int) (i - r * (size_t) nsb);
+ Sub sb;
+ sb.m0 = sb.m1 = 0.0f;
+ decode<T>(raw + r * ld, kb, tab, sb);
+ const float d0 = __fmul_rn(sb.d, (float) sb.s0), d1 = __fmul_rn(sb.d, (float) sb.s1);
+ uint16_t* o = dst + r * (size_t) k + (size_t) kb * 32;
+#pragma unroll
+ for (int j = 0; j < 32; ++j) {
+ const int q = (int) (int8_t) ((uint32_t) sb.q[j >> 2] >> (8 * (j & 3)));
+ const float v = j < 16 ? __fsub_rn(__fmul_rn(d0, (float) q), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q), sb.m1);
+ o[j] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));
+ }
+ }
+}
+
} // namespace
+bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {
+ if (raw == nullptr || k % 256 != 0 || n <= 0) return false;
+ const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);
+#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)
+ switch (type) {
+ case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;
+ case kIQ2_XS: DS4_DQ(kIQ2_XS); break;
+ case kIQ2_S: DS4_DQ(kIQ2_S); break;
+ case kIQ3_XXS: DS4_DQ(kIQ3_XXS); break;
+ case kIQ3_S: DS4_DQ(kIQ3_S); break;
+ case kIQ4_XS: DS4_DQ(kIQ4_XS); break;
+ case kIQ1_M: DS4_DQ(kIQ1_M); break;
+ case kQ4_K: DS4_DQ(kQ4_K); break;
+ case kQ5_K: DS4_DQ(kQ5_K); break;
+ case kQ6_K: DS4_DQ(kQ6_K); break;
+ case kQ2_K: DS4_DQ(kQ2_K); break;
+ case kQ3_K: DS4_DQ(kQ3_K); break;
+ default: return false;
+ }
+#undef DS4_DQ
+ return cudaGetLastError() == cudaSuccess;
+}
+
bool supported(int t) {
// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)
switch (t) {
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 0cf49c957..62dc1b574 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -116,6 +116,10 @@ struct MvJob {
};
bool mv(const MvJob* jobs, int n, cudaStream_t s);
+/// n rows of k values of a matrix of the type (ld bytes a row) as BF16 (bf16) or F16 bits, dst[n][k]: the chunked prompt path's
+/// dense GEMM operand. false: a type not covered.
+bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s);
+
/// The trellis types' kernels (ds4_tq.cu), reached from the entry points above for those types: arithmetic as the CPU lane's
/// (ds4_tq_cpu.cpp: int8 activations with an F32 scale a 32, exact weights), on the tensor cores of gfx12 (WMMA) or in FMAs.
namespace tq {
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index 889f30405..be716c935 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -71,7 +71,7 @@ std::vector<int> parse_ids(const std::string& s) {
void usage() {
std::fprintf(stderr,
"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"
- " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--no-expert-cache] [--verbose]\n"
+ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"
" [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"
" [--vision on|off] [--mmap-experts on|off|auto]\n"
" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
@@ -666,6 +666,11 @@ int main(int argc, char** argv) {
else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());
else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());
else if (a == "--expert-profile") o.profile = val("--expert-profile");
+ else if (a == "--ssd-cache") o.ssd_cache_gb = std::atof(val("--ssd-cache").c_str());
+ else if (a == "--ssd-policy") {
+ const std::string v = val("--ssd-policy");
+ o.ssd_policy = v == "lfu" ? 1 : v == "lfulru" ? 2 : 0;
+ }
else if (a == "--eos") eos = val("--eos");
else if (a == "--no-expert-cache") o.expert_cache = false;
else if (a == "--dspark") {
@@ -813,7 +818,7 @@ int main(int argc, char** argv) {
out.push_back(tok);
std::printf("%d,", tok);
std::fflush(stdout);
- if (tok == 1 || (int) out.size() >= n) break;
+ if ((tok == 1 && std::getenv("DS4_NO_EOS") == nullptr) || (int) out.size() >= n) break; // (DS4_NO_EOS: decode n tokens whatever)
if (got.empty()) {
int off = 0, acc = 0;
const bool try_spec = spec && !cool;
@@ -870,6 +875,18 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "decode left out %.1f experts a token not in RAM (%.1f%% of the routed)\n",
(double) (s.skipped - s0.skipped) / nt,
100.0 * (double) (s.skipped - s0.skipped) / std::max<double>(1.0, (double) (s.experts - s0.experts)));
+ if (s.store_slots_max > 0) { // --ssd-cache
+ const double h = (double) (s.store_hits - s0.store_hits), m = (double) (s.store_misses - s0.store_misses);
+ std::fprintf(stderr,
+ "store: %.1f%% of the CPU lane's experts were in RAM (%.1f hits, %.1f misses a token); read %.1f MB a token "
+ "(demand and guesses), the CPU lane waited %.1f ms a token for them; %lld prefetched, %lld used, %lld given up "
+ "unused; slots %lld of %lld, %lld evictions, %lld failed reads\n",
+ 100.0 * h / std::max(1.0, h + m), h / nt, m / nt, (double) (s.store_bytes - s0.store_bytes) / 1e6 / nt,
+ (s.store_wait_ms - s0.store_wait_ms) / nt, (long long) (s.store_prefetched - s0.store_prefetched),
+ (long long) (s.store_pf_used - s0.store_pf_used), (long long) (s.store_pf_wasted - s0.store_pf_wasted),
+ (long long) s.store_slots, (long long) s.store_slots_max, (long long) (s.store_evictions - s0.store_evictions),
+ (long long) (s.store_fails - s0.store_fails));
+ }
if (s.ahead_picks > s0.ahead_picks) { // lookahead
const double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);
std::fprintf(stderr, "lookahead: the guesses a layer before held %.1f%% of the picks (their top k %.1f%%, two layers "
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 774cfce73..8829c8594 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -20,6 +20,7 @@
#include "ds4_kernels.cuh"
#include "ds4_os.hpp"
#include "ds4_pool.hpp"
+#include "ds4_store.hpp"
#include "ds4_vision.cuh"
#include "strata/dsv41/exl3_cpu.hpp"
#include "strata/dsv41/exl3.hpp"
@@ -111,7 +112,11 @@ std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
// each NUMA node's physical cores (the first logical CPU of each core), in order
std::vector<std::vector<int>> node_cores() {
#if defined(_WIN32)
- std::vector<std::vector<int>> nodes = os::node_cores();
+ static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)
+ const char* v = std::getenv("DS4_CPU_SMT");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ std::vector<std::vector<int>> nodes = os::node_cores(smt);
#else
std::vector<std::vector<int>> nodes;
for (int n = 0; n < 64; ++n) {
@@ -981,6 +986,8 @@ struct Model::Impl {
const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
return v != nullptr ? std::max(0, std::atoi(v)) : 0;
}();
+ // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
+ std::unique_ptr<ExpertStore> store;
bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
if (l.lay.fmt == gg::kFmtMXFP4) {
@@ -1586,6 +1593,10 @@ struct Model::Impl {
return true;
}
const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
+ if (store) {
+ const int s = store->find(l.il, e);
+ if (s >= 0) return store->mat(s, r);
+ }
if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
: ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
@@ -2389,6 +2400,7 @@ struct Model::Impl {
const uint64_t avail = ram_avail();
warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
+ if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)
std::fprintf(stderr,
"ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
"(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
@@ -2416,6 +2428,7 @@ struct Model::Impl {
if (!place_experts(err)) return false;
if (!load_experts(err)) return false;
if (pk.gguf) make_pools();
+ init_store();
if (chunk > 0) {
// the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
const double tr = now_ms();
@@ -4171,6 +4184,46 @@ struct Model::Impl {
return s.str();
}
+ double store_gb() const {
+ if (!pk.gguf) return 0.0;
+ if (const char* v = std::getenv("DS4_STORE_GB")) return std::atof(v);
+ return o.ssd_cache_gb;
+ }
+ void init_store() {
+ const double gb = store_gb();
+ if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;
+ std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ ExpertStore::LayerGeo& gl = geo[(size_t) l.il];
+ gl.n_exp = l.n_exp;
+ for (int r = 0; r < 3; ++r) {
+ gl.file[r] = l.mx_file[r];
+ gl.off[r] = l.mx_off[r];
+ gl.stride[r] = l.mx_stride[r];
+ gl.bytes[r] = mx_part_bytes(l, r);
+ }
+ }
+ std::vector<std::string> paths;
+ ExpertStore::Cfg cfg;
+ cfg.budget_bytes = (size_t) (gb * 1e9);
+ cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
+ cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
+ cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
+ cfg.ram_avail = [] { return ram_avail(); };
+ bool direct = true;
+ for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
+ auto st = std::make_unique<ExpertStore>();
+ if (!st->init(cfg, geo, pk.fds, pk.dfds, pk.pfds)) {
+ std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
+ return;
+ }
+ store = std::move(st);
+ std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
+ store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
+ (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
+ }
+
int cpu_threads() const {
int n = 0;
for (const auto& pl : pools) n += pl->size() - 1;
@@ -4599,7 +4652,7 @@ struct Model::Impl {
// more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
std::vector<std::pair<const void*, size_t>> want;
for (const Ent& en : ents)
- if (l.ram.empty() || l.ram[(size_t) en.e] < 0)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
os::prefetch_cold(want);
}
@@ -4689,7 +4742,7 @@ struct Model::Impl {
{ // (os::prefetch, as above)
std::vector<std::pair<const void*, size_t>> want;
for (const Ent& en : ents)
- if (l.ram.empty() || l.ram[(size_t) en.e] < 0)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
os::prefetch(want);
}
@@ -4747,10 +4800,31 @@ struct Model::Impl {
run_nodes(ne * DB, f);
}
+ // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
+ struct StoreHold {
+ Impl* m = nullptr;
+ int layer = 0;
+ std::vector<int> es;
+ std::vector<char> ok;
+ StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
+ if (!m_->store) return;
+ m = m_;
+ layer = l.il;
+ for (const Ent& en : ents)
+ if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
+ m->store->ensure(layer, es.data(), (int) es.size(), ok);
+ }
+ ~StoreHold() {
+ if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
+ }
+ };
void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
const int* ord = nullptr) {
if (ldx == 0) ldx = (size_t) g.dim;
- if (l.lay.fmt == gg::kFmtMXFP4) return cpu_experts_mx(l, x, ents, ldx, ord);
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ StoreHold hold(this, l, ents);
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
const int ne = (int) ents.size();
@@ -5215,7 +5289,9 @@ struct Model::Impl {
if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
if ((fl[e] & 8) || !file_read(n, e)) continue;
fl[e] |= 8;
- for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
+ if (store) store->prefetch(j, e);
+ else
+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
++st.ahead_pf;
}
}
@@ -7930,7 +8006,25 @@ void Model::checkpoint(int slot) { p_->checkpoint(slot); }
bool Model::restore(int64_t limit) { return p_->restore(limit); }
int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
void Model::save_profile() const { p_->save_profile(); }
-const Stats& Model::stats() const { return p_->st; }
+const Stats& Model::stats() const {
+ Stats& st = p_->st;
+ if (p_->store) { // (the store keeps its own counters: copied in when asked)
+ const ExpertStore& x = *p_->store;
+ st.store_hits = x.stats().hits.load();
+ st.store_misses = x.stats().misses.load();
+ st.store_prefetched = x.stats().prefetched.load();
+ st.store_pf_used = x.stats().prefetch_used.load();
+ st.store_pf_wasted = x.stats().prefetch_wasted.load();
+ st.store_evictions = x.stats().evictions.load();
+ st.store_bytes = x.stats().bytes_read.load();
+ st.store_fails = x.stats().fails.load();
+ st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
+ st.store_busy_ms = x.busy_ms();
+ st.store_slots = (int64_t) x.slots_allocated();
+ st.store_slots_max = (int64_t) x.slots_max();
+ }
+ return st;
+}
std::string Model::describe() const { return p_->describe(); }
std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index 91c72f771..41b87a847 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -32,6 +32,11 @@ struct Options {
// the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of
// copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold them
int mmap_experts = -1;
+ // the experts the VRAM leaves over are cached in RAM by a store of their own (ds4_store.hpp): whole experts read unbuffered into
+ // slots, a replacement policy (0 lru, 1 lfu, 2 lfu then lru), filled on demand and by the lookahead's guesses. GB of RAM at
+ // most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before
+ double ssd_cache_gb = 0.0;
+ int ssd_policy = 0;
bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this
// tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the
@@ -73,6 +78,12 @@ struct Stats {
int64_t ahead_picks = 0, ahead_hit_k = 0, ahead_hit = 0, ahead_hit2 = 0;
int64_t ahead_cold = 0, ahead_cold_hit = 0, ahead_pf = 0;
int64_t ahead_rank_n[16] = {}, ahead_rank_hit[16] = {}; // the next-layer guesses by rank, and how many were picked
+ // the RAM store (--ssd-cache), totals since the load: reads asked for and found there (hits), missed (misses), started ahead
+ // of use (prefetched) of which used / given up unused, slots given up, bytes read, ms the CPU lane waited for the reads
+ int64_t store_hits = 0, store_misses = 0, store_prefetched = 0, store_pf_used = 0, store_pf_wasted = 0, store_evictions = 0;
+ int64_t store_bytes = 0, store_fails = 0;
+ double store_wait_ms = 0.0, store_busy_ms = 0.0;
+ int64_t store_slots = 0, store_slots_max = 0;
int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;
// per draft position (the 1st .. 8th of a round): verified with all the drafts before it accepted, and accepted
int64_t draft_reached[8] = {}, draft_kept[8] = {};
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
index 689f5ec79..d93d9632b 100644
--- a/src/dsv41/ds4_os.hpp
+++ b/src/dsv41/ds4_os.hpp
@@ -61,8 +61,16 @@ inline fd_t open_ro(const std::string& path) {
/// kBadFd where the filesystem refuses it, or on Windows, whose unbuffered reads of a mapped file take a slow path)
inline fd_t open_direct(const std::string& path) {
#if defined(_WIN32)
- (void) path;
- return kBadFd;
+ // FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED: the disk straight into an aligned buffer, any number of reads in flight on
+ // one handle (DS4_WIN_DIRECT=0: none, the page cache's buffered reads as before)
+ static const bool off = [] {
+ const char* v = std::getenv("DS4_WIN_DIRECT");
+ return v != nullptr && std::atoi(v) == 0;
+ }();
+ if (off) return kBadFd;
+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
+ FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);
+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
#else
const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);
return fd < 0 ? kBadFd : (fd_t) fd;
@@ -78,8 +86,40 @@ inline uint64_t direct_span(uint64_t off, uint64_t n) {
}
inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {
#if defined(_WIN32)
- (void) f, (void) dst, (void) n, (void) off;
- return false;
+ struct Ev { // (each thread its own, closed when it ends)
+ HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+ ~Ev() {
+ if (h != nullptr) CloseHandle(h);
+ }
+ };
+ static thread_local Ev ev;
+ if (ev.h == nullptr) return false;
+ const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;
+ uint8_t* p = (uint8_t*) dst;
+ uint64_t got = 0;
+ while (got < need) {
+ const DWORD ask = (DWORD) std::min<uint64_t>(need - got, 1ull << 30); // (a multiple of 4096)
+ OVERLAPPED ov{};
+ const uint64_t at = a0 + got;
+ ov.Offset = (DWORD) (at & 0xffffffffu);
+ ov.OffsetHigh = (DWORD) (at >> 32);
+ ov.hEvent = ev.h;
+ ResetEvent(ev.h);
+ DWORD r = 0;
+ if (!ReadFile((HANDLE) f, p + got, ask, nullptr, &ov)) {
+ const DWORD e = GetLastError();
+ if (e == ERROR_HANDLE_EOF) break;
+ if (e != ERROR_IO_PENDING) return false;
+ }
+ if (!GetOverlappedResult((HANDLE) f, &ov, &r, TRUE)) {
+ if (GetLastError() == ERROR_HANDLE_EOF) break;
+ return false;
+ }
+ if (r == 0) break; // (the file's end: the last aligned read is short)
+ got += r;
+ if (r % kDirectAlign != 0) break;
+ }
+ return got >= want;
#else
const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;
uint8_t* p = (uint8_t*) dst;
@@ -370,7 +410,7 @@ inline void pin_thread(std::thread& t, int cpu) {
#if defined(_WIN32)
/// Each NUMA node's physical cores (the first logical CPU of each), processor group 0 - Windows' answer to
/// /sys/devices/system/node; empty if the OS does not say.
-inline std::vector<std::vector<int>> node_cores() {
+inline std::vector<std::vector<int>> node_cores(bool smt = false) { // (smt: every logical CPU of a core, not its first)
std::vector<std::vector<int>> nodes;
auto query = [](LOGICAL_PROCESSOR_RELATIONSHIP rel, std::vector<uint8_t>& buf) {
DWORD len = 0;
@@ -398,7 +438,12 @@ inline std::vector<std::vector<int>> node_cores() {
break;
}
for (size_t n = 0; n < masks.size() && first >= 0; ++n)
- if ((masks[n] >> first) & 1) nodes[n].push_back(first);
+ if ((masks[n] >> first) & 1) {
+ nodes[n].push_back(first);
+ if (smt)
+ for (int b = first + 1; b < 64; ++b)
+ if ((m >> b) & 1) nodes[n].push_back(b);
+ }
}
o += e->Size;
}
diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hpp
new file mode 100644
index 000000000..1a407ba83
--- /dev/null
+++ b/src/dsv41/ds4_store.hpp
@@ -0,0 +1,329 @@
+// Strata. src/dsv41/ds4_store.hpp - the RAM tier of the DeepSeek-V4.1 routed experts as a cache of its own: whole experts read
+// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots
+// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).
+//
+// An expert (layer, e) is three matrices at three file offsets (the GGUF's one tensor a role and layer). A slot holds their
+// bytes at 4096-aligned places (unbuffered reads land the file range rounded out to 4096 at an aligned address; the matrix's
+// first byte is `lead` bytes into it), so mat(slot, r) is the pointer the CPU lane's rows read - the same bytes as the file's
+// mapping, so every answer is the same. Nothing here touches a device.
+//
+// Memory: slots are allocated in chunks as they are first needed (VirtualAlloc / mmap, private memory, not pinned), never past
+// `max_slots` nor past the RAM this process can still get less `floor_bytes` (the owner's session and the system keep theirs).
+//
+// Policy (pick the slot to give up, among the unpinned ready ones): lru (oldest touch), lfu (lowest decayed hit count), or
+// lfu with an LRU tie-break. A slot whose expert a window is computing is pinned from ensure() to release().
+#pragma once
+
+#include "ds4_os.hpp"
+#include "ds4_pool.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+class ExpertStore {
+public:
+ struct LayerGeo {
+ int n_exp = 0;
+ int file[3] = {-1, -1, -1};
+ uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0}; // matrix r of expert e: file[r], off[r] + e stride[r]
+ uint64_t bytes[3] = {0, 0, 0};
+ };
+ enum Policy { kLru = 0, kLfu = 1, kLfuLru = 2 };
+ struct Stats {
+ std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};
+ std::atomic<int64_t> bytes_read{0}, wait_ns{0};
+ };
+
+ struct Cfg {
+ const std::vector<std::string>* paths = nullptr; // the files, by the layers' file indices
+ int threads = 8;
+ size_t budget_bytes = 0; // the arena at most (grown in chunks as slots are first needed)
+ size_t floor_bytes = 8ull << 30; // RAM left alone
+ Policy policy = kLru;
+ float decay = 0.97f; // lfu: heat x decay every `decay_every` accesses of its layer
+ int decay_every = 240;
+ std::function<uint64_t()> ram_avail; // the RAM this process can still get (~0: unknown)
+ };
+
+ bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& fds, const std::vector<os::fd_t>& dfds,
+ const std::vector<os::fd_t>& pfds) {
+ cfg_ = c;
+ geo_ = layers;
+ fds_ = fds;
+ dfds_ = dfds;
+ pfds_ = pfds;
+ slot_bytes_ = 0;
+ for (const LayerGeo& g : geo_) {
+ size_t need = 0;
+ for (int r = 0; r < 3; ++r) need += (size_t) g.bytes[r] + 2 * os::kDirectAlign;
+ slot_bytes_ = std::max(slot_bytes_, need);
+ }
+ slot_bytes_ = (slot_bytes_ + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);
+ if (slot_bytes_ == 0 || c.budget_bytes < slot_bytes_) return false;
+ nmax_ = c.budget_bytes / slot_bytes_;
+ slots_.reset(new Slot[nmax_]);
+ where_.resize(geo_.size());
+ heat_.resize(geo_.size());
+ hits_in_layer_.assign(geo_.size(), 0);
+ for (size_t l = 0; l < geo_.size(); ++l) {
+ where_[l].assign((size_t) geo_[l].n_exp, -1);
+ heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);
+ }
+ io_.reset(new IoQueue(std::max(1, c.threads)));
+ return true;
+ }
+ ~ExpertStore() {
+ io_.reset();
+ for (auto& ch : chunks_) os::anon_free(ch.first, ch.second);
+ }
+
+ size_t slot_bytes() const { return slot_bytes_; }
+ size_t slots_allocated() const { return nalloc_; }
+ size_t slots_max() const { return nmax_; }
+ const Stats& stats() const { return st_; }
+ double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }
+
+ /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned.)
+ int find(int layer, int e) const {
+ if (layer < 0 || layer >= (int) where_.size()) return -1; // (a layer the store does not cover: the DSpark drafter's)
+ const int s = where_[(size_t) layer][(size_t) e];
+ if (s < 0) return -1;
+ Slot& sp = slots_[(size_t) s];
+ try_ready(sp);
+ return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;
+ }
+ const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + slots_[(size_t) slot].lead[r]; }
+
+ /// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the
+ /// caller releases them. False if a read failed or no slot could be had for one (then none of the failed ones is
+ /// pinned: ok[i] says which are; the caller reads those another way).
+ bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {
+ ok.assign((size_t) n, 0);
+ if (layer < 0 || layer >= (int) geo_.size()) return false;
+ std::vector<int> sl((size_t) n, -1);
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ for (int i = 0; i < n; ++i) {
+ const int e = es[i];
+ touch_heat(layer, e);
+ int s = where_[(size_t) layer][(size_t) e];
+ if (s >= 0) {
+ Slot& sp = slots_[(size_t) s];
+ try_ready(sp);
+ if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);
+ if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);
+ sp.prefetch = false;
+ sp.pin++;
+ sp.tick = ++clock_;
+ sl[(size_t) i] = s;
+ continue;
+ }
+ st_.misses.fetch_add(1, std::memory_order_relaxed);
+ s = start_fill(layer, e, true);
+ sl[(size_t) i] = s;
+ }
+ }
+ const auto t0 = std::chrono::steady_clock::now();
+ bool all = true;
+ for (int i = 0; i < n; ++i) {
+ const int s = sl[(size_t) i];
+ if (s < 0) {
+ all = false;
+ continue;
+ }
+ Slot& sp = slots_[(size_t) s];
+ while (sp.left.load(std::memory_order_acquire) > 0) io_->wait_any(2000);
+ std::lock_guard<std::mutex> lk(mu_);
+ finish(sp);
+ if (sp.state.load() == kReady) ok[(size_t) i] = 1;
+ else {
+ sp.pin--;
+ sl[(size_t) i] = -1;
+ all = false;
+ }
+ }
+ st_.wait_ns.fetch_add(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count(),
+ std::memory_order_relaxed);
+ return all;
+ }
+ void release(int layer, const int* es, int n, const std::vector<char>& ok) {
+ if (layer < 0 || layer >= (int) geo_.size()) return;
+ std::lock_guard<std::mutex> lk(mu_);
+ for (int i = 0; i < n; ++i)
+ if (ok[(size_t) i]) {
+ const int s = where_[(size_t) layer][(size_t) es[i]];
+ if (s >= 0 && slots_[(size_t) s].pin > 0) slots_[(size_t) s].pin--;
+ }
+ }
+ /// Starts reading (layer, e) if it is neither here nor on its way (a guess: unpinned, may be given up before it is used).
+ void prefetch(int layer, int e) {
+ if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return;
+ if (where_[(size_t) layer][(size_t) e] >= 0) return;
+ std::lock_guard<std::mutex> lk(mu_);
+ if (where_[(size_t) layer][(size_t) e] >= 0) return;
+ if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);
+ }
+ /// Heat of an expert as the policy sees it (tests / the report).
+ float heat(int layer, int e) const { return heat_[(size_t) layer][(size_t) e]; }
+
+private:
+ static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;
+ struct Slot {
+ uint8_t* base = nullptr;
+ uint32_t lead[3] = {0, 0, 0};
+ int layer = -1, expert = -1, pin = 0;
+ bool prefetch = false;
+ std::atomic<int> state{kFree}, left{0};
+ std::atomic<bool> bad{false};
+ uint64_t tick = 0;
+ };
+
+ void touch_heat(int layer, int e) {
+ heat_[(size_t) layer][(size_t) e] += 1.0f;
+ if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0) {
+ // (the layer's heat x decay: a linear pass over 384 floats every 240 accesses of that layer)
+ for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;
+ }
+ }
+ // a slot for (layer, e) and its reads started; -1 if there is none to be had. mu_ held.
+ int start_fill(int layer, int e, bool demand) {
+ int s = take_slot(demand);
+ if (s < 0) {
+ st_.fails.fetch_add(1, std::memory_order_relaxed);
+ return -1;
+ }
+ Slot& sp = slots_[(size_t) s];
+ if (sp.layer >= 0 && sp.expert >= 0) {
+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+ if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);
+ st_.evictions.fetch_add(1, std::memory_order_relaxed);
+ }
+ sp.layer = layer;
+ sp.expert = e;
+ sp.pin = demand ? 1 : 0;
+ sp.prefetch = !demand;
+ sp.tick = ++clock_;
+ sp.state.store(kLoading, std::memory_order_release);
+ sp.bad.store(false);
+ sp.left.store(3, std::memory_order_release);
+ where_[(size_t) layer][(size_t) e] = s;
+ const LayerGeo& g = geo_[(size_t) layer];
+ IoQueue::Job jobs[3];
+ size_t at = 0;
+ for (int r = 0; r < 3; ++r) {
+ const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];
+ const uint64_t lead = off % os::kDirectAlign;
+ sp.lead[r] = (uint32_t) (at + lead);
+ IoQueue::Job& j = jobs[r];
+ j.fd = pfds_[(size_t) g.file[r]];
+ j.dfd = dfds_[(size_t) g.file[r]];
+ j.off = off;
+ j.len = g.bytes[r];
+ j.dst = sp.base + at;
+ j.left = &sp.left;
+ j.bad = &sp.bad;
+ at += os::direct_span(off, g.bytes[r]);
+ }
+ st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);
+ io_->push(jobs, 3);
+ return s;
+ }
+ // loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)
+ void try_ready(Slot& sp) const {
+ int want = kLoading;
+ if (sp.left.load(std::memory_order_acquire) == 0 && !sp.bad.load(std::memory_order_acquire))
+ sp.state.compare_exchange_strong(want, kReady, std::memory_order_acq_rel);
+ }
+ void finish(Slot& sp) { // mu_ held; reads done
+ if (sp.state.load() != kLoading) return;
+ if (sp.bad.load()) {
+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
+ sp.layer = sp.expert = -1;
+ sp.state.store(kFree, std::memory_order_release);
+ sp.prefetch = false;
+ st_.fails.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ sp.state.store(kReady, std::memory_order_release);
+ }
+ bool grow() {
+ if (nalloc_ >= nmax_) return false;
+ constexpr size_t kChunk = 64; // slots
+ const size_t n = std::min(kChunk, nmax_ - nalloc_);
+ const size_t bytes = n * slot_bytes_;
+ const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};
+ if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;
+ void* p = os::anon_alloc(bytes, -1, 1);
+ if (p == nullptr) return false;
+ chunks_.push_back({p, bytes});
+ for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_;
+ nalloc_ += n;
+ return true;
+ }
+ // the slot to give up (or a free one): the policy's lowest among the unpinned ready ones. mu_ held.
+ int take_slot(bool demand) {
+ for (size_t i = free_hint_; i < nalloc_; ++i)
+ if (slots_[i].state.load(std::memory_order_acquire) == kFree) {
+ free_hint_ = i + 1;
+ return (int) i;
+ }
+ if (grow()) return take_slot(demand);
+ int best = -1;
+ double bv = 0;
+ for (size_t i = 0; i < nalloc_; ++i) {
+ Slot& sp = slots_[i];
+ if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
+ try_ready(sp);
+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp); // (failed: freed)
+ if (sp.state.load(std::memory_order_acquire) == kFree) {
+ free_hint_ = std::min(free_hint_, i);
+ return (int) i;
+ }
+ }
+ if (sp.pin > 0 || sp.state.load(std::memory_order_acquire) != kReady) continue;
+ double v;
+ if (cfg_.policy == kLru) v = (double) sp.tick;
+ else {
+ const double h = heat_[(size_t) sp.layer][(size_t) sp.expert];
+ v = cfg_.policy == kLfu ? h : h * 1e12 + (double) sp.tick; // lfu with the age as tie-break
+ if (cfg_.policy == kLfuLru) v = h + 1e-9 * (double) sp.tick;
+ }
+ if (best < 0 || v < bv) {
+ best = (int) i;
+ bv = v;
+ }
+ }
+ if (best < 0) return -1;
+ free_hint_ = std::min<size_t>(free_hint_, (size_t) best);
+ return best;
+ }
+
+ Cfg cfg_;
+ std::vector<LayerGeo> geo_;
+ std::vector<os::fd_t> fds_, dfds_, pfds_;
+ size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0;
+ std::unique_ptr<Slot[]> slots_;
+ std::vector<std::pair<void*, size_t>> chunks_;
+ std::vector<std::vector<int>> where_;
+ std::vector<std::vector<float>> heat_;
+ std::vector<uint64_t> hits_in_layer_;
+ uint64_t clock_ = 0;
+ std::mutex mu_;
+ std::unique_ptr<IoQueue> io_;
+ Stats st_;
+};
+
+} // namespace strata::ds4
diff --git a/tests/dsv41/store_test.cpp b/tests/dsv41/store_test.cpp
new file mode 100644
index 000000000..4181db041
--- /dev/null
+++ b/tests/dsv41/store_test.cpp
@@ -0,0 +1,86 @@
+// Strata. ExpertStore (src/dsv41/ds4_store.hpp) on real expert files, no device: the bytes a slot holds equal the file's, the
+// unbuffered reads' rate, the policy's hits on a skewed random access stream.
+// ds4_store_test <gguf> <layer0 gate off> <layer0 up off> <layer0 down off> <expert bytes (gate/up/down)> [budget GB] [policy 0|1|2] [accesses]
+// (offsets from the GGUF's tensor directory: tools/dsv41 header scan; the three tensors of a layer are [384][...] so expert e of
+// a matrix lies at off + e * bytes)
+#include "ds4_store.hpp"
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <random>
+
+using namespace strata::ds4;
+
+int main(int argc, char** argv) {
+ if (argc < 6) {
+ std::printf("usage: ds4_store_test <gguf> <gate off> <up off> <down off> <matrix bytes> [budget GB] [policy] [accesses]\n");
+ return 2;
+ }
+ const std::string path = argv[1];
+ const uint64_t offs[3] = {std::strtoull(argv[2], nullptr, 10), std::strtoull(argv[3], nullptr, 10), std::strtoull(argv[4], nullptr, 10)};
+ const uint64_t mb = std::strtoull(argv[5], nullptr, 10);
+ const double gb = argc > 6 ? std::atof(argv[6]) : 1.0;
+ const int policy = argc > 7 ? std::atoi(argv[7]) : 0;
+ const int n_acc = argc > 8 ? std::atoi(argv[8]) : 2000;
+ const os::fd_t fd = os::open_ro(path), dfd = os::open_direct(path), pfd = os::open_par(path);
+ if (fd == os::kBadFd || pfd == os::kBadFd) {
+ std::printf("cannot open %s\n", path.c_str());
+ return 2;
+ }
+ std::printf("direct handle: %s\n", dfd != os::kBadFd ? "yes" : "NO (buffered)");
+ ExpertStore::LayerGeo geo;
+ geo.n_exp = 384;
+ for (int r = 0; r < 3; ++r) {
+ geo.file[r] = 0;
+ geo.off[r] = offs[r];
+ geo.stride[r] = mb;
+ geo.bytes[r] = mb;
+ }
+ ExpertStore st;
+ ExpertStore::Cfg cfg;
+ cfg.budget_bytes = (size_t) (gb * 1e9);
+ cfg.threads = std::getenv("STORE_TEST_THREADS") ? std::atoi(std::getenv("STORE_TEST_THREADS")) : 8;
+ cfg.policy = (ExpertStore::Policy) policy;
+ cfg.floor_bytes = 4ull << 30;
+ cfg.ram_avail = [] { return os::ram_avail(); };
+ if (!st.init(cfg, {geo}, {fd}, {dfd}, {pfd})) {
+ std::printf("init failed\n");
+ return 2;
+ }
+ std::printf("slot %.2f MB, up to %zu slots\n", st.slot_bytes() / 1e6, st.slots_max());
+ // a skewed stream: 70% of the accesses from the 64 hottest experts, the rest uniform; 6 a "token"
+ std::mt19937_64 rng(7);
+ auto pick = [&] { return rng() % 100 < 70 ? (int) (rng() % 64) : (int) (rng() % 384); };
+ std::vector<uint8_t> ref(mb);
+ int bad = 0;
+ long checked = 0;
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int a = 0; a < n_acc; ++a) {
+ int es[6];
+ for (int i = 0; i < 6; ++i) {
+ es[i] = pick();
+ for (int j = 0; j < i; ++j)
+ if (es[j] == es[i]) es[i] = (es[i] + 1 + j) % 384;
+ }
+ std::vector<char> ok;
+ st.ensure(0, es, 6, ok);
+ if (a % 17 == 0 && !std::getenv("STORE_TEST_NOCMP")) // a sample compared with the file read plainly
+ for (int i = 0; i < 6; ++i)
+ if (ok[(size_t) i])
+ for (int r = 0; r < 3; ++r) {
+ const int s = st.find(0, es[i]);
+ os::pread_full(fd, ref.data(), mb, offs[r] + (uint64_t) es[i] * mb);
+ ++checked;
+ if (s < 0 || std::memcmp(st.mat(s, r), ref.data(), mb) != 0) ++bad;
+ }
+ st.release(0, es, 6, ok);
+ }
+ const double dt = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+ const auto& s = st.stats();
+ std::printf("%d accesses of 6 experts in %.2f s: hits %lld misses %lld evictions %lld fails %lld; read %.2f GB (%.2f GB/s); waited %.2f s; "
+ "compared %ld matrices, %d differ\n",
+ n_acc, dt, (long long) s.hits.load(), (long long) s.misses.load(), (long long) s.evictions.load(), (long long) s.fails.load(),
+ s.bytes_read.load() / 1e9, s.bytes_read.load() / 1e9 / dt, s.wait_ns.load() / 1e9, checked, bad);
+ return bad == 0 ? 0 : 1;
+}
diff --git a/tools/dsv41/win/cachesim.py b/tools/dsv41/win/cachesim.py
index e89027823..b6c171afb 100644
--- a/tools/dsv41/win/cachesim.py
+++ b/tools/dsv41/win/cachesim.py
@@ -22,6 +22,9 @@ import struct
import sys
+WARM = 0 # accesses before this index warm the cache but are not counted
+
+
def read_trace(path, k=6):
"""-> list of steps; a step = list of (layer, [rows of expert ids], guesses or None)."""
data = open(path, 'rb').read()
@@ -85,15 +88,16 @@ class Sizes:
def sim_lru(acc, size, cap):
od = collections.OrderedDict()
used = hits = hitb = tot = totb = 0
- for _, l, e in acc:
+ for i, (_, l, e) in enumerate(acc):
k = (l, e)
sz = size(l)
- tot += 1
- totb += sz
+ cnt = i >= WARM
+ tot += cnt
+ totb += sz * cnt
if k in od:
od.move_to_end(k)
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
continue
od[k] = sz
used += sz
@@ -111,12 +115,13 @@ def sim_lfu(acc, size, cap, half_life_acc=20000):
for t, (_, l, e) in enumerate(acc):
k = (l, e)
sz = size(l)
- tot += 1
- totb += sz
+ cnt = t >= WARM
+ tot += cnt
+ totb += sz * cnt
# CRF: decayed count, kept in log space: v = log(count) + lam * t
if k in res:
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
old = score.get(k)
v = lam * t
score[k] = v if old is None else v + math.log1p(math.exp(old - v)) if old - v < 50 else old
@@ -153,13 +158,14 @@ def pick_static(freq, size, cap):
def sim_static(acc, size, chosen):
hits = hitb = tot = totb = 0
- for _, l, e in acc:
+ for i, (_, l, e) in enumerate(acc):
sz = size(l)
- tot += 1
- totb += sz
+ cnt = i >= WARM
+ tot += cnt
+ totb += sz * cnt
if (l, e) in chosen:
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
return hits, hitb, tot, totb
@@ -167,19 +173,20 @@ def sim_static_lru(acc, size, cap, chosen, pinned_bytes):
od = collections.OrderedDict()
used = hits = hitb = tot = totb = 0
room = cap - pinned_bytes
- for _, l, e in acc:
+ for i, (_, l, e) in enumerate(acc):
k = (l, e)
sz = size(l)
- tot += 1
- totb += sz
+ cnt = i >= WARM
+ tot += cnt
+ totb += sz * cnt
if k in chosen:
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
continue
if k in od:
od.move_to_end(k)
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
continue
od[k] = sz
used += sz
@@ -203,11 +210,12 @@ def sim_belady(acc, size, cap):
for i, (_, l, e) in enumerate(acc):
k = (l, e)
sz = size(l)
- tot += 1
- totb += sz
+ cnt = i >= WARM
+ tot += cnt
+ totb += sz * cnt
if k in res:
- hits += 1
- hitb += sz
+ hits += cnt
+ hitb += sz * cnt
res[k] = nxt[i]
heapq.heappush(heap, (-nxt[i], k))
continue
@@ -300,9 +308,11 @@ def main():
ap.add_argument('--sizes', default='C:/ds-ista-work/sizes.json')
ap.add_argument('--caps', default='20,30,40,44', help='cache sizes in GB (VRAM hot set + pinned RAM)')
ap.add_argument('--train', nargs='*', default=[], help='traces the static profile is built from (others than the test)')
+ ap.add_argument('--warm-steps', type=int, default=0, help='steps at the start that fill the cache but are not counted')
ap.add_argument('--skip', default='', help='policies to skip (comma): lru,lfu,static,static+lru,belady,ahead')
a = ap.parse_args()
size = Sizes(a.sizes)
+ global WARM
skip = set(a.skip.split(','))
train_acc = []
for t in a.train:
@@ -311,6 +321,12 @@ def main():
steps = read_trace(path)
acc = accesses(steps)
nsteps = len(steps)
+ WARM = 0
+ if a.warm_steps:
+ WARM = next((i for i, x in enumerate(acc) if x[0] >= a.warm_steps), len(acc))
+ nsteps -= a.warm_steps
+ uniq = len({(l, e) for _, l, e in acc})
+ print(' unique experts touched: %d of 15360 (%.1f%%), %.1f GB' % (uniq, 100 * uniq / 15360, sum(size(l) for l, e in {(l, e) for _, l, e in acc}) / 1e9))
print('== %s: %d steps, %d expert accesses (%.1f a step)' % (os.path.basename(path), nsteps, len(acc), len(acc) / max(1, nsteps)))
tb = sum(size(l) for _, l, _ in acc) / nsteps
print(' bytes a step (union per layer): %.2f GB' % (tb / 1e9))