Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

WIP not-default: DS4_NO_WAIT_SPLIT (no commit at the routing wait; A/B unfinished) and DS4_PHASES (decode host phase timers)

Niko12211b667c7abbe0
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 23499135c..c773eeed5 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -2561,7 +2561,8 @@ extern "C" void* ds4_ctx_mark() {
return g_cx->mark();
}
extern "C" void ds4_ctx_wait(void* token) {
- if (g_cx) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
+ static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;
+ if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
if (g_cx && token) g_cx->wait_mark(token);
}
// commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index e09028f78..4f605d077 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1,8815 +1,8826 @@
-// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
-// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
-// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).
-//
-// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):
-// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables
-// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)
-// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus
-// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the
-// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b
-// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])
-// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert
-// h = post * moe + comb^T h; pre = f_pre
-// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
-#include "ds4_model.hpp"
-#include "ds4_engram_io.hpp"
-
-#include "ds4_gguf.cuh"
-#include "ds4_gguf_cpu.hpp"
-#include "ds4_kernels.cuh"
-#include "ds4_os.hpp"
-#include "ds4_pool.hpp"
-#include "ds4_store.hpp"
-#include "ds4_vision.cuh"
-#include "strata/dsv41/exl3_cpu.hpp"
-#include "strata/dsv41/exl3.hpp"
-#include "strata/kernels/f16_bits.hpp"
-#include "strata/dsv41/moe_win.hpp"
-#include "strata/dsv41/sampler.hpp"
-
-#include <cublas_v2.h>
-#include <cuda_runtime.h>
-
-#if defined(STRATA_METAL)
-extern "C" void ds4_busy_begin();
-extern "C" void ds4_busy_end();
-#include <sys/sysctl.h>
-#include <mach/mach.h>
-#endif
-#include <algorithm>
-#include <array>
-#include <condition_variable>
-#include <deque>
-#include <atomic>
-#include <chrono>
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <fstream>
-#include <functional>
-#include <future>
-#include <map>
-#include <mutex>
-#include <numeric>
-#include <sstream>
-#include <thread>
-#include <unordered_map>
-
-#if defined(_MSC_VER)
-#define strtok_r strtok_s
-#endif
-
-namespace strata::ds4 {
-namespace x3 = strata::dsv41k::exl3;
-namespace kc = strata::dsv41k::cpu;
-namespace gf = strata::dsv41k::glmf;
-namespace gg = strata::ds4::gguf;
-
-#define CK(call) \
- do { \
- cudaError_t e_ = (call); \
- if (e_ != cudaSuccess) { \
- err = std::string(#call) + ": " + cudaGetErrorString(e_); \
- return false; \
- } \
- } while (0)
-
-namespace {
-
-double now_ms() {
- return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
-}
-
-size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }
-
-// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:
-// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):
-// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before
-void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {
- const int b = whole / 128, q = b / np, r = b % np;
- auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };
- int at = 0;
- for (int i = 0; i < p; ++i) at += blocks(i);
- off = 128 * at;
- n = 128 * blocks(p);
-}
-
-std::string read_file(const std::string& p) {
- std::ifstream f(p);
- std::stringstream ss;
- ss << f.rdbuf();
- return ss.str();
-}
-std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
- std::vector<int> v;
- std::stringstream ss(s);
- std::string part;
- while (std::getline(ss, part, ',')) {
- if (part.empty() || part == "\n") continue;
- const size_t dash = part.find('-');
- const int a = std::atoi(part.c_str());
- const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);
- for (int c = a; c <= b; ++c) v.push_back(c);
- }
- return v;
-}
-// each NUMA node's physical cores (the first logical CPU of each core), in order
-std::vector<std::vector<int>> node_cores() {
-#if defined(_WIN32)
- static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)
- const char* v = std::getenv("DS4_CPU_SMT");
- return v != nullptr && std::atoi(v) != 0;
- }();
- std::vector<std::vector<int>> nodes = os::node_cores(smt);
-#else
- std::vector<std::vector<int>> nodes;
- for (int n = 0; n < 64; ++n) {
- const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");
- if (cl.empty()) break;
- std::vector<int> cores;
- for (int c : cpu_list(cl)) {
- const std::vector<int> sib = cpu_list(
- read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));
- if (sib.empty() || sib[0] == c) cores.push_back(c);
- }
- if (!cores.empty()) nodes.push_back(cores);
- }
-#endif
- if (nodes.empty()) {
- std::vector<int> all;
- for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);
- nodes.push_back(all);
- }
- return nodes;
-}
-// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages
-void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }
-void pin_self(int cpu) { os::pin_self(cpu); }
-// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root
-// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known
-uint64_t ram_avail() {
-#if defined(_WIN32)
- return os::ram_avail();
-#endif
- uint64_t avail = ~uint64_t{0};
- {
- std::istringstream mi(read_file("/proc/meminfo"));
- std::string k;
- uint64_t kb = 0;
- while (mi >> k >> kb) {
- if (k == "MemAvailable:") avail = kb << 10;
- mi.ignore(64, '\n');
- }
- }
- std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."
- const size_t at = cg.find("0::");
- if (at == std::string::npos) return avail;
- cg = cg.substr(at + 3);
- cg = cg.substr(0, cg.find('\n'));
- for (;;) {
- const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);
- const std::string mx = read_file(dir + "/memory.max");
- if (!mx.empty() && mx.compare(0, 3, "max") != 0) {
- const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);
- const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);
- uint64_t file = 0;
- std::istringstream st(read_file(dir + "/memory.stat"));
- std::string k;
- uint64_t v = 0;
- while (st >> k >> v)
- if (k == "file") file = v;
- const uint64_t used = cur > file ? cur - file : 0;
- avail = std::min(avail, lim > used ? lim - used : 0);
- }
- if (cg.empty() || cg == "/") break;
- const size_t sl = cg.rfind('/');
- cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);
- }
- return avail;
-}
-
-float bf16_to_f32(uint16_t b) {
- uint32_t u = (uint32_t) b << 16;
- float f;
- std::memcpy(&f, &u, 4);
- return f;
-}
-float f16_to_f32(uint16_t h) {
- const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
- uint32_t u;
- if (e == 0) {
- if (m == 0) u = s;
- else { // subnormal
- int ee = -1;
- uint32_t mm = m;
- do {
- ++ee;
- mm <<= 1;
- } while ((mm & 0x400) == 0);
- u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;
- }
- } else if (e == 31) u = s | 0x7f800000u | m << 13;
- else u = s | (e - 15 + 127) << 23 | m << 13;
- float f;
- std::memcpy(&f, &u, 4);
- return f;
-}
-
-// ------------------------------------------------------------------ the pack
-struct Tensor {
- std::string dtype;
- int file = -1;
- uint64_t off = 0, bytes = 0;
- std::vector<int64_t> shape;
-};
-
-struct Pack {
- std::string dir, model_dir;
- std::unordered_map<std::string, std::string> kv;
- std::vector<std::string> files;
- std::vector<os::fd_t> fds;
- std::unordered_map<std::string, Tensor> t;
- int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;
- std::vector<int32_t> eg_map;
- std::vector<int64_t> eg_mult, eg_primes, eg_off;
- // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the
- // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)
- bool gguf = false;
-
- std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)
- std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)
- std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)
-
- ~Pack() {
- for (auto& m : maps) os::unmap(m.first, m.second);
- for (os::fd_t fd : fds) os::close_fd(fd);
- for (os::fd_t fd : dfds) os::close_fd(fd);
- for (os::fd_t fd : pfds) os::close_fd(fd);
- }
- // file i mapped whole, read-only (once; not thread-safe: called while loading)
- const uint8_t* map(int file) {
- if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});
- auto& m = maps[(size_t) file];
- if (m.first == nullptr) {
- const uint64_t size = os::file_size(fds[(size_t) file]);
- void* p = os::map_ro(fds[(size_t) file], size);
- if (p == nullptr) return nullptr;
- m = {(uint8_t*) p, (size_t) size};
- }
- return m.first;
- }
-
- bool load(const std::string& d, std::string& err) {
- dir = d;
- {
- std::ifstream f(d + "/ds4.txt");
- if (!f) {
- err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";
- return false;
- }
- std::string line;
- while (std::getline(f, line)) {
- const size_t sp = line.find(' ');
- if (sp == std::string::npos) continue;
- kv[line.substr(0, sp)] = line.substr(sp + 1);
- }
- }
- model_dir = str("model_dir");
- std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");
- if (f == nullptr) {
- err = d + "/tensors.txt: not found";
- return false;
- }
- std::vector<char> buf(1 << 16);
- while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
- char* s = buf.data();
- if (s[0] == 'f') { // file <i> <name>
- char name[4096];
- int i = 0;
- if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
- if ((int) files.size() <= i) files.resize((size_t) i + 1);
- files[(size_t) i] = name;
- }
- continue;
- }
- if (s[0] != 't') continue;
- char* save = nullptr;
- ::strtok_r(s, " \n", &save);
- const char* name = ::strtok_r(nullptr, " \n", &save);
- const char* dt = ::strtok_r(nullptr, " \n", &save);
- const char* fi = ::strtok_r(nullptr, " \n", &save);
- const char* off = ::strtok_r(nullptr, " \n", &save);
- const char* nb = ::strtok_r(nullptr, " \n", &save);
- if (name == nullptr || nb == nullptr) continue;
- Tensor x;
- x.dtype = dt;
- x.file = std::atoi(fi);
- x.off = std::strtoull(off, nullptr, 10);
- x.bytes = std::strtoull(nb, nullptr, 10);
- while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));
- t.emplace(name, std::move(x));
- }
- std::fclose(f);
- for (const std::string& fn : files) {
- const std::string p = model_dir + "/" + fn;
- const os::fd_t fd = os::open_ro(p);
- if (fd == os::kBadFd) {
- err = p + ": cannot open";
- return false;
- }
- fds.push_back(fd);
- }
- gguf = str("format") == "gguf";
- if (gguf)
- for (size_t i = 0; i < fds.size(); ++i) {
- if (map((int) i) == nullptr) {
- err = model_dir + "/" + files[i] + ": cannot map";
- return false;
- }
- dfds.push_back(os::open_direct(model_dir + "/" + files[i]));
- pfds.push_back(os::open_par(model_dir + "/" + files[i]));
- }
- // the Engram tables
- std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");
- if (e == nullptr) {
- err = d + "/engram.bin: not found";
- return false;
- }
- char magic[4];
- int32_t hdr[6];
- bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&
- std::fread(hdr, 4, 6, e) == 6;
- if (ok) {
- eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];
- const int cols = (eg_ngram - 1) * eg_heads;
- eg_map.resize((size_t) eg_vocab);
- eg_mult.resize((size_t) eg_layers * eg_ngram);
- eg_primes.resize((size_t) eg_layers * cols);
- eg_off.resize((size_t) eg_layers * cols);
- ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&
- std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&
- std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&
- std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();
- }
- std::fclose(e);
- if (!ok) {
- err = d + "/engram.bin: unreadable";
- return false;
- }
- return true;
- }
-
- std::string str(const std::string& k, const std::string& def = "") const {
- auto it = kv.find(k);
- return it == kv.end() ? def : it->second;
- }
- double num(const std::string& k, double def) const {
- auto it = kv.find(k);
- return it == kv.end() ? def : std::atof(it->second.c_str());
- }
- std::vector<int> list(const std::string& k) const {
- std::vector<int> v;
- std::stringstream ss(str(k));
- std::string x;
- while (std::getline(ss, x, ','))
- if (!x.empty()) v.push_back(std::atoi(x.c_str()));
- return v;
- }
- const Tensor* get(const std::string& n) const {
- auto it = t.find(n);
- return it == t.end() ? nullptr : &it->second;
- }
- bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)
- if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)
- return os::pread_par(pfds[(size_t) file], dst, n, off);
- return os::pread_full(fds[(size_t) file], dst, n, off);
- }
- bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }
- const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)
-};
-
-// ------------------------------------------------------------------ geometry
-struct Geo {
- int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,
- k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,
- cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;
- float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;
- double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;
- std::vector<int> ratios, kv_src, ix_src, eg_layers;
- // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks
- int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;
- std::vector<int> d_targets;
-
- void from(const Pack& p) {
- vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);
- nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);
- hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);
- q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);
- o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);
- k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);
- win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);
- sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);
- ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);
- cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);
- cand_blk = (int) p.num("candidate_block_size", 0);
- orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);
- eos = (int) p.num("eos_token_id", 1);
- eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);
- route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);
- rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);
- factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);
- beta_slow = p.num("rope_scaling.beta_slow", 1);
- ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");
- ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");
- d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);
- d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);
- d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");
- // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)
- n_mtp = d_block > 0 && (int) ratios.size() > nl &&
- (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)
- ? (int) ratios.size() - nl : 0;
- }
-};
-
-// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them
-std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {
- std::vector<float> f((size_t) dim / 2);
- for (int i = 0; i < dim / 2; ++i) {
- const float e = (float) (2 * i) / (float) dim;
- f[(size_t) i] = 1.0f / std::pow((float) base, e);
- }
- if (original > 0) {
- auto corr = [&](double rot) {
- return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)
- };
- const double low = std::max(std::floor(corr(beta_fast)), 0.0);
- const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);
- const float den = (float) std::max(high - low, 1e-3);
- for (int i = 0; i < dim / 2; ++i) {
- float ramp = ((float) i - (float) low) / den;
- ramp = std::min(1.0f, std::max(0.0f, ramp));
- const float smooth = 1.0f - ramp;
- f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;
- }
- }
- return f;
-}
-
-// ------------------------------------------------------------------ devices, layers
-struct Dev {
- int id = 0;
- cudaStream_t st = nullptr;
- cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)
- int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)
- float* rw_a = nullptr;
- size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;
- std::vector<void*> allocs;
- // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)
- float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,
- *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,
- *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,
- *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,
- *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,
- *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,
- *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,
- *ed = nullptr;
- float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)
- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;
- unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)
- uint8_t* eg_rows = nullptr;
- uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows
- AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)
- // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the
- // driver's staging, ~1.7 ms a copy): h | pre | topk | cand
- uint8_t* bounce = nullptr;
- cudaEvent_t ev_out = nullptr;
- float* tap = nullptr; // a window's stream means before a DSpark target layer
- // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's
- // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window
- // MoE's grouping scratch
- int grp = 0, mi = 0;
- unsigned ar_seq = 0;
- unsigned* xcnt = nullptr;
- // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
- // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
- x3::Mat head;
- float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;
- unsigned head_seq = 0;
- // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the
- // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)
- uint16_t *mk_share = nullptr, *mk_erow = nullptr;
- int* mk_ints = nullptr;
- int mk_v0 = 0, mk_vn = 0;
- uint8_t* hand = nullptr;
- unsigned* hand_flag = nullptr;
- unsigned long long* grp_ptr = nullptr;
- int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
-};
-
-// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of
-// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS
-struct Chunk {
- int C = 0, A = 0, AI = 0;
- float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,
- *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,
- *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,
- *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,
- *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;
- uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,
- *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;
- size_t w16_elems = 0;
- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;
- uint8_t* eg_rows = nullptr;
- int* bounds_pin = nullptr; // pinned: the sort's bounds, read back
- // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,
- // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)
- static constexpr int kParts = 4;
- uint8_t* ring = nullptr;
- size_t ring_stride = 0;
- int ring_n = 0;
- cudaStream_t cs = nullptr;
- cudaEvent_t filled[kParts] = {}, freed[kParts] = {};
- // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of
- // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,
- // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s
- uint8_t* stage = nullptr;
- size_t stage_stride = 0;
- std::unique_ptr<Pool> readers;
- cublasHandle_t blas = nullptr;
- // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each
- // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)
- uint8_t* hand[2] = {nullptr, nullptr};
- unsigned* hand_flag[2] = {nullptr, nullptr};
- unsigned* hand_ack = nullptr;
- unsigned arb_seq = 0; // the group's chunk all-reduces
- // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)
- uint16_t* emb_host[2] = {nullptr, nullptr};
- uint8_t* eg_host[2] = {nullptr, nullptr};
- cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};
- double stream_bytes = 0; // the experts streamed in, in all
- // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a
- // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the
- // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once
- // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io
- // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits
- // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot
- // i % ring_n; the compute takes the items as they land, in runs of what has.
- struct Item {
- int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)
- int mi = 0; // the device's member index in the layer's group
- bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA
- const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read
- const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)
- uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there
- int64_t after = -1; // (layer-major) the items up to this one read first
- std::atomic<int> left{0}; // parts still being read
- std::atomic<bool> bad{false};
- };
- bool flow = false;
- bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring
- int dev = -1; // the device (index into devs) this chunk state belongs to
- std::unique_ptr<IoQueue> io;
- std::deque<Item> items; // items[0] is item `base`; the indices below are absolute
- size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;
- uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)
- int S = 0; // staging slots, stage_stride apart
- std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)
- std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use
- int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
- size_t spec_begin = 0;
- std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
- int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
- int *bc_pin = nullptr, *bc_dev = nullptr;
- uint64_t n_batch = 0;
- double wait_ms = 0; // the compute waiting for a read
- uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;
- uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started
- // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through
- // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in
- // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for
- // its later chunks, so each is read once a run
- bool lm = false, lm_pinned = false;
- uint8_t* lm_state = nullptr;
- size_t lm_state_bytes = 0;
- uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side
- size_t lc_bytes = 0;
- int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all
- size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read
- std::vector<int> lc_of; // expert -> its slot (-1: none)
- std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it
- uint64_t it_lc = 0; // items taken from the layer cache
-};
-
-struct Layer {
- int il = 0, dev = 0, ratio = 0;
- // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the
- // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert
- // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)
- int grp = 0, nm = 1, mi = 0;
- std::vector<Layer> mates;
- x3::ExpertLayout hlay;
- size_t hblob = 0;
- int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)
- // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more
- // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their
- // attention adds nothing to the sum
- int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;
- // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's
- // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member
- int* vs_pin = nullptr;
- int* vs_dev = nullptr;
- std::vector<int> host_list;
- std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."
- int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)
- bool mtp = false; // a DSpark block (index stage = il - n_layers)
- bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;
- int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)
- int ik_src = -1; // the kv source whose index keys its indexer reads
- int eg = -1; // engram layer index, or -1
- x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;
- float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,
- *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,
- *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,
- *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)
- uint16_t* i_wproj = nullptr;
- float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)
- uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)
- float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)
- // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row
- uint8_t* snap_win = nullptr;
- float *snap_kv = nullptr, *snap_sc = nullptr;
- // routed experts
- x3::ExpertLayout lay;
- size_t blob = 0; // bytes of one expert's blob
- int n_vram = 0;
- bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's
- uint8_t* vram_base = nullptr;
- std::vector<int> slot; // expert -> VRAM slot or -1
- std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
- std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)
- // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),
- // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)
- struct Half {
- size_t trellis[3], suh[3], svh[3], bytes;
- int k[3], n[3];
- } hl;
- std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes
- // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it
- // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)
- int n_slots = 0, spare_vram = -1;
- std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};
- std::vector<float> heat;
- std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked
- bool swap_pending = false;
- unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes
- // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and
- // the window's raw rows (kv, then score) to replay the accepted ones
- uint8_t* ring_bak = nullptr;
- float *c_snap = nullptr, *c_rows = nullptr;
- unsigned long long* vram_tab = nullptr; // device [n_exp]
- std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe
- unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it
- std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
- std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
- std::vector<uint64_t> region_bytes;
- std::vector<int> region_file;
- size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region
- size_t piece_bytes[3][3];
- size_t dense_bytes = 0;
- // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +
- // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane
- int mx_file[3] = {-1, -1, -1};
- uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};
-};
-
-struct Src {
- int layer = 0, ratio = 1;
- int64_t cap = 0;
- // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)
- std::vector<uint8_t*> ckv, ik;
- uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices
- cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;
- // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows
- // (ckv rows, then ik rows at kWin * hd) and its flag
- bool remote = false;
- uint8_t* x_stage = nullptr;
- unsigned* x_flag = nullptr;
- // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements
- // [slot][device] (32 apart); readers: the devices outside the source's group that read it
- uint8_t* x_stage1 = nullptr;
- unsigned* x_flag1 = nullptr;
- unsigned* x_ack = nullptr;
- std::vector<int> readers;
- uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }
- unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }
-};
-
-// a host thread that enqueues one device's work (threads mode)
-struct Worker {
- std::thread th;
- std::mutex mu;
- std::condition_variable cv;
- std::function<void()> job;
- bool busy = false, quit = false;
- void start() {
- th = std::thread([this] {
- for (;;) {
- std::function<void()> j;
- {
- std::unique_lock<std::mutex> lk(mu);
- cv.wait(lk, [&] { return quit || job != nullptr; });
- if (quit) return;
- j = std::move(job);
- job = nullptr;
- }
- j();
- {
- std::lock_guard<std::mutex> lk(mu);
- busy = false;
- }
- cv.notify_all();
- }
- });
- }
- void submit(std::function<void()> f) {
- {
- std::lock_guard<std::mutex> lk(mu);
- job = std::move(f);
- busy = true;
- }
- cv.notify_all();
- }
- void wait() {
- std::unique_lock<std::mutex> lk(mu);
- cv.wait(lk, [&] { return !busy; });
- }
- bool idle() {
- std::lock_guard<std::mutex> lk(mu);
- return !busy;
- }
- void stop() {
- {
- std::lock_guard<std::mutex> lk(mu);
- quit = true;
- }
- cv.notify_all();
- if (th.joinable()) th.join();
- }
-};
-
-struct Group {
- std::vector<int> mem; // device indices, member order (member 0 the lead)
- ArGroup ar, arb; // a window's all-reduces, a chunk's
- // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the
- // lead's acknowledgements (host-mapped)
- float* lg_stage = nullptr;
- unsigned *lg_flag = nullptr, *lg_ack = nullptr;
-};
-
-constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers
-
-// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the
-// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate
-// blocks as each layer sees them, the window's hand-off seq
-struct WinPlan {
- bool par = false, async = false;
- unsigned win = 0;
- int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2
- int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)
- int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)
- unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)
- unsigned seq[kMaxL] = {};
- int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};
-};
-
-} // namespace
-
-// ------------------------------------------------------------------ the model
-struct Model::Impl {
- Options o;
- Pack pk;
- Geo g;
- std::vector<Dev> devs;
- std::vector<Layer> L;
- std::vector<Src> srcs;
- std::vector<Group> groups;
- std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device
- bool par = false; // threads mode (the windows: decode)
- bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked
- unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)
- // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows
- int chunk = 0, chunk_min = 512;
- static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)
- // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk
- // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers
- // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token
- static constexpr int kMaxChunk = 4 * kVlRows;
- std::vector<std::unique_ptr<Chunk>> chunks; // per device
- size_t eg_cap = kWin; // tokens h_eg holds
- int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
- bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
- std::vector<float> chunk_logits;
- std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
- std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
- std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
- // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
- // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took
- struct PathModel {
- bool on = false;
- double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)
- // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and
- // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)
- double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};
- int64_t kn[2][4] = {};
- double win_s = 0; // windows: seconds a window took, so far
- int64_t win_n = 0;
- int last_cls = 0, last_nw = 0; // the last pick's size class, its windows
- double m = 1; // the picks' spread, against the profile's
- int64_t n_m = 0;
- int last = -1; // the last pick: 1 the flow, 0 windows
- double last_pred = 0; // its prediction (s, before its k)
- } pm;
- static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)
- static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)
- std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through
- WinPlan wplan;
- int head_dev = 0;
- x3::Mat head;
- float* norm_w = nullptr; // on the head device
- // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the
- // confidence head (on the head device: the drafts' logits come from the trunk's head)
- x3::Mat d_main_proj;
- float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
- uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
- // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
- // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
- // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
- bool mk_split = false;
- uint16_t* h_mk_embed = nullptr;
- unsigned* mk_slots = nullptr;
- unsigned mk_seq = 0;
- int d_dev = 0; // the drafter's first block's device
- bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)
- float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps
- cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it
- float* d_confv = nullptr; // on the head device
- int* d_ints = nullptr; // on the head device: drafts in, picks out
- float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;
- // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for
- // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()
- // checks it is its own)
- strata::dsv41k::SpecList* d_specq = nullptr;
- std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);
- bool drafts_q = false;
- float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;
- int dq_top_k = 0;
- uint64_t dq_seed = 0, dq_counter = 0;
- // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the
- // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5
- // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the
- // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs
- // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)
- bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;
- float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;
- int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;
- std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;
- float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)
- uint16_t* h_dump_ring = nullptr;
-
- uint16_t* embed_pinned = nullptr;
- std::vector<float> freqs_win, freqs_comp;
- std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores
- std::vector<std::vector<int>> cores; // each node's physical cores
- int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)
- // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps
- // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)
- bool file_experts = false;
- uint64_t warm_bytes = 0;
- static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)
- uint8_t* file_part(const Layer& l, int e) const {
- return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];
- }
- // host side of the CPU lane, a window's rows
- float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;
- int* h_idx = nullptr;
- int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks
- uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales
- // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one
- // an Engram layer: asked for when the window's tokens are known, collected right before that layer)
- EgReader eg_io;
- EgRowCache eg_cache;
- std::mutex eg_cache_mu;
- std::vector<std::unique_ptr<EgBatch>> eg_win;
- // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1
- // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8
- // codes with their E8M0 scales in a tensor of their own (see eg_issue)
- std::vector<uint8_t> eg_fmt;
- std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
- uint16_t* h_emb = nullptr; // the window's embedding rows
- std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
- // the state
- int64_t pos = 0;
- std::vector<int32_t> toks, ctoks;
- struct Check { // the state at a position: every window ring, the compressors' partial groups
- int64_t pos = -1;
- std::vector<std::vector<uint8_t>> win;
- std::vector<std::vector<float>> comp;
- };
- Check ck[2]; // the end of the last prompt, and the position before it
- std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
- Stats st;
- int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
- // (plan_window's: the windows' plans are made in order)
- int n_logits = 0; // the last forward's logit rows
- // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread
- // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum
- Req* reqs = nullptr;
- Resp* resps = nullptr;
- unsigned seq_issued = 0;
- std::atomic<int> layer_of[kRing];
- std::thread service;
- std::atomic<bool> quit{false};
-#if defined(STRATA_METAL)
- // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the
- // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)
- bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;
-#else
- bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
-#endif
- // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
- // before the wait for the CPU lane - the same bits, no faster)
- bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
- int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this
- // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same
- // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files
- // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next
- // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the
- // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's
- // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).
- int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;
- int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;
- // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)
- bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;
- std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;
- // 8 asked for (prefetch)
- std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)
- std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for
- std::thread pf_th;
- std::mutex pf_mu;
- std::condition_variable pf_cv;
- std::vector<std::pair<const void*, size_t>> pf_q;
- // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster
- // on one V100 - more for a PC to try: each its staging buffers and
- // copy streams), switched between windows
- struct Swap {
- int layer = 0, in = -1, out = -1;
- int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM
- };
- static constexpr int kMaxSwaps = 8;
- // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay
- // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)
- static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }
- const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),
- c_decay = env_f("DS4_CACHE_DECAY", 0.9f);
- const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));
- // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the
- // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the
- // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the
- // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in
- // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4
- // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.
- struct RamSwap {
- int layer = 0, in = -1, out = -1, slot = -1;
- };
- std::vector<uint8_t*> ram_slots, ram_chunks;
- std::vector<size_t> ram_chunk_bytes;
- bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)
- size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers
- std::vector<int> ram_free;
- std::deque<RamSwap> ram_queue;
- std::vector<RamSwap> ram_ready;
- std::condition_variable ram_cv;
- std::thread ram_th;
- int ram_inflight = 0, ram_held = 0;
- bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;
- // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a
- // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route
- // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every
- // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the
- // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left
- // out still counts as used (the expert cache, the profile).
- const float disk_skip_w = [] {
- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");
- return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;
- }();
- const int disk_skip_from = [] {
- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
- return v != nullptr ? std::max(0, std::atoi(v)) : 0;
- }();
- // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
- std::unique_ptr<ExpertStore> store;
- uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token
- size_t zc_stride = 0;
- bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves
- bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- for (int r = 0; r < 3; ++r)
- if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;
- return true;
- }
- return !file_experts || os::in_ram(file_part(l, e), l.blob);
- }
- // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads
- // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between
- // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of
- // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'
- // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes
- // the RAM place of the one it takes in)
- // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private
- // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever
- // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the
- // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds
- // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the
- // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold
- // them (the log's WARNING, a NOTE and INFO for the server)
- double commit_kept = 0, pagefile_hint = 0;
- double commit_margin() const {
- const os::Commit c = os::commit();
- return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);
- }
- // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)
- double commit_room() const {
- const os::Commit c = os::commit();
- double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;
- if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);
- return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());
- }
- // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)
- bool commit_charged(const Dev& d) const {
- if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;
-#if defined(_WIN32)
- int v = 0;
- if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {
- (void) cudaGetLastError();
- return false;
- }
- return v != 0;
-#else
- (void) d;
- return false;
-#endif
- }
- bool copy_mode = false;
- std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts
- std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes
- std::vector<int> copy_spare; // layer -> its spare slot (-1: none)
- std::mutex sw_mu;
- std::condition_variable sw_cv;
- std::deque<Swap> sw_queue;
- std::vector<Swap> sw_ready;
- struct SwapBufs {
- uint8_t *a = nullptr, *b = nullptr, *h = nullptr;
- std::vector<cudaStream_t> st; // a copy stream a device
- };
- std::vector<SwapBufs> sw_bufs;
- std::vector<std::thread> cache_ths;
- int sw_inflight = 0;
- int64_t decay_tokens = 0;
- bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;
- std::vector<float> host_logits;
- std::vector<void*> host_allocs;
- std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;
- // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or
- // chunk path (tests: where the two part)
- std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;
- // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records
- // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)
- std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;
- std::vector<int> dump_s_layers = [] {
- std::vector<int> v;
- const char* e = std::getenv("DS4_DUMP_S_LAYERS");
- std::string str = e ? e : "0,1,2,3";
- size_t a = 0;
- while (a < str.size()) {
- size_t b = str.find(',', a);
- if (b == std::string::npos) b = str.size();
- if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));
- a = b + 1;
- }
- return v;
- }();
- // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),
- // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the
- // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias
- // [n_exp] (F32)
- std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;
- // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,
- // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]
- // (F16) and y rows [rows dim] (F32)
- int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;
- std::FILE* dump_y = [] {
- const char* v = std::getenv("DS4_DUMP_Y");
- const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;
- return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;
- }();
- std::mutex dump_mu;
- // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its
- // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU
- // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it
- bool prof_on = std::getenv("DS4_PROF") != nullptr;
- std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;
- struct ProfMark {
- const Dev* d;
- cudaEvent_t ev;
- int stage;
- };
- std::mutex prof_mu;
- std::vector<ProfMark> prof_marks;
- std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on
- bool prof_window = false; // (decode windows only: T == 1)
- // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
- // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
- bool prof_chunk = false;
- double cprof_ms[4] = {};
- void prof_mark(const Dev& d, int stage) {
- if (!prof_window && !prof_chunk) return;
- std::lock_guard<std::mutex> lk(prof_mu);
- int cur = 0;
- cudaGetDevice(&cur);
- cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")
- cudaEvent_t e = nullptr;
- std::vector<cudaEvent_t>& pool = prof_pool[d.id];
- if (pool.empty()) cudaEventCreate(&e);
- else {
- e = pool.back();
- pool.pop_back();
- }
- cudaEventRecord(e, d.st);
- cudaSetDevice(cur);
- prof_marks.push_back({&d, e, stage});
- }
- void prof_collect() {
- if (!prof_window && !prof_chunk) return;
- std::lock_guard<std::mutex> lk(prof_mu);
- std::vector<std::pair<const Dev*, cudaEvent_t>> last;
- for (const ProfMark& m : prof_marks) {
- auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
- if (it != last.end()) {
- float ms = 0.0f;
- cudaSetDevice(m.d->id);
- if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
- (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
- it->second = m.ev;
- } else {
- last.push_back({m.d, m.ev});
- }
- }
- for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);
- prof_marks.clear();
- if (!prof_chunk) ++st.prof_tokens;
- }
- void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
- std::vector<float> v((size_t) T * g.hc * g.dim);
- cudaSetDevice(d.id);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
- cudaDeviceSynchronize();
-#endif
- cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[3] = {il, p0, T};
- std::fwrite(hdr, 8, 3, dump_l);
- std::fwrite(v.data(), 4, v.size(), dump_l);
- std::fflush(dump_l);
- }
-
- bool dump_s_on(int il) const {
- if (dump_s == nullptr) return false;
- for (int x : dump_s_layers)
- if (x == il) return true;
- return false;
- }
- bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)
- static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();
- return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;
- }
- void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {
- if (!dump_s_on(il) || !dump_s_name(name)) return;
- std::vector<uint8_t> v(nbytes);
- cudaSetDevice(d.id);
-#if defined(STRATA_METAL)
- ds4_metal_flush();
- cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)
-#endif
- cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- dump_sub_host(name, il, p0, T, v.data(), nbytes);
- }
- void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {
- if (!dump_s_on(il) || !dump_s_name(name)) return;
- std::lock_guard<std::mutex> lk(dump_mu);
- char nm[16] = {};
- std::strncpy(nm, name, 15);
- const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};
- std::fwrite(nm, 1, 16, dump_s);
- std::fwrite(hdr, 8, 4, dump_s);
- std::fwrite(host, 1, nbytes, dump_s);
- std::fflush(dump_s);
- }
-
- ~Impl() {
- eg_io.stop(); // (its threads write into h_eg)
- for (auto& w : workers) w->stop();
- for (size_t i = 0; i < chunks.size(); ++i)
- if (chunks[i] && chunks[i]->blas) {
- cudaSetDevice(devs[i].id);
- cublasDestroy(chunks[i]->blas);
- }
- quit.store(true);
- if (service.joinable()) service.join();
- {
- std::lock_guard<std::mutex> lk(pf_mu);
- }
- pf_cv.notify_all();
- if (pf_th.joinable()) pf_th.join();
- {
- std::lock_guard<std::mutex> lk(sw_mu);
- }
- sw_cv.notify_all();
- ram_cv.notify_all();
- for (std::thread& t : cache_ths)
- if (t.joinable()) t.join();
- if (ram_th.joinable()) ram_th.join();
- for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- for (void* p : d.allocs) cudaFree(p);
- if (d.st) cudaStreamDestroy(d.st);
- if (d.ev) cudaEventDestroy(d.ev);
- }
- for (void* p : host_allocs) cudaFreeHost(p);
- for (Layer& l : L)
- for (uint8_t* b : l.host)
- if (b != nullptr) {
- (void) b; // host blobs live in one mapping per layer, unmapped below
- }
- for (auto& m : host_maps) {
- if (chunk > 0) cudaHostUnregister(m.first);
- os::anon_free(m.first, m.second);
- }
- }
- std::vector<std::pair<void*, size_t>> host_maps;
-
- template <class T>
- bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {
- void* v = nullptr;
- cudaSetDevice(d.id);
- const size_t b = align256(n * sizeof(T));
- if (cudaMalloc(&v, b) != cudaSuccess) {
- err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st
- // does not order against - a zeroing still queued there could wipe the weights just copied
- if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {
- cudaFree(v);
- err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- d.allocs.push_back(v);
- if (acct) *acct += b;
- else d.scratch += b;
- p = (T*) v;
- return true;
- }
- static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
- static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
- // bytes of a VRAM slot of l's experts (a member's part of each, in a group)
- static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }
- // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64
- // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share
- int head_members(int nm) const {
- int na = nm;
- while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;
- return na;
- }
- // host-mapped memory (the same address on the host and every device), zeroed
- template <class T>
- bool hmap(T*& p, size_t n, std::string& err) {
- void* v = nullptr;
- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";
- return false;
- }
- std::memset(v, 0, n * sizeof(T) + 64);
- host_allocs.push_back(v);
- p = (T*) v;
- return true;
- }
- template <class T>
- bool halloc(T*& p, size_t n, std::string& err) {
- void* v = nullptr;
- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {
- err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";
- return false;
- }
- host_allocs.push_back(v);
- p = (T*) v;
- return true;
- }
-
- // ---- weights
- bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {
- std::vector<uint8_t> tmp(t.bytes);
- if (!pk.read(t, tmp.data())) {
- err = "read failed";
- return false;
- }
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
- return true;
- }
- // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs
- bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,
- int np, bool cols) {
- if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {
- err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";
- return false;
- }
- int n = (int) t.shape[0], k = (int) t.shape[1];
- const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);
- std::vector<uint8_t> w((size_t) n * rb);
- if (t.bytes != w.size() || !pk.read(t, w.data())) {
- err = pre + ": read failed";
- return false;
- }
- if (np > 1) {
- const int whole = cols ? n : k, wd = whole / np;
- if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {
- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";
- return false;
- }
- std::vector<uint8_t> p;
- if (cols) {
- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
- n = wd;
- } else {
- const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;
- p.resize((size_t) n * pb);
- for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);
- k = wd;
- }
- w.swap(p);
- }
- uint8_t* dp = nullptr;
- if (!dalloc(d, dp, w.size(), err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
- m = x3::Mat{};
- m.fmt = gg::kFmtGG;
- m.K = ggt;
- m.raw = dp;
- m.k = k;
- m.n = n;
- m.ld = (int) gg::type_row_bytes(ggt, k);
- return true;
- }
- // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
- // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
- // standalone [n][k] matrix
- bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
- int np, bool cols) {
- const bool q8 = t.dtype == "Q8_0";
- // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)
- const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;
- if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);
- if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
- err = pre + ": a " + t.dtype + " tensor that is not a matrix";
- return false;
- }
- int n = (int) t.shape[0], k = (int) t.shape[1];
- // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)
- auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };
- std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));
- if (t.bytes != w.size() || !pk.read(t, w.data())) {
- err = pre + ": read failed";
- return false;
- }
- if (np > 1) {
- const int whole = cols ? n : k, wd = whole / np;
- if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {
- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +
- std::to_string(np) + " parts";
- return false;
- }
- const size_t rb = bytes_of((size_t) k);
- std::vector<uint8_t> p;
- if (cols) {
- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
- n = wd;
- } else {
- const size_t pb = bytes_of((size_t) wd);
- p.resize((size_t) n * pb);
- for (size_t r = 0; r < (size_t) n; ++r)
- std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);
- k = wd;
- }
- w.swap(p);
- }
- uint8_t* dp = nullptr;
- if (!dalloc(d, dp, w.size(), err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
- m = x3::Mat{};
- m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;
- m.raw = dp;
- m.k = k;
- m.n = n;
- m.ld = q8 ? (int) bytes_of((size_t) k) : k;
- return true;
- }
- // a Q8_0 tensor's values as F32 (host)
- static std::vector<float> q8_values(const uint8_t* q, size_t n) {
- std::vector<float> v(n);
- for (size_t b = 0; b < n / 32; ++b) {
- const uint8_t* p = q + b * gg::kQ8Bytes;
- const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));
- for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];
- }
- return v;
- }
- // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone
- // matrix of its own (the rotations' 128-blocks stay whole)
- // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix
- // splits evenly, its groups only as wide as that goes)
- bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
- bool cols = true, int rot = 0) {
- if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))
- return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
- const Tensor* tr = pk.get(pre + ".trellis");
- const Tensor* su = pk.get(pre + ".suh");
- const Tensor* sv = pk.get(pre + ".svh");
- if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {
- err = pre + ": not an EXL3 matrix in the pack";
- return false;
- }
- if (pk.get(pre + ".mul1") == nullptr) {
- err = pre + ": only the mul1 codebook is covered";
- return false;
- }
- m.k = (int) tr->shape[0] * 16;
- m.n = (int) tr->shape[1] * 16;
- m.K = (int) tr->shape[2] / 16;
- m.cb = 2;
- m.ld = 0;
- if (np > 1) {
- const int whole = cols ? m.n : m.k;
- int at = 0, w = 0;
- if (whole % 128 != 0 || whole / 128 < np) {
- err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +
- std::to_string(np) + " parts of 128-blocks";
- return false;
- }
- split128(whole, np, part, at, w, rot);
- std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);
- if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {
- err = pre + ": read failed";
- return false;
- }
- const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));
- std::vector<uint8_t> t1, u1, v1;
- if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)
- const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;
- t1.resize((size_t) (m.k / 16) * nth * tile);
- for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)
- std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);
- u1 = u0;
- v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));
- m.n = w;
- } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)
- const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;
- t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));
- u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));
- v1 = v0;
- m.k = w;
- }
- uint32_t* dt = nullptr;
- uint16_t *ds = nullptr, *dv = nullptr;
- if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||
- !dalloc(d, dv, v1.size() / 2, err, acct))
- return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));
- CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));
- CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));
- m.trellis = dt;
- m.suh = ds;
- m.svh = dv;
- return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);
- }
- uint32_t* dt = nullptr;
- uint16_t *ds = nullptr, *dv = nullptr;
- if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||
- !dalloc(d, dv, sv->bytes / 2, err, acct))
- return false;
- if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;
- m.trellis = dt;
- m.suh = ds;
- m.svh = dv;
- if (!x3::supported(m)) {
- err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +
- " not covered by the EXL3 kernels";
- return false;
- }
- return true;
- }
- // a float tensor (F32 / F16 / BF16) as F32 on the device
- bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {
- const Tensor* t = pk.get(name);
- if (t == nullptr) {
- err = name + ": not in the pack";
- return false;
- }
- std::vector<uint8_t> raw(t->bytes);
- if (!pk.read(*t, raw.data())) {
- err = name + ": read failed";
- return false;
- }
- size_t n = 0;
- std::vector<float> v;
- if (t->dtype == "F32") {
- n = t->bytes / 4;
- v.resize(n);
- std::memcpy(v.data(), raw.data(), t->bytes);
- } else if (t->dtype == "F16" || t->dtype == "BF16") {
- n = t->bytes / 2;
- v.resize(n);
- const uint16_t* s = (const uint16_t*) raw.data();
- for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);
- } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)
- n = t->bytes / gg::kQ8Bytes * 32;
- v = q8_values(raw.data(), n);
- } else {
- err = name + ": dtype " + t->dtype + " is not a float";
- return false;
- }
- if (want != 0 && n != want) {
- err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);
- return false;
- }
- if (!dalloc(d, p, n, err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));
- return true;
- }
- bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {
- const Tensor* t = pk.get(name);
- if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)
- const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);
- std::vector<uint8_t> raw(t->bytes);
- if (!pk.read(*t, raw.data())) {
- err = name + ": read failed";
- return false;
- }
- const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();
- std::vector<uint16_t> h(n);
- for (size_t i = 0; i < n; ++i) {
- float v;
- if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);
- else if (t->dtype == "Q8_0") v = q[i];
- else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);
- h[i] = strata::kernels::f16_from_f32(v);
- }
- if (!dalloc(d, p, n, err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));
- return true;
- }
- if (t == nullptr || t->dtype != "F16") {
- err = name + ": not an F16 tensor in the pack";
- return false;
- }
- if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;
- return upload(d, *t, p, err);
- }
-
- size_t mat_bytes(const std::string& pre) const {
- if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
- size_t b = 0;
- for (const char* s : {".trellis", ".suh", ".svh"})
- if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
- return b;
- }
- size_t tensor_bytes(const std::string& name) const {
- const Tensor* t = pk.get(name);
- return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
- }
-
- // ---- expert regions and blob layout
- // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every
- // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
- // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
- static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})
- if (dtype == gg::type_name(t)) return t;
- return -1;
- }
- bool expert_layout_mx(Layer& l, std::string& err) {
- const std::string P = l.name + "ffn.experts.";
- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;
- const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");
- const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;
- const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;
- const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;
- if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {
- err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +
- "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";
- return false;
- }
- l.mx_file[r] = t0->file;
- l.mx_off[r] = t0->off;
- l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;
- for (int e = 0; e < l.n_exp; ++e) {
- const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");
- if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||
- t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {
- err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";
- return false;
- }
- }
- l.lay.trellis[r] = off;
- l.lay.suh[r] = l.lay.svh[r] = off;
- l.lay.K[r] = ty;
- off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)
- }
- l.lay.fmt = gg::kFmtMXFP4;
- if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
- gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||
- gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {
- err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +
- " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";
- return false;
- }
- l.blob = off;
- // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where
- // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM
- for (int r = 0; r < 3; ++r)
- if (!gg::type_on_device(l.lay.K[r])) {
- l.cpu_only = true;
- std::fprintf(stderr,
- "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "
- "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",
- gg::type_name(l.lay.K[r]));
- break;
- }
- return true;
- }
- const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
- if (store) {
- const int s = store->find(l.il, e);
- if (s >= 0) return store->mat(s, r);
- }
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
- return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
- : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
- return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);
- }
- size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)
- return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);
- }
- // (copy_mode) matrix r's rows, and the rows of one NUMA part of it
- int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }
- size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }
- // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them
- const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {
- if (copy_mode && l.ram[(size_t) e] >= 0)
- return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
- return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- }
- // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)
- void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);
- }
- }
- void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);
- }
- }
- // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down
- // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back
- bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {
- if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);
- else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);
- return true;
- }
- return read_mx_file(l, e, blob);
- }
- bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)
- for (int r = 0; r < 3; ++r)
- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),
- blob + l.lay.trellis[r]))
- return false;
- return true;
- }
- void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);
- for (size_t row = 0; row < (size_t) g.dim; ++row)
- std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);
- }
- void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);
- for (size_t row = 0; row < (size_t) g.dim; ++row)
- std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);
- }
-
- bool expert_layout(Layer& l, std::string& err) {
- if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)
- return expert_layout_mx(l, err);
- const std::string P = l.name + "ffn.experts.";
- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
- const char* pieces[3] = {".trellis", ".suh", ".svh"};
- l.region_off.resize((size_t) l.n_exp);
- l.region_bytes.resize((size_t) l.n_exp);
- l.region_file.resize((size_t) l.n_exp);
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");
- if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {
- err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";
- return false;
- }
- l.lay.K[r] = (int) tr->shape[2] / 16;
- l.lay.trellis[r] = off;
- off += align256(tr->bytes);
- l.lay.suh[r] = off;
- off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);
- l.lay.svh[r] = off;
- off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);
- }
- l.lay.cb = 2;
- l.blob = off;
- for (int e = 0; e < l.n_exp; ++e) {
- const std::string E = P + std::to_string(e) + ".";
- uint64_t lo = UINT64_MAX, hi = 0;
- int file = -1;
- for (int r = 0; r < 3; ++r)
- for (int q = 0; q < 3; ++q) {
- const Tensor* t = pk.get(E + parts[r] + pieces[q]);
- if (t == nullptr) {
- err = E + parts[r] + pieces[q] + ": missing";
- return false;
- }
- if (file < 0) file = t->file;
- if (t->file != file) {
- err = E + ": split across files";
- return false;
- }
- lo = std::min(lo, t->off);
- hi = std::max(hi, t->off + t->bytes);
- if (e == 0) l.piece_bytes[r][q] = t->bytes;
- else if (t->bytes != l.piece_bytes[r][q]) {
- err = E + parts[r] + ": not the shape of expert 0's";
- return false;
- }
- }
- l.region_off[(size_t) e] = lo;
- l.region_bytes[(size_t) e] = hi - lo;
- l.region_file[(size_t) e] = file;
- for (int r = 0; r < 3; ++r)
- for (int q = 0; q < 3; ++q) {
- const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;
- if (e == 0) l.piece_src[r][q] = src;
- else if (src != l.piece_src[r][q]) {
- err = E + ": the pieces lie differently from expert 0's";
- return false;
- }
- }
- }
- return true;
- }
- // the region of expert e (read from the file) -> its blob
- void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};
- for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);
- }
- }
-
- // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)
- void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- const uint8_t* tr = region + l.piece_src[r][0];
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- (size_t) nth * tile);
- std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);
- std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
- }
- }
-
- // ---- loading
- bool load(const Options& opt, std::string& err) {
- o = opt;
- const double t0 = now_ms();
- if (!pk.load(o.pack, err)) return false;
- g.from(pk);
- if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {
- err = o.pack + "/ds4.txt: incomplete geometry";
- return false;
- }
- if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {
- err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +
- std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";
- return false;
- }
- freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);
- freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);
-
- // devices
- int ndev = 0;
- if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
- err = "no CUDA device";
- return false;
- }
- std::vector<int> ids = o.gpus;
- if (ids.empty())
- for (int i = 0; i < ndev; ++i) ids.push_back(i);
- devs.resize(ids.size());
- for (size_t i = 0; i < ids.size(); ++i) {
- Dev& d = devs[i];
- d.id = ids[i];
- CK(cudaSetDevice(d.id));
- CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));
- CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));
- if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {
- err = "exl3 scratch on CUDA" + std::to_string(d.id);
- return false;
- }
- CK(cudaMemGetInfo(&d.free0, &d.total));
- }
- // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each
- // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)
- int tp = o.tp;
- if (pk.gguf && tp > 1) {
- int blk = 32;
- for (const auto& kv : pk.t) {
- const std::string& n = kv.first;
- if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||
- n.compare(n.size() - 9, 9, "w2.weight") != 0)
- continue;
- const int ty = gg_type_of(kv.second.dtype);
- if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
- }
- while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||
- g.o_groups % tp != 0))
- --tp;
- if (tp != o.tp)
- std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "
- "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);
- }
- // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone
- // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the
- // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the
- // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down
- // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
- par = o.threads || tp > 1;
- prompt_par = par || o.tp > 1;
-#if defined(STRATA_METAL)
- if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path
-#endif
- const bool tp_all = tp > 1 && tp == (int) devs.size();
- for (size_t i = 0; i < devs.size();) {
- Group G;
- G.mem.push_back((int) i);
- size_t j = i + 1;
- while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))
- G.mem.push_back((int) j++);
- if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own
- for (int m : G.mem) {
- Group one;
- one.mem.push_back(m);
- groups.push_back(one);
- }
- } else groups.push_back(G);
- i = j;
- }
- for (size_t gi = 0; gi < groups.size(); ++gi)
- for (size_t m = 0; m < groups[gi].mem.size(); ++m) {
- devs[(size_t) groups[gi].mem[m]].grp = (int) gi;
- devs[(size_t) groups[gi].mem[m]].mi = (int) m;
- }
- // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the
- // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM
- // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place
- // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times
- if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM
- // after VRAM's are in, by --mmap-experts)
- else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;
- else {
- double exp_b = 0, other_b = 0, room = 0;
- for (const auto& kt : pk.t) {
- const std::string& n = kt.first;
- if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
- (!o.dspark && n.rfind("mtp.", 0) == 0))
- continue;
- (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
- }
- for (const Dev& d : devs)
- room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -
- (prompt_par ? 1.5 : 0.4) * (1 << 30)) *
- o.vram_experts;
- file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
- }
- chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;
- const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)
- if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
- if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
- if (chunk > 0) {
- chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));
- eg_cap = (size_t) chunk;
- tap_cap = std::max(kWin, 128);
- } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)
- std::fprintf(stderr,
- "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"
- " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"
- " 461 / 1010 in chunks of %d) - %s\n",
- kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,
- o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"
- : "leave --tp 1 out, or pass --threads on, for chunks");
- // the vision tower on the first GPU, before the split: the layers get what it leaves
- if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,
- if (o.vision_only) return false; // and the server is told pictures are off - info_lines)
- std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "
- "text only\n", err.c_str());
- vision_err = err.empty() ? "unknown reason" : err;
- err.clear();
- vt.release();
- img_tok = -1;
- }
- if (o.vision_only) return true;
-
- // layers: roles and the sources they read
- if (!o.dspark) g.n_mtp = 0;
- L.resize((size_t) (g.nl + g.n_mtp));
- int last_src = -1;
- for (int il = 0; il < g.nl + g.n_mtp; ++il) {
- Layer& l = L[(size_t) il];
- l.il = il;
- l.mtp = il >= g.nl;
- l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
- l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
- l.k_exp = l.mtp ? g.d_k : g.k_exp;
- l.ratio = g.ratios[(size_t) il];
- if (l.mtp) { // window attention only: no compressor, indexer or Engram
- if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
- err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +
- std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";
- return false;
- }
- if (!expert_layout(l, err)) return false;
- continue;
- }
- l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();
- l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();
- l.owns_k = l.kv_src;
- l.cand_src = il == g.cand_src;
- l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;
- if (l.kv_src) {
- Src s;
- s.layer = il;
- s.ratio = l.ratio;
- s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;
- srcs.push_back(s);
- last_src = (int) srcs.size() - 1;
- }
- if (l.ratio > 0) {
- if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {
- err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +
- " without a KV source of that ratio before it";
- return false;
- }
- l.src = last_src;
- l.ik_src = last_src;
- }
- auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);
- if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());
- if (!expert_layout(l, err)) return false;
- }
-
- // the split: each layer's dense weights + its experts, the head on the last GPU
- for (Layer& l : L) {
- const std::string& P = l.name;
- size_t b = 0;
- if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");
- for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",
- "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",
- "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})
- b += mat_bytes(P + m);
- for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));
- b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");
- b += (size_t) g.win * win_rb() + (2u << 20);
- l.dense_bytes = b;
- }
- const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);
- const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;
- const size_t kv_per_src_dev = 0; // counted after the split
- (void) kv_per_src_dev;
- std::vector<double> avail(groups.size(), 0.0);
- double avail_sum = 0;
- for (size_t gi = 0; gi < groups.size(); ++gi) {
- for (int m : groups[gi].mem)
- avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));
- avail_sum += avail[gi];
- }
- double need_sum = (double) head_b;
- for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;
- {
- // contiguous layer ranges, each group's share of the total by its free memory
- size_t d = 0;
- double cum = 0, target = avail[0] / avail_sum * need_sum;
- std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers
- for (Layer& l : L)
- if (l.mtp) order.push_back(&l);
- for (Layer& l : L)
- if (!l.mtp) order.push_back(&l);
- for (Layer* lp : order) {
- Layer& l = *lp;
- const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;
- if (cum + lb / 2 > target && d + 1 < groups.size()) {
- ++d;
- target += avail[d] / avail_sum * need_sum;
- }
- l.grp = (int) d;
- cum += lb;
- }
- // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its
- // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage
- // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all
- if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {
- const int G = (int) groups.size(), NL = g.nl;
- std::vector<double> cap(groups.size());
- for (int gi = 0; gi < G; ++gi) {
- const int nm = (int) groups[(size_t) gi].mem.size();
- double mn = 1e300;
- for (int m : groups[(size_t) gi].mem)
- mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -
- 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)
- cap[(size_t) gi] = mn * nm;
- }
- for (const Layer& l : L)
- if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;
- cap[(size_t) G - 1] -= (double) head_b;
- std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense
- for (int il = 0; il < NL; ++il) {
- pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;
- pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;
- }
- auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi
- const int nm = (int) groups[(size_t) gi].mem.size();
- const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);
- return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;
- };
- // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)
- std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));
- std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));
- for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);
- for (int gi = 1; gi < G; ++gi)
- for (int b = gi + 1; b <= NL; ++b)
- for (int a = gi; a < b; ++a) {
- const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));
- if (v < best[(size_t) gi][(size_t) b]) {
- best[(size_t) gi][(size_t) b] = v;
- cut[(size_t) gi][(size_t) b] = a;
- }
- }
- int b = NL;
- for (int gi = G - 1; gi >= 0; --gi) {
- const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;
- for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;
- b = a;
- }
- }
- // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)
- if (const char* sp = std::getenv("DS4_SPLIT")) {
- std::vector<int> n;
- for (const char* q = sp; *q;) {
- n.push_back(std::atoi(q));
- while (*q && *q != ',') ++q;
- if (*q == ',') ++q;
- }
- int il = 0;
- for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)
- for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;
- for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;
- }
- for (Layer& l : L) {
- l.dev = groups[(size_t) l.grp].mem[0];
- l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)
- }
- head_dev = L[(size_t) g.nl - 1].dev;
- d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;
- }
- // the groups' members: copies of each layer's device state, a member's part of an expert
- for (Layer& l : L) {
- l.ff0 = 0;
- l.ffn = g.ff;
- l.ah0 = l.og0 = 0;
- l.ahn = g.heads;
- l.ogn = g.o_groups;
- if (l.nm == 1) continue;
- const int na = head_members(l.nm);
- if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {
- err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";
- return false;
- }
- for (int m = 1; m < l.nm; ++m) {
- Layer c = l;
- c.mates.clear();
- c.mi = m;
- c.dev = groups[(size_t) l.grp].mem[(size_t) m];
- l.mates.push_back(std::move(c));
- }
- // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even
- // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its
- // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)
- const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- const int ha = m - (l.nm - na); // (its place among the members with heads)
- x.ahn = ha < 0 ? 0 : g.heads / na;
- x.ah0 = ha < 0 ? 0 : ha * x.ahn;
- x.ogn = ha < 0 ? 0 : g.o_groups / na;
- x.og0 = ha < 0 ? 0 : ha * x.ogn;
- if (gguf_x) {
- x.ffn = g.ff / l.nm;
- x.ff0 = m * x.ffn;
- } else
- split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;
- x.hlay.K[r] = l.lay.K[r];
- x.hlay.trellis[r] = off;
- if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks
- const int ty = gg::expert_type(l.lay, r);
- if (k % gg::type_block(ty) != 0) {
- err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +
- " blocks do not split";
- return false;
- }
- x.hlay.suh[r] = x.hlay.svh[r] = off;
- off += align256((size_t) n * gg::type_row_bytes(ty, k));
- continue;
- }
- off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));
- x.hlay.suh[r] = off;
- off += align256((size_t) k * 2);
- x.hlay.svh[r] = off;
- off += align256((size_t) n * 2);
- }
- x.hlay.cb = l.lay.cb;
- x.hlay.fmt = l.lay.fmt;
- x.hblob = off;
- }
- }
- // which devices hold each source's compressed KV / index keys
- for (Src& s : srcs) {
- s.ckv.assign(devs.size(), nullptr);
- s.ik.assign(devs.size(), nullptr);
- }
- for (const Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m) {
- const Layer& l = member(l0, m);
- if (l.src >= 0) {
- srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;
- if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;
- }
- if (l.ix_src && l.ik_src >= 0) {
- srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;
- if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;
- }
- }
-
- // per device: the scratch, the caches, the dense weights
- for (size_t di = 0; di < devs.size(); ++di) {
- Dev& d = devs[di];
- const int D = g.dim, H = g.hc;
- const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;
- bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&
- dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&
- dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&
- dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&
- dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&
- dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&
- dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&
- dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&
- dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&
- dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&
- dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&
- dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&
- dalloc(d, d.rw, W * kPicks, err) &&
- dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&
- dalloc(d, d.moe_out, W * D, err) &&
- dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&
- dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&
- dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&
- dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&
- dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&
- dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
- dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
- dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
- if (ok && std::getenv("DS4_DBG_DEVH"))
- std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);
- if (!ok) return false;
- d.attn_scr.max_t = kAttnScratchT;
- d.attn_scr.cap = (int) topk_stride();
- if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;
- if (!g.eg_layers.empty()) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
- ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&
- dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);
- if (!ok) return false;
- }
- if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
- if (!halloc(d.bounce, bounce_bytes(), err)) return false;
- if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
- !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
- !dalloc(d, d.n_groups, 4, err))
- return false;
- if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;
- if (g.n_mtp > 0 && (int) di == head_dev &&
- (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))
- return false;
- CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));
- cudaSetDevice(d.id);
- CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));
- CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));
- for (Src& s : srcs) {
- if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)
- if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))
- return false;
- CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));
- CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));
- CK(cudaSetDevice(d.id));
- }
- if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))
- return false;
- if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))
- return false;
- }
- }
- for (Src& sr : srcs) {
- if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))
- return false;
- if (sr.remote && chunk > 0 &&
- (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||
- !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))
- return false;
- const int sg = L[(size_t) sr.layer].grp;
- for (size_t di = 0; di < devs.size(); ++di)
- if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);
- }
- for (Group& G : groups) {
- G.ar.nm = (int) G.mem.size();
- G.ar.slots = 4;
- G.ar.stride = (size_t) kWin * g.dim;
- if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||
- !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))
- return false;
- // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)
- if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||
- !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))
- return false;
- if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&
- (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||
- !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))
- return false;
- G.arb = G.ar; // (a chunk's: two slots of chunk rows)
- G.arb.slots = 2;
- G.arb.stride = (size_t) chunk * g.dim;
- if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||
- !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))
- return false;
- if (chunk > 0 && G.arb.nm > 2 &&
- (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||
- !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))
- return false;
- }
- if (g.n_mtp > 0) {
- if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;
- CK(cudaSetDevice(devs[(size_t) d_dev].id));
- CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));
- CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));
- }
- // the dense weights, each device's layers by a thread of its own
- {
- std::vector<std::string> errs(devs.size());
- std::vector<std::thread> th;
- for (size_t di = 0; di < devs.size(); ++di)
- th.emplace_back([this, di, &errs] {
- std::string e;
- if (!load_dense((int) di, e)) errs[di] = e;
- });
- for (auto& t : th) t.join();
- for (auto& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- }
- if (o.verbose)
- std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);
- // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E
- // host-mapped, the steps' slots
- // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)
- bool mk_rows = g.n_mtp > 0 && head_nm() > 1;
- for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;
- if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {
- const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");
- if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {
- err = "the markov embed: not a BF16 [vocab][rank] table";
- return false;
- }
- if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||
- !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {
- if (err.empty()) err = "the markov embed: read failed";
- return false;
- }
- devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)
- mk_split = true;
- }
-
- // the embedding table, pinned on the host
- {
- const Tensor* t = pk.get("embed.weight");
- const size_t nv = (size_t) g.vocab * g.dim;
- const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
- // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):
- // each row through ggml's to_float, to the same BF16 the host table holds
- const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;
- const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;
- const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;
- if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {
- err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";
- return false;
- }
- if (!halloc(embed_pinned, nv, err)) return false;
- if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)
- std::vector<uint8_t> row(erb);
- for (size_t r = 0; r < (size_t) g.vocab; ++r) {
- if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {
- err = "embed.weight: read failed";
- return false;
- }
- if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {
- err = "embed.weight: ggml has no decode for " + t->dtype;
- return false;
- }
- }
- } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
- std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
- for (size_t r = 0; r < (size_t) g.vocab; ++r) {
- if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
- err = "embed.weight: read failed";
- return false;
- }
- const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);
- for (size_t c = 0; c < (size_t) g.dim; ++c) {
- uint32_t u;
- std::memcpy(&u, &v[c], 4);
- u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even
- embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);
- }
- }
- } else if (!pk.read(*t, embed_pinned)) {
- err = "embed.weight: read failed";
- return false;
- }
- if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,
- "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",
- (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],
- (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],
- (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);
-
- }
- // the host side of the CPU lane and the Engram rows
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
- if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||
- !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||
- !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||
- !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))
- return false;
- if (pk.eg_layers > 0) {
- // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);
- // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
- const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
- const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
- int nth = (int) std::min(16u, std::max(4u, hw / 2));
- if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
- const uint64_t avail = ram_avail();
- uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
- if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);
- eg_cache.init((size_t) ((mb << 20) / row), row);
- std::vector<std::string> paths;
- for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
- eg_io.start(paths, nth);
- for (int li = 0; li < pk.eg_layers; ++li) {
- eg_win.emplace_back(new EgBatch());
- const std::string id = std::to_string(g.eg_layers[(size_t) li]);
- const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");
- // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read
- // as some other format's bytes - the old code then read the scale tensor of a pack that has none and
- // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.
- int fmt;
- if (tw == nullptr) {
- std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());
- std::exit(2);
- } else if (tw->dtype == "MXFP8") {
- fmt = 0;
- } else if (tw->dtype == "Q6_K") {
- fmt = 1;
- } else if (tw->dtype == "IQ4_NL") {
- fmt = 2;
- } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {
- fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)
- } else {
- std::fprintf(stderr,
- "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "
- "(MXFP8, Q6_K, IQ4_NL)\n",
- id.c_str(), tw->dtype.c_str());
- std::exit(2);
- }
- if (fmt == 1 && pk.eg_hd % 256 != 0) {
- err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +
- " values (not whole superblocks of 256)";
- return false;
- }
- if (fmt == 2 && pk.eg_hd % 32 != 0) {
- err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +
- " values (not whole blocks of 32)";
- return false;
- }
- eg_fmt.push_back((uint8_t) fmt);
- }
- if (o.verbose)
- std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
- "their layer; row cache %llu MB (%zu rows)\n",
- nth, (unsigned long long) mb, eg_cache.capacity());
- }
- prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);
- if (!o.profile.empty()) {
- std::FILE* f = std::fopen(o.profile.c_str(), "rb");
- if (f != nullptr) {
- std::vector<uint32_t> p(prof.size());
- const size_t n = std::fread(p.data(), 4, p.size(), f);
- if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());
- std::fclose(f);
- }
- }
-
- // the chunked prompt path's buffers (before the experts: they take what is left)
- if (chunk > 0 && !alloc_chunks(err)) return false;
- // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM
- // now leaves over
- if (file_experts) {
- double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- size_t fr = 0, tot = 0;
- CK(cudaMemGetInfo(&fr, &tot));
- room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;
- }
- for (const Layer& l : L) all += (double) l.blob * l.n_exp;
- const double need = std::max(0.0, all - room);
- const uint64_t avail = ram_avail();
- warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
- if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
- if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)
- std::fprintf(stderr,
- "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
- "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
- need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"
- : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",
- avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);
- }
- // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in
- // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)
- cores = node_cores();
- NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());
- if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;
- if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)
- {
- const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;
- c_xh.resize((size_t) NH * S * 2 * E);
- c_xsb.resize((size_t) NH * S * 2 * (E / 128));
- c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));
- c_xhd.resize(S * FF);
- c_xsd.resize(S * (FF / 128));
- c_accd.resize((size_t) NH * S * DQ * (E / NH));
- c_yd.resize(S * E);
- }
- // the experts: VRAM slots in what each device has left, the rest on the host
- if (!place_experts(err)) return false;
- if (!load_experts(err)) return false;
- if (pk.gguf) make_pools();
- if (chunk > 0) {
- // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
- const double tr = now_ms();
- for (auto& m : host_maps)
- if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {
- err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";
- return false;
- }
- if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);
- if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)
- size_t mr = 0;
- for (const Layer& l : L)
- if (!l.mtp) {
- for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);
- if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)
- mr = std::max(mr, l.blob);
- for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));
- }
- }
- const char* nr = std::getenv("DS4_READERS");
- const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);
- const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;
- // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the
- // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the
- // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)
- const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;
- for (size_t di = 0; di < chunks.size(); ++di) {
- auto& cp = chunks[di];
- if (cp == nullptr || cp->ring == nullptr) continue;
- cp->stage_stride = (mr + 4095) & ~(size_t) 4095;
- // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)
- bool fl = flow_on;
- int n_exp_max = 0;
- for (const Layer& l : L)
- if (!l.mtp)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di) {
- // (a group's members: each its share of the down's blocks in every row)
- fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&
- (l.nm == 1 || (flow_tp && (g.ff / l.nm) %
- std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));
- n_exp_max = std::max(n_exp_max, l.n_exp);
- }
- if (fl) // (each part read unbuffered from its offset rounded down: a page more each)
- cp->stage_stride = std::max(cp->stage_stride, flow_stride());
- cp->S = cp->ring_n;
- if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;
- cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);
- if (!fl) continue;
- cudaSetDevice(devs[di].id);
- cp->flow = true;
-#if defined(STRATA_METAL)
- cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;
-#endif
- cp->dev = (int) di;
- cp->io = std::make_unique<IoQueue>(n_readers);
- cp->stage_item.assign((size_t) cp->S, -1);
- cp->stage_ev.resize((size_t) cp->S);
- cp->ring_fill.resize((size_t) cp->ring_n);
- cp->ring_free.resize((size_t) cp->ring_n);
- for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})
- for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
- // (a region is written again only bc_n batches later - more than a layer's items, so after that
- // layer's bounds wait, which every copy from it precedes)
- cp->bc_n = n_exp_max + 2;
- cp->bc_w = cp->ring_n + 1;
- if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||
- !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))
- return false;
- }
- // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one
- // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each
- // window, a page fault at a time) - DS4_CHUNK_MIN still overrides
- bool all_flow = true;
- for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);
- if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;
- }
- for (Layer& l : L) {
- if (l.mtp) continue;
- for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)
- Layer& x = member(l, m);
- if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;
- if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;
- }
- }
- }
- {
- void* m = nullptr;
- if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory for the decode path's requests";
- return false;
- }
- host_allocs.push_back(m);
- reqs = (Req*) m;
- if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory for the decode path's answers";
- return false;
- }
- host_allocs.push_back(m);
- resps = (Resp*) m;
- std::memset((void*) reqs, 0, sizeof(Req) * kRing);
- std::memset((void*) resps, 0, sizeof(Resp) * kRing);
- for (auto& x : layer_of) x.store(-1);
- ahead_fl.assign(L.size() * 1024, 0);
- ahead_rank.assign(L.size() * 1024, 0);
- ahead_for.assign(L.size(), 0);
- const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;
- service = std::thread([this, core] {
- if (core >= 0) pin_self(core);
- serve_lane();
- });
- if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });
- }
- if (o.expert_cache) {
- size_t maxb = 0;
- for (const Layer& l : L) maxb = std::max(maxb, l.blob);
- const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));
- sw_bufs.resize((size_t) nt);
- for (SwapBufs& B : sw_bufs) {
- if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStream_t cs = nullptr;
- CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
- B.st.push_back(cs);
- }
- }
- for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- CK(cudaStreamSynchronize(d.st));
- }
- if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)
- for (size_t di = 0; di < devs.size(); ++di) {
- workers.push_back(std::make_unique<Worker>());
- workers.back()->start();
- }
- calibrate_paths();
- init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)
- preload_store();
- if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
- reset();
- return true;
- }
-
-
- // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed
- // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)
- size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }
- size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
- size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
-
- // ---- the chunked prompt path
- size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
- size_t chunk_hand_bytes() const {
- return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
- }
-
- bool alloc_chunks(std::string& err) {
- chunks.resize(devs.size());
- const int D = g.dim, H = g.hc, HD = g.hd;
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- for (size_t di = 0; di < devs.size(); ++di) {
- Dev& d = devs[di];
- cudaSetDevice(d.id);
- chunks[di] = std::make_unique<Chunk>();
- Chunk& c = *chunks[di];
- const int nm = (int) groups[(size_t) d.grp].mem.size();
- const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);
- int FF = 0, ff0_ = 0;
- split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
- const size_t C = (size_t) chunk;
- c.C = chunk;
- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
- // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
- c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
- c.w16_elems = (size_t) 8 << 20;
- bool has_eg = false, has_first = false;
- size_t stride = 0;
- for (const Layer& l : L)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di && !l.mtp) {
- has_eg = has_eg || l.eg >= 0;
- has_first = has_first || l.il == 0;
- stride = std::max(stride, align256(unit(member(l, m))));
- }
- const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;
- bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&
- dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&
- dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&
- dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&
- dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&
- dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&
- dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&
- dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&
- dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&
- dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&
- dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&
- dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&
- dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&
- dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&
- dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&
- dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
- dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
- dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
- dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
- halloc(c.bounds_pin, 2048, err);
- if (!ok) return false;
- if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
- if (has_eg) {
- const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);
- if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||
- !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))
- return false;
- }
- if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches
- const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;
- c.ring_stride = stride;
- c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);
- if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;
- }
- CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));
- for (int i = 0; i < Chunk::kParts; ++i) {
- CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));
- }
- if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {
- err = "cuBLAS on CUDA" + std::to_string(d.id);
- return false;
- }
- for (int sl = 0; sl < 2; ++sl) {
- if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;
- if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;
- if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;
- CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));
- }
- if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;
- }
- return true;
- }
-
- // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS
- bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
- std::string& err) {
- if (rows <= 0) return true;
- const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
- for (int c0 = 0; c0 < m.n; c0 += cols) {
- const int nc = std::min(cols, m.n - c0);
- if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
- else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
- const float one = 1.0f;
- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
- m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,
- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
- if (st != CUBLAS_STATUS_SUCCESS) {
- err = "cublasGemmEx: status " + std::to_string((int) st);
- return false;
- }
- }
- return true;
- }
- // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)
- bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,
- float* y, int ldy, std::string& err) {
- const float zero = 0.0f;
- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,
- x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT_TENSOR_OP);
- if (st != CUBLAS_STATUS_SUCCESS) {
- err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);
- return false;
- }
- return true;
- }
- // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale
- bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {
- const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
-#if defined(STRATA_METAL)
- if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass
- hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);
- return true;
- }
-#endif
- const float one = 1.0f, zero = 0.0f;
- if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=
- CUBLAS_STATUS_SUCCESS) {
- err = "cublasSgemm (mHC mixes)";
- return false;
- }
- hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);
- return true;
- }
-
- // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full
- // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file
- // region read into pinned staging)
- bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,
- const uint8_t* src = nullptr) {
- if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into
- // pinned staging as its blob) or from the file's mapping
- auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };
- bool ok = true;
- if (l.nm == 1) {
- if (src != nullptr)
- ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- else
- for (int r = 0; r < 3; ++r)
- ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),
- cudaMemcpyHostToDevice, cs) == cudaSuccess;
- } else {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,
- (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,
- (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- }
- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- return ok;
- }
- const auto& parts = l.hh[(size_t) e];
- const int D = g.dim, FF = g.ff;
- auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {
- return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- };
- auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {
- return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- };
- bool ok = true;
- if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets
- const uint8_t* p = src != nullptr ? src : parts[0];
- if (l.nm == 1) {
- for (int r = 0; r < 3; ++r)
- ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&
- cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&
- cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);
- } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;
- ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,
- nt * tile, nth * tile, (size_t) D / 16) &&
- cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&
- cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];
- ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,
- (size_t) (fn / 16) * row) &&
- cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&
- cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);
- }
- } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh
- for (int X = 0; X < 2; ++X) {
- ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,
- (size_t) (k / 16));
- ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);
- }
- ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
- }
- } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of
- // both parts' columns
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)
- for (int X = 0; X < 2; ++X) {
- const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);
- if (a >= b) continue;
- ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,
- (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&
- cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,
- (size_t) (b - a) * 2);
- }
- ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);
- }
- const size_t tile = (size_t) 32 * l.lay.K[2];
- const int nh = D / 2;
- for (int X = 0; X < 2; ++X) {
- ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,
- (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));
- ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);
- }
- ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- return ok;
- }
-
- // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk
- // in layer order; the copies run at most kParts batches ahead of the compute
- struct Batch {
- int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range
- };
- struct Streamer {
- std::vector<Batch> b;
- size_t cp = 0, kp = 0; // the next batch to copy, to compute
- std::atomic<size_t> done{0}; // cp, for read_ahead's thread
- };
- // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel
- // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -
- // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone
- void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {
- static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *
- 1048576.0;
- std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied
- double ahead = 0;
- for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {
- const size_t done = S.done.load(std::memory_order_relaxed);
- while (!issued.empty() && issued.front().first < done) {
- ahead -= issued.front().second;
- issued.pop_front();
- }
- if (ahead > cap) {
- std::this_thread::sleep_for(std::chrono::microseconds(500));
- continue;
- }
- const Batch& bt = S.b[q];
- const Layer& l0 = L[(size_t) bt.il];
- double nb = 0;
- if (l0.dev == di)
- for (int i = 0; i < bt.n; ++i) {
- const int e = l0.host_list[(size_t) (bt.j0 + i)];
- if (l0.lay.fmt == gg::kFmtMXFP4) {
- if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);
- nb += (double) l0.blob;
- continue;
- }
- os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);
- nb += (double) l0.region_bytes[(size_t) e];
- }
- issued.push_back({q++, nb});
- ahead += nb;
- }
- }
- bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {
- const int P = Chunk::kParts;
- while (S.cp < S.b.size() && S.cp < S.kp + P) {
- const Batch& bt = S.b[S.cp];
- const int half = (int) (S.cp % P);
- const Layer& l0 = L[(size_t) bt.il];
- int mi = 0;
- for (int m = 0; m < l0.nm; ++m)
- if (member(l0, m).dev == di) mi = m;
- if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));
- uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots
- if (c.stage != nullptr) {
- if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)
- sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;
- std::atomic<bool> bad{false};
- c.readers->run(bt.n, [&](int i) {
- const int e = l0.host_list[(size_t) (bt.j0 + i)];
- if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)
- : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],
- l0.region_bytes[(size_t) e],
- sb + (size_t) i * c.stage_stride))
- bad.store(true);
- });
- if (bad.load()) {
- err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";
- return false;
- }
- }
- for (int i = 0; i < bt.n; ++i) {
- uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;
- if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,
- sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))
- return false;
- c.stream_bytes += (double) unit(member(l0, mi));
- }
- CK(cudaEventRecord(c.filled[half], c.cs));
- ++S.cp;
- S.done.store(S.cp, std::memory_order_relaxed);
- }
- return true;
- }
-
- // ---- the flow (see Chunk)
- // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute
- // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing
- static void kick(cudaStream_t s) {
-#if defined(_WIN32) && defined(STRATA_USE_HIP)
- (void) cudaStreamQuery(s);
- (void) cudaGetLastError();
-#else
- (void) s;
-#endif
- }
- size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
- return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
- }
- size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }
- static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }
- // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from
- // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of
- // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole
- size_t mx_read_off(const Layer& l, int m, int r) const {
- return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : 0;
- }
- size_t mx_read_len(const Layer& l, int r) const {
- return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : mx_part_bytes(l, r);
- }
- // the device's member index in L0's group (0 alone)
- int flow_mi(const Chunk& c, const Layer& L0) const {
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == c.dev) return m;
- return 0;
- }
- void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {
- c.items.emplace_back();
- Chunk::Item& it = c.items.back();
- it.il = l.il;
- it.mi = mi;
- it.k = k;
- it.e = e;
- it.v = v;
- bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;
- const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)
- if (!in_ram && store && l.nm == 1) {
- const int ss = store->find(l.il, e);
- if (ss >= 0) {
- store_blob = store->blob(ss);
- in_ram = true;
- }
- }
- if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)
- const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];
- for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);
- ++c.it_ram;
- return;
- }
- if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)
- it.ram = store_blob != nullptr ? store_blob
- : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];
- bool hit = false;
- if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after
- const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;
- if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again
- c.lc_il = l.il; // once that layer's items are all read - `after` below)
- c.lc_used = 0;
- c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);
- c.lc_first = i;
- std::fill(c.lc_of.begin(), c.lc_of.end(), -1);
- }
- const int j = c.lc_of[(size_t) e];
- if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)
- it.ram = c.lc + (size_t) j * stride;
- it.after = c.lc_writer[(size_t) j];
- hit = true;
- } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)
- const int jn = c.lc_used++;
- c.lc_of[(size_t) e] = jn;
- c.lc_writer[(size_t) jn] = (int64_t) i;
- it.keep = c.lc + (size_t) jn * stride;
- it.after = (int64_t) c.lc_first - 1;
- }
- }
- if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)
- it.direct = true;
- it.stage = -1;
- it.left.store(3, std::memory_order_relaxed);
- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
- return;
- }
- it.stage = (int) (c.n_file++ % (uint64_t) c.S);
- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
- for (int r = 0; r < 3; ++r) {
- const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);
- it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);
- sb += flow_region(r);
- }
- it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);
- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
- }
- // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance
- // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the
- // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -
- // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is
- // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer
- // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the
- // rest of its picks are streamed in)
- // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same
- // profile and the same routing - and each reads only its part of every expert)
- void flow_spec(Chunk& c, int k, int il, int T) {
- static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;
- Layer& l = L[(size_t) il];
- const int mi = flow_mi(c, l);
- int* vs = member(l, mi).vs_pin;
- c.spec_k = k;
- c.spec_il = il;
- c.spec_begin = c.base + c.items.size();
- const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += pr[e];
- std::vector<std::pair<double, int>> cand;
- const double picks = (double) l.k_exp * T;
- // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE
- // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole
- // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376
- // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and
- // 21 of 1568, the early reads 9-12% faster there)
- static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;
- const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;
- for (int e : l.host_list) {
- if (!early) break;
- const double p = sum > 0 ? pr[e] / sum : 0.0;
- const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));
- if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {
- return a.first > b.first;
- });
- if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));
- std::vector<char> is((size_t) l.n_exp, 0);
- int j = 0;
- for (const auto& x : cand) {
- vs[x.second] = l.n_slots + j++;
- is[(size_t) x.second] = 1;
- }
- for (int e : l.host_list)
- if (!is[(size_t) e]) vs[e] = l.n_slots + j++;
- for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);
- c.n_spec = (int) cand.size();
- c.it_spec += cand.size();
- }
- // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones
- void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {
- const Layer& l = L[(size_t) il];
- const int mi = flow_mi(c, l);
- const int* vs = member(l, mi).vs_pin;
- for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {
- const int v = fit(c, i).v;
- if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;
- }
- for (int e : l.host_list) { // (index order: their virtual slots ascending)
- const int v = vs[e];
- if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
- }
- }
- // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)
- bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
- const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
- const size_t i = (size_t) (it - c.fl_layers.begin());
- if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
- if (k + 1 < (int) c.fl_T.size()) {
- k2 = k + 1;
- il2 = il;
- return true;
- }
- if (i + 1 >= c.fl_layers.size()) return false;
- k2 = 0;
- il2 = c.fl_layers[i + 1];
- return true;
- }
- if (i + 1 < c.fl_layers.size()) {
- k2 = k;
- il2 = c.fl_layers[i + 1];
- return true;
- }
- if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {
- k2 = k + 1;
- il2 = c.fl_layers[0];
- return true;
- }
- return false;
- }
- // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each
- // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now
- bool flow_pump(Chunk& c, std::string& err) {
- const size_t end = c.base + c.items.size();
- IoQueue::Job jobs[6];
- while (c.n_read < end) {
- Chunk::Item& it = fit(c, c.n_read);
- if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot
- // written again once the layer before's items are all read (none still copies out of it)
- bool wait = false;
- if (it.keep == nullptr)
- wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;
- else {
- while (c.lc_chk < c.n_read &&
- (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))
- ++c.lc_chk;
- wait = c.lc_chk <= (size_t) it.after;
- }
- if (wait) break;
- }
- if (it.direct) {
- const size_t R = (size_t) c.ring_n, r = c.n_read % R;
- if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it
- if (c.n_comp <= c.n_read - R) break;
- CK(cudaEventSynchronize(c.ring_free[r]));
- }
- const Layer& l = L[(size_t) it.il];
-#if defined(STRATA_METAL)
- uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)
-#else
- uint8_t* dst = c.ring + r * c.ring_stride;
-#endif
- const x3::ExpertLayout& lay = l.lay;
- int nj = 0;
- for (int q = 0; q < 3; ++q) {
- IoQueue::Job& j = jobs[nj++];
- j = IoQueue::Job{};
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];
- j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];
- j.len = mx_part_bytes(l, q);
- j.dst = dst + lay.trellis[q];
- j.exact = true;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;
- j.map = pk.at(l.mx_file[q], j.off);
- j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;
- }
- c.io->push(jobs, nj);
- } else if (it.stage >= 0) {
- const int64_t prev = c.stage_item[(size_t) it.stage];
- if (prev >= 0) {
- if ((size_t) prev >= c.n_dma) break;
- const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);
- if (q == cudaErrorNotReady) break;
- CK(q);
- }
- c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;
- const Layer& l = L[(size_t) it.il];
- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
- const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)
- int nj = 0;
- for (int r = 0; r < 3; ++r) {
- if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X) {
- IoQueue::Job& j = jobs[nj++];
- j = IoQueue::Job{};
- j.len = nb;
- j.dst = sb + (size_t) X * nb;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
- }
- sb += flow_region(r);
- continue;
- }
- IoQueue::Job& j = jobs[nj++];
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
- j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);
- j.len = mx_read_len(l, r);
- j.dst = sb;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;
- j.map = pk.at(l.mx_file[r], j.off);
- j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;
- sb += flow_region(r);
- }
- c.io->push(jobs, nj);
- }
- ++c.n_read;
- }
- const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;
- struct Kick { // (every way out: what was queued is submitted)
- cudaStream_t s;
- const size_t &a, b;
- ~Kick() {
- if (a != b) kick(s);
- }
- } kq{c.cs, c.n_dma, dma0};
- while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {
- Chunk::Item& it = fit(c, c.n_dma);
- if (it.left.load(std::memory_order_acquire) != 0) break;
- if (it.bad.load(std::memory_order_acquire)) {
- err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +
- " from the model's files failed";
- return false;
- }
- const size_t r = c.n_dma % R;
- if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));
- const Layer& l = L[(size_t) it.il];
- uint8_t* dst = c.ring + r * c.ring_stride;
- const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;
- if (it.direct) { // (already in the slot)
- CK(cudaEventRecord(c.ring_fill[r], c.cs));
- c.stream_bytes += (double) unit(l);
- ++c.n_dma;
- continue;
- }
- for (int p = 0; p < 2; ++p)
- CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));
- if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)
- const int t2 = gg::expert_type(l.lay, 2);
- const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);
- CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,
- cudaMemcpyHostToDevice, c.cs));
- } else
- CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));
- CK(cudaEventRecord(c.ring_fill[r], c.cs));
- if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));
- c.stream_bytes += (double) unit(l);
- ++c.n_dma;
- }
- return true;
- }
- // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM
- // ones in shares between them (the device works while the reads run)
- bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {
- cudaStream_t s = d.st;
- const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)
- if (c.spec_k != k || c.spec_il != L0.il) {
- err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";
- return false;
- }
- flow_rest(c, k, L0.il, b);
- const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
- int k2 = 0, il2 = 0;
- if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);
- const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
- int v_done = 0;
- while (c.n_comp < le) {
- // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what
- // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so
- // the sums come out the same on every run whatever the reads' timing
- const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});
- if (!flow_pump(c, err)) return false;
- if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy
- const double tw = now_ms();
- kick(s);
- if (c.n_read > c.n_dma) c.io->wait_any(500);
- else {
- const Chunk::Item& it = fit(c, c.n_read);
- if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));
- }
- c.wait_ms += now_ms() - tw;
- continue;
- }
- const int n = (int) (i1 - i0);
- int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
- int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
- ++c.n_batch;
- // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)
- for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];
- bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];
- if (bc[n] > bc[0]) {
- CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));
- CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));
- if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))
- return false;
- }
- for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));
- kick(s);
- c.n_comp = i1;
- if (!flow_pump(c, err)) return false;
- const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));
- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;
- v_done = v_to;
- }
- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
- while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)
- c.items.pop_front();
- ++c.base;
- }
- return true;
- }
-
- // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)
- // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order
- bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,
- const int* b, int i0, int i1, int T, std::string& err) {
- const int D = g.dim, FF = l.ffn;
- while (i0 < i1) {
- int j = i0 + 1;
- while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;
- const int a = b[i0], nrows = b[j] - a;
- int max_rows = 0;
- for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);
- if (nrows > 0) {
- x3::SetArgs sa;
- sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)
- sa.stride = stride;
- sa.lay = l.nm > 1 ? l.hlay : l.lay;
- sa.n_exp = j - i0;
- sa.bounds = d_bounds + i0;
- sa.row0 = a;
- sa.max_rows = max_rows;
- sa.hbounds = b + i0;
- sa.total_rows = nrows;
- sa.n_embd = D;
- sa.n_ff = FF;
- sa.limit = g.lim;
- if (sa.lay.fmt == gg::kFmtMXFP4) {
- gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
- gg::set_down(sa, c.h16, FF, c.y, D, d.st);
- } else {
- x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
- x3::set_down(sa, c.h16, FF, c.y, D, d.st);
- }
- if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {
- std::vector<int> bb(b + i0, b + j + 1);
- std::vector<uint16_t> hv((size_t) nrows * FF);
- std::vector<float> yv((size_t) nrows * D);
- CK(cudaStreamSynchronize(d.st));
- CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&
- wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};
- std::fwrite(hdr, 8, 6, dump_y);
- std::fwrite(bb.data(), 4, bb.size(), dump_y);
- std::fwrite(hv.data(), 2, hv.size(), dump_y);
- std::fwrite(yv.data(), 4, yv.size(), dump_y);
- std::fflush(dump_y);
- }
- moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);
- }
- i0 = j;
- }
- (void) err;
- return true;
- }
-
- bool load_dense(int di, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- size_t* acct = &d.weights;
- for (Layer& l0 : L)
- for (int mi = 0; mi < l0.nm; ++mi) {
- Layer& l = member(l0, mi);
- if (l.dev != di) continue;
- // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,
- // of the shared expert's FFN; the rest whole)
- const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);
- const std::string& P = l.name;
- const std::string A = P + "attn.";
- bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&
- (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&
- mat(d, A + "wkv", l.wkv, err, acct) &&
- (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&
- f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&
- f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&
- f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&
- f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&
- f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&
- f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&
- f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&
- f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&
- f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&
- f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&
- f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&
- (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||
- f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&
- mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&
- mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&
- mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);
- if (!ok) return false;
- l.sink += (size_t) l.ah0;
- for (int gi = 0; gi < l.ogn; ++gi)
- if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;
- if (l.kv_src) {
- const std::string C = A + "compressor.";
- if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;
- if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;
- if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||
- !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))
- return false;
- }
- if (l.ix_src) {
- const std::string I = A + "indexer.";
- if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))
- return false;
- if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))
- return false;
- }
- if (l.eg >= 0) {
- if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;
- if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already
- if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;
- } else {
- float *qw = nullptr, *kw = nullptr;
- if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))
- return false;
- std::vector<float> a((size_t) g.hc * g.dim), b(a.size());
- CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));
- for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product
- CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));
- l.eg_w = qw;
- }
- }
- if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;
- if (g.n_mtp > 0 && !l.mtp) {
- if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;
- if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||
- !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))
- return false;
- }
- if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;
- }
- if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns
- const int hn = head_nm();
- if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))
- return false;
- if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))
- return false;
- if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;
- if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head
- const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");
- const int R = g.d_rank;
- head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)
- // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same
- // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)
- if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {
- std::vector<uint16_t> rows((size_t) d.mk_vn * R);
- if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {
- err = "the markov head: read failed";
- return false;
- }
- if (th->dtype == "BF16")
- for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));
- if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||
- !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))
- return false;
- CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));
- }
- }
- }
- if (di == head_dev) {
- head = d.head;
- norm_w = d.norm_w;
- d_norm = d.d_norm;
- if (g.n_mtp > 0) {
- const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";
- if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))
- return false;
- const Tensor* te = pk.get(Z + "markov_head.embed.weight");
- const Tensor* th = pk.get(Z + "markov_head.head.weight");
- if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {
- err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";
- return false;
- }
- if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
- !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))
- return false;
- }
- }
- if (g.n_mtp > 0 && di == d_dev) {
- if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||
- !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))
- return false;
- }
- return true;
- }
-
-#if defined(STRATA_METAL)
- // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays
- // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and
- // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most
- // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.
- double metal_expert_cap() const {
- uint64_t phys = 0;
- size_t len = sizeof phys;
- sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);
- unsigned long long pages = 0;
- size_t pl = sizeof pages;
- double floor_b = 0;
- if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;
- size_t fr = 0, tot = 0;
- cudaMemGetInfo(&fr, &tot);
- const double gb = 1e9, P = (double) phys;
- double keep = std::max(7.0 * gb, 0.15 * P);
- if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;
- double share = 0.40;
- if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);
- double leave = keep;
- if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});
- const double cap = std::min((double) tot, P - leave);
- task_vm_info_data_t ti;
- mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;
- double foot = 0;
- if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;
- return std::max(4.0 * gb, cap - foot - 1.0 * gb);
- }
-#endif
-#if defined(STRATA_METAL)
- // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it
- // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).
- bool zc_try(Layer& l, int e) {
- static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)
- if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;
- if (l.vm[(size_t) e][0].raw != nullptr) return true;
- uint64_t a[3];
- for (int r = 0; r < 3; ++r) {
- a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));
- if (a[r] == 0) return false;
- }
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m = x3::Mat{};
- m.K = gg::expert_type(l.lay, r);
- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
- m.raw = (const void*) a[r];
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- }
- return true;
- }
-#endif
- void set_vm(Layer& l, int e, int slot) {
- uint8_t* b = l.vram_base + (size_t) slot * l.blob;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m = x3::Mat{};
- m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)
- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
- m.raw = b + l.lay.trellis[r];
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- }
- return;
- }
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);
- m.suh = (const uint16_t*) (b + l.lay.suh[r]);
- m.svh = (const uint16_t*) (b + l.lay.svh[r]);
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- m.K = l.lay.K[r];
- m.cb = 2;
- m.ld = 0;
- }
- }
- // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)
- void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- const uint8_t* tr = blob + l.lay.trellis[r];
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- (size_t) nth * tile);
- std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);
- std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
- }
- }
- void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- uint8_t* tr = blob + l.lay.trellis[r];
- for (int X = 0; X < NH; ++X)
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);
- std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
- for (int X = 0; X < NH; ++X)
- std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],
- (size_t) l.hl.n[r] * 2);
- }
- }
-
- // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back
- void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
- if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
- std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,
- blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);
- std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);
- std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
- std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);
- std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
- std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);
- }
- void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
- if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
- std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,
- src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);
- if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);
- std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);
- }
- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
- std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);
- std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);
- if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);
- }
-
- bool place_experts(std::string& err) {
- // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)
- double room = 1e300, kept = 0;
- for (const Dev& d : devs)
- if (commit_charged(d)) room = commit_room();
- const double room0 = room;
- // per group: what its tightest member has left (each member holds its part of every VRAM expert of the
- // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round
- // the members layer by layer, so each member's total comes to about the mean's)
- auto ucount = [](const Layer& l) {
- double s = 0;
- for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));
- return s / l.nm;
- };
- for (size_t gi = 0; gi < groups.size(); ++gi) {
- double budget = 1e300;
- bool charged = false;
- for (int m : groups[gi].mem) {
- cudaSetDevice(devs[(size_t) m].id);
- size_t fr = 0, tot = 0;
- CK(cudaMemGetInfo(&fr, &tot));
- budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *
- o.vram_experts);
-#if defined(STRATA_METAL)
- if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {
- const double cap = metal_expert_cap();
- if (cap < budget) {
- std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",
- cap / 1e9, cap / 1e9);
- budget = cap;
- }
- }
-#endif
- charged = charged || commit_charged(devs[(size_t) m]);
- }
- if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)
- double all = 0;
- for (const Layer& l : L)
- if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;
- const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);
- kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;
- budget = capped;
- room = std::max(0.0, room - std::min(budget, all) * nm);
- }
- double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)
- for (Layer& l : L)
- if (l.grp == (int) gi && l.mtp) {
- l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));
- budget_t -= (double) l.n_vram * ucount(l);
- }
- double want = 0;
- for (const Layer& l : L)
- if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;
- const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
- for (Layer& l : L) {
- if (l.grp != (int) gi || l.mtp) continue;
- if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)
- l.n_vram = 0;
- continue;
- }
- l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
- // never more than fits: the floor of each layer's share can still round up the total
- while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
- }
- double used = 0;
- for (Layer& l : L)
- if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
- // what the floors left: whole experts, layer by layer
- for (Layer& l : L) {
- if (l.grp != (int) gi || l.cpu_only) continue;
- while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
- ++l.n_vram;
- used += (double) ucount(l);
- }
- }
- }
- if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it
- const os::Commit c = os::commit();
- const double gib = (double) (1ull << 30);
- const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;
- commit_kept = kept;
- pagefile_hint = std::ceil((pf + kept) / gib);
- std::fprintf(stderr,
- "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "
- "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "
- "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "
- "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",
- kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);
- } else if (room0 < 1e300 && o.verbose) {
- std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "
- "programs)\n", room0 / 1e9, commit_margin() / 1e9);
- }
- for (Layer& l : L) {
- // the hottest experts of the profile first, then by index
- std::vector<int> order((size_t) l.n_exp);
- std::iota(order.begin(), order.end(), 0);
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });
- l.slot.assign((size_t) l.n_exp, -1);
- l.host.assign((size_t) l.n_exp, nullptr);
- // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps
- l.n_slots = std::min(l.n_vram, l.n_exp);
- if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)
- l.n_vram = l.n_slots - 1;
- l.spare_vram = l.n_slots - 1;
- } else {
- l.n_vram = l.n_slots;
- l.spare_vram = -1;
- }
- for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;
- for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {
- Layer& x = member(l, m);
- Dev& d = devs[(size_t) x.dev];
- cudaSetDevice(d.id);
- void* v = nullptr;
- if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {
- err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +
- " experts on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- d.allocs.push_back(v);
- d.experts += (size_t) l.n_slots * unit(x);
- x.vram_base = (uint8_t*) v;
- }
- l.heat.assign((size_t) l.n_exp, 0.0f);
- { // the profile as the starting heat: its counts scaled to ~512 hits a layer
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- const double sc = sum > 0 ? 512.0 / sum : 0.0;
- for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);
- }
- const int n_host = l.n_exp - l.n_vram;
- { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;
- l.hl.k[r] = k;
- l.hl.n[r] = n;
- l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;
- off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);
- l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;
- off += align256((size_t) k * 2);
- l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;
- off += align256((size_t) n * 2);
- }
- l.hl.bytes = off;
- }
- l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});
- if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);
- } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)
- for (int e = 0; e < l.n_exp; ++e)
- if (pk.map(l.region_file[(size_t) e]) == nullptr) {
- err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";
- return false;
- }
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) {
- l.hh[(size_t) e][0] = file_part(l, e);
- l.host[(size_t) e] = l.hh[(size_t) e][0];
- }
- } else if (n_host > 0) {
- const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)
- const size_t bytes = (size_t) n_parts * l.hl.bytes;
- for (int X = 0; X < NH; ++X) {
- void* m = node_map(bytes, X, NH);
- if (m == nullptr) {
- err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +
- " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";
- return false;
- }
- host_maps.emplace_back(m, bytes);
- int hi = 0;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;
- if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;
- }
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];
- }
- l.vm.assign((size_t) l.n_exp, {});
- if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);
- if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0)
- tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));
- cudaSetDevice(devs[(size_t) x.dev].id);
- CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));
- }
- }
- return true;
- }
-
- // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the
- // file: the mapping reads them on first use)
- bool load_experts_mx(std::string& err) {
- struct Job {
- int layer, e;
- };
- std::vector<Job> jobs;
- size_t maxb = 0;
- for (const Layer& l : L) {
- maxb = std::max(maxb, l.blob);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
- });
- std::atomic<size_t> next{0};
- std::atomic<bool> failed{false};
- std::mutex emu;
- std::string ferr;
- std::atomic<int64_t> done_bytes{0};
- const double t0 = now_ms();
- std::vector<std::thread> th;
- for (int t = 0; t < 12; ++t)
- th.emplace_back([&] {
- uint8_t *stage = nullptr, *stage2 = nullptr;
- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
- for (;;) {
- const size_t i = next.fetch_add(1);
- if (i >= jobs.size() || failed.load()) break;
- const Layer& l = L[(size_t) jobs[i].layer];
- const int e = jobs[i].e;
- bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);
- for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- if (l.nm > 1) blob_to_half(l, stage, m, stage2);
- ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,
- unit(l), cudaMemcpyHostToDevice) == cudaSuccess;
- }
- if (!ok) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- failed.store(true);
- break;
- }
- const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
- if (o.verbose && (i % 1024) == 0)
- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
- db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- if (stage) cudaFreeHost(stage);
- if (stage2) cudaFreeHost(stage2);
- });
- for (auto& t : th) t.join();
- if (failed.load()) {
- err = ferr;
- return false;
- }
- // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the
- // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them
- struct Warm {
- int layer, e;
- double share;
- };
- std::vector<Warm> warm;
- for (const Layer& l : L) {
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- for (int e = 0; e < l.n_exp; ++e) {
- if (l.slot[(size_t) e] >= 0) {
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);
- } else {
- warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});
- }
- }
- }
- std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });
- if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left
- std::vector<std::pair<int, int>> hot;
- for (const Warm& w : warm) hot.push_back({w.layer, w.e});
- if (!copy_experts(hot, err)) return false;
- if (copy_mode) {
- warm.erase(std::remove_if(warm.begin(), warm.end(),
- [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),
- warm.end());
- const uint64_t av = ram_avail();
- if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;
- }
- }
- if (ram_tier && warm_bytes > 0 && !warm.empty()) {
- // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for
- // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not
- // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on
- // Windows - the engine's own pinned buffers come after this
- const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));
- while (ram_slots.size() < want + 1) {
- const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());
- uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);
- if (c == nullptr) break;
- ram_chunks.push_back(c);
- ram_chunk_bytes.push_back(n * maxb);
- for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);
- }
- ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;
- for (Layer& l : L)
- if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);
- std::atomic<size_t> ri{0};
- std::atomic<bool> rfail{false};
- std::vector<std::thread> rt;
- for (int t = 0; t < 12 && ram_held > 0; ++t)
- rt.emplace_back([&] {
- for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {
- const Layer& l = L[(size_t) warm[i].layer];
- if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],
- mx_part_bytes(l, r), false);
- }
- });
- for (auto& t : rt) t.join();
- if (rfail.load()) {
- err = "reading the RAM tier's experts from the model's files failed";
- return false;
- }
- for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;
- if (ram_held > 0) ram_free.push_back(ram_held);
- if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });
- if (o.verbose)
- std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "
- "read from the model's files (%.1f s)\n",
- ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,
- (now_ms() - t0) / 1000.0);
- return true;
- }
- size_t nw = 0;
- for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)
- b += (double) L[(size_t) warm[nw].layer].blob;
- std::atomic<size_t> wi{0};
- std::vector<std::thread> wt;
- for (int t = 0; t < 12 && nw > 0; ++t)
- wt.emplace_back([&] {
- std::vector<uint8_t> buf(maxb);
- for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());
- });
- for (auto& t : wt) t.join();
- if (o.verbose && nw > 0)
- std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,
- nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
- return true;
- }
-
- // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU
- // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the
- // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:
- // none - read in place: the page cache follows a conversation's experts where a static set (the profile's
- // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the
- // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file
- // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what
- // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place
- bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {
- if (o.mmap_experts > 0 || hot.empty()) return true;
- const double t0 = now_ms();
- auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };
- double all = 0;
- {
- std::vector<char> seen(L.size(), 0);
- for (const auto& h : hot) {
- const Layer& l = L[(size_t) h.first];
- all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));
- }
- }
- const uint64_t avail = ram_avail();
- // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
- const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),
- commit_room());
- // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
- // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
- // on 64 GB and on 32 GB of RAM)
- double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;
- if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;
- if (o.mmap_experts == 0 && all > can)
- std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "
- "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "
- "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
- if (budget <= 0) {
- if (o.verbose)
- std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "
- "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",
- all / 1e9, can / 1e9, kRamMargin / 1e9);
- return true;
- }
- // NUMA parts: whole rows of each matrix, so any row count the nodes divide
- int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));
- if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)
- nh = std::atoi(v) >= 2 ? 2 : 1;
- if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;
- NH = nh;
- cp_lay.assign(L.size(), {0, 0, 0, 0});
- for (const Layer& l : L) {
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- cp_lay[(size_t) l.il][(size_t) r] = off;
- off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));
- }
- cp_lay[(size_t) l.il][3] = off;
- }
- // the hottest that fit the budget (a layer's first one brings its spare)
- std::vector<int> n_copy(L.size(), 0);
- double used = 0;
- size_t take = 0;
- for (; take < hot.size(); ++take) {
- const Layer& l = L[(size_t) hot[take].first];
- const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));
- if (used + add > budget) break;
- used += add;
- ++n_copy[(size_t) l.il];
- }
- // each layer's slots: one block per node of (copies + spare) parts
- std::vector<int> first(L.size(), -1);
- copy_spare.assign(L.size(), -1);
- for (Layer& l : L) {
- l.ram.assign((size_t) l.n_exp, -1);
- const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);
- if (ns == 0) continue;
- const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
- std::array<uint8_t*, 2> base = {nullptr, nullptr};
- bool ok = true;
- for (int X = 0; X < NH && ok; ++X) {
- base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
- ok = base[(size_t) X] != nullptr;
- if (ok) {
- ram_chunks.push_back(base[(size_t) X]);
- ram_chunk_bytes.push_back(bytes);
- }
- }
- if (!ok) {
- std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,
- ns, (double) bytes * NH / 1e9);
- n_copy[(size_t) l.il] = 0;
- continue;
- }
- first[(size_t) l.il] = (int) ram_part.size();
- for (int i = 0; i < ns; ++i)
- ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});
- if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;
- }
- struct Job {
- int layer, e, slot;
- };
- std::vector<Job> jobs;
- {
- std::vector<int> next = first;
- for (size_t i = 0; i < take; ++i) {
- const int il = hot[i].first;
- if (n_copy[(size_t) il] == 0) continue;
- jobs.push_back({il, hot[i].second, next[(size_t) il]++});
- }
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
- });
- std::atomic<size_t> ji{0};
- std::atomic<bool> bad{false};
- std::atomic<int64_t> done_b{0};
- std::vector<std::thread> th;
- for (int t = 0; t < 12 && !jobs.empty(); ++t)
- th.emplace_back([&] {
- for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {
- const Job& j = jobs[i];
- const Layer& l = L[(size_t) j.layer];
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,
- ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))
- bad.store(true);
- }
- const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
- if (o.verbose && i % 2048 == 0)
- std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
- db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- });
- for (auto& t : th) t.join();
- if (bad.load()) {
- err = "copying the CPU lane's experts from the model's files failed";
- return false;
- }
- for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;
- copy_mode = true;
- ram_held = (int) jobs.size();
- double cb = 0;
- for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;
- std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",
- jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,
- jobs.size() < hot.size() ? "; the others read in place" : "");
- return true;
- }
-
- bool load_experts(std::string& err) {
- if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);
- // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place
- // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their
- // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are
- // dropped from it again
- struct Job {
- int layer, e;
- };
- std::vector<Job> jobs;
- for (const Layer& l : L)
- for (int e = 0; e < l.n_exp; ++e)
- if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
- uint64_t warm = 0;
- if (file_experts) {
- std::vector<std::pair<double, Job>> cand;
- for (const Layer& l : L) {
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
- for (const auto& c : cand) {
- const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];
- if (warm + b > warm_bytes) break;
- warm += b;
- jobs.push_back(c.second);
- }
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])
- return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];
- return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];
- });
- std::atomic<size_t> next{0};
- std::atomic<bool> failed{false};
- std::mutex emu;
- std::string ferr;
- const int nth = 12;
- size_t maxr = 0, maxb = 0;
- for (const Layer& l : L) {
- maxb = std::max(maxb, l.blob);
- for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);
- }
- std::atomic<int64_t> done_bytes{0};
- const double t0 = now_ms();
- std::vector<std::thread> th;
- for (int t = 0; t < nth; ++t)
- th.emplace_back([&] {
- std::vector<uint8_t> region(maxr);
- uint8_t *stage = nullptr, *stage2 = nullptr;
- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
- int cur_dev = -1;
- for (;;) {
- const size_t i = next.fetch_add(1);
- if (i >= jobs.size() || failed.load()) break;
- const Layer& l = L[(size_t) jobs[i].layer];
- const int e = jobs[i].e;
- if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],
- region.data())) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- failed.store(true);
- break;
- }
- if (l.slot[(size_t) e] >= 0) {
- repack(l, region.data(), stage);
- bool up = true;
- for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)
- const Layer& x = member(l, m);
- const Dev& d = devs[(size_t) x.dev];
- if (cur_dev != d.id) {
- cudaSetDevice(d.id);
- cur_dev = d.id;
- }
- if (l.nm > 1) blob_to_half(l, stage, m, stage2);
- up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,
- unit(x), cudaMemcpyHostToDevice) == cudaSuccess;
- }
- if (!up) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);
- failed.store(true);
- break;
- }
- if (file_experts)
- os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);
- } else if (!file_experts) {
- for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);
- }
- const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +
- (int64_t) l.region_bytes[(size_t) e];
- if (o.verbose && (i % 1024) == 0)
- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),
- db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- if (stage) cudaFreeHost(stage);
- if (stage2) cudaFreeHost(stage2);
- });
- for (auto& t : th) t.join();
- if (failed.load()) {
- err = ferr;
- return false;
- }
- if (file_experts)
- std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",
- (now_ms() - t0) / 1000.0, warm / 1e9);
- return true;
- }
-
- std::string describe() const {
- std::ostringstream s;
- for (size_t di = 0; di < devs.size(); ++di) {
- const Dev& d = devs[di];
- int a = -1, b = -1, ev = 0, et = 0;
- for (const Layer& l : L)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di) {
- if (a < 0) a = l.il;
- b = l.il;
- ev += l.n_vram;
- et += l.n_exp;
- }
- size_t fr = 0, tot = 0;
- cudaSetDevice(d.id);
- cudaMemGetInfo(&fr, &tot);
- const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();
- char tp[64] = "";
- if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);
- char line[384];
- std::snprintf(line, sizeof line,
- "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "
- "%.2f GB, %.0f MiB free\n",
- d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,
- d.caches / 1e9, fr / 1048576.0);
- s << line;
- }
- int nh = 0;
- size_t hb_ = 0;
- for (const Layer& l : L) {
- nh += l.n_exp - l.n_vram;
- hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;
- }
- if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)
- uint64_t h = 1469598103934665603ull;
- for (const Layer& l : L)
- for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }
- char hl[96];
- std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);
- s << hl;
- }
- char line[200];
- if (copy_mode)
- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "
- "parts), the rest in place in the model's files; %d threads\n",
- nh, hb_ / 1e9, ram_held, NH, cpu_threads());
- else if (file_experts)
- std::snprintf(line, sizeof line,
- "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",
- nh, hb_ / 1e9, cpu_threads());
- else
- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,
- hb_ / 1e9, NH, cpu_threads());
- s << line;
- return s.str();
- }
-
- double store_gb() const {
- static double cached = -2.0; // (decided once: the layout of the blobs depends on it)
- if (cached > -1.5) return cached;
- double gb = o.ssd_cache_gb;
- if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);
- if (!pk.gguf) gb = 0.0;
- else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most
- const uint64_t avail = ram_avail();
- const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;
- gb = room >= 4.0 ? std::min(48.0, room) : 0.0;
- }
- return cached = gb;
- }
- void init_store() {
- const double gb = store_gb();
- if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;
- std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- ExpertStore::LayerGeo& gl = geo[(size_t) l.il];
- gl.n_exp = l.n_exp;
- for (int r = 0; r < 3; ++r) {
- gl.file[r] = l.mx_file[r];
- gl.off[r] = l.mx_off[r];
- gl.stride[r] = l.mx_stride[r];
- gl.bytes[r] = mx_part_bytes(l, r);
- gl.tr[r] = l.lay.trellis[r];
- }
- gl.blob = l.blob;
- }
- ExpertStore::Cfg cfg;
- cfg.budget_bytes = (size_t) (gb * 1e9);
- cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
- cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
- cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
- cfg.ram_avail = [] { return ram_avail(); };
- cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
- cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;
- // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a
- // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane
- // took which expert, the CPU lane's are the mmap path's byte for byte)
- const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&
- L[0].nm == 1;
- if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)
- const int dev0 = devs[0].id;
- cfg.map_chunk = [dev0](void* h, size_t n) -> void* {
- cudaSetDevice(dev0);
- if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {
- (void) cudaGetLastError();
- return nullptr;
- }
- void* d = nullptr;
- if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {
- (void) cudaGetLastError();
- cudaHostUnregister(h);
- return nullptr;
- }
- if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);
- return d;
- };
- cfg.unmap_chunk = [dev0](void* h) {
- cudaSetDevice(dev0);
- cudaHostUnregister(h);
- };
- }
- bool direct = true;
- for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
- auto st = std::make_unique<ExpertStore>();
- if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {
- std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
- return;
- }
- store = std::move(st);
- for (Layer& l : L)
- if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);
- std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
- store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
- (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
- if (want_zc) {
- if (!store->gpu_mapped()) {
- std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");
- for (Layer& l : L) l.zc.clear();
- } else {
- zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;
- cudaSetDevice(devs[0].id);
- if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {
- (void) cudaGetLastError();
- zc_stage_buf = nullptr;
- for (Layer& l : L) l.zc.clear();
- } else {
- for (Layer& l : L)
- if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);
- else if (!l.zc.empty()) {
- (void) cudaGetLastError();
- l.zc.clear();
- }
- }
- }
- std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",
- zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
- }
- }
- // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)
- void preload_store() {
- if (!store) return;
- if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;
- std::vector<std::pair<double, std::pair<int, int>>> cand;
- double tot = 0;
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e) tot += p[e];
- }
- if (tot <= 0) return; // (no profile yet: the first tokens fill it)
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
- std::vector<std::pair<int, int>> list;
- list.reserve(cand.size());
- for (const auto& c : cand) list.push_back(c.second);
- const double t0 = now_ms();
- const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;
- const size_t n = store->preload(list, leave);
- std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,
- (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
- }
- // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
- // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
- void store_sync() {
- if (!store || !store->gpu_mapped()) return;
- std::vector<ExpertStore::Vis> ev;
- store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });
- if (ev.empty()) return;
- std::vector<char> dirty(L.size(), 0);
- for (const auto& v : ev) {
- Layer& l = L[(size_t) v.layer];
- if (l.zc.empty()) continue;
- l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;
- dirty[(size_t) v.layer] = 1;
- }
- for (Layer& l : L) {
- if (!dirty[(size_t) l.il]) continue;
- if (l.zc_tab == nullptr) continue;
- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.zc[(size_t) e]) {
- const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;
- tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;
- if (s < 0) l.zc[(size_t) e] = 0;
- }
- cudaSetDevice(devs[0].id);
- cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);
- }
- }
-
- int cpu_threads() const {
- int n = 0;
- for (const auto& pl : pools) n += pl->size() - 1;
- return n + 1;
- }
- // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over
- // every node's cores (the pages lie where the cache put them)
- void make_pools() {
- std::vector<std::vector<int>> cs = cores;
- if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)
- std::vector<int> all = cs[0];
- cs.assign((size_t) NH, {});
- for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);
- }
- for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {
- std::vector<int> c = cs[(size_t) X];
- if (file_experts && NH == 1)
- for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());
- if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)
- int nw = (int) c.size();
- if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));
- pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)
- }
- (void) gg::mx_tier(); // (measured here, not on the first token)
- if (o.verbose) {
- int nthr = 0;
- for (auto& p : pools) nthr += p->size();
- std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),
- gg::mx_tier_report());
- }
- }
-
- void reset() {
- pos = 0;
- toks.clear();
- ctoks.clear();
- ck[0].pos = ck[1].pos = -1;
- for (Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m) {
- Layer& l = member(l0, m);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);
- if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);
- if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- }
-
- // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
- // state is indexed by position, and what lies past a restored position is written again)
- void checkpoint(int slot) {
- Check& c = ck[slot & 1];
- c.pos = pos;
- c.win.resize(L.size());
- c.comp.resize(L.size() * 2);
- for (size_t i = 0; i < L.size(); ++i) {
- Layer& l = L[i];
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- c.win[i].resize((size_t) g.win * win_rb());
- cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);
- if (l.c_kv) {
- c.comp[2 * i].resize((size_t) l.ratio * g.hd);
- c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
- cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);
- cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,
- cudaMemcpyDeviceToHost, d.st);
- }
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- }
-
- // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
- int best_check(int64_t limit) const {
- int b = -1;
- for (int i = 0; i < 2; ++i)
- if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
- return b;
- }
- int64_t restorable(int64_t limit) const {
- const int b = best_check(limit);
- return b < 0 ? -1 : ck[b].pos;
- }
-
- bool restore(int64_t limit) {
- const int b = best_check(limit);
- if (b < 0) return false;
- const Check& c = ck[b];
- for (size_t i = 0; i < L.size(); ++i)
- for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)
- Layer& l = member(L[i], m);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);
- if (l.c_kv) {
- cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);
- cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,
- cudaMemcpyHostToDevice, d.st);
- }
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- pos = c.pos;
- toks.resize((size_t) pos);
- ctoks.resize((size_t) pos);
- for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
- if (o2.pos > pos) o2.pos = -1;
- return true;
- }
-
- size_t bounce_bytes() const {
- return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
- }
- size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }
- size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }
- size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }
- static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need
-
- // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the
- // host; a prompt's pictures as their spans' rows (set_images)
- VisionTower vt;
- VisionGeo vg;
- int img_tok = -1;
- std::string vision_err; // why the tower did not load (pictures off), or empty
- std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]
- std::vector<Image> imgs; // the pictures of the prompt being fed, by start
- // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst
- bool read_bf16(const Tensor& t, size_t n, void* dst) {
- if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);
- if (t.dtype != "F32" || t.bytes != n * 4) return false;
- std::vector<float> f(n);
- if (!pk.read(t, f.data())) return false;
- uint16_t* o = (uint16_t*) dst;
- for (size_t i = 0; i < n; ++i) {
- uint32_t u;
- std::memcpy(&u, &f[i], 4);
- o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);
- }
- return true;
- }
- bool load_vision(std::string& err) {
- vg.layers = (int) pk.num("vision.num_hidden_layers", 0);
- const int tok = (int) pk.num("image_token_id", -1);
- if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none
- vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);
- vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);
- vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);
- vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;
- const char* mark[3] = {"image_start", "image_newline", "image_end"};
- for (int i = 0; i < 3; ++i) {
- const Tensor* t = pk.get(mark[i]);
- img_mark[i].resize((size_t) g.dim);
- if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {
- err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";
- return false;
- }
- }
- const double t0 = now_ms();
- Dev& d = devs[0];
- // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which
- // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)
- auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {
- const Tensor* t = pk.get(n);
- if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {
- const std::string pre = n.substr(0, n.size() - 13);
- const Tensor* tg = pk.get(pre + "mlp.gate.weight");
- const Tensor* tu = pk.get(pre + "mlp.up.weight");
- if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;
- shape = {tg->shape[0] * 2, tg->shape[1]};
- const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];
- return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));
- }
- if (t == nullptr) return false;
- shape = t->shape;
- size_t n_el = 1;
- for (int64_t x : t->shape) n_el *= (size_t) x;
- return dst == nullptr || read_bf16(*t, n_el, dst);
- };
- if (!vt.load(vg, d.id, read, err)) return false;
- img_tok = tok;
- CK(cudaSetDevice(d.id));
- CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)
- if (o.verbose)
- std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",
- vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),
- (now_ms() - t0) / 1000.0);
- return true;
- }
- bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {
- if (img_tok < 0) {
- err = "this model has no vision tower (or it is off)";
- return false;
- }
- std::vector<uint16_t> rows;
- int nh = 0, nw = 0;
- if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;
- const size_t D = (size_t) g.dim;
- span.resize(((size_t) nh * (nw + 1) + 2) * D);
- uint16_t* o = span.data();
- auto put = [&](const uint16_t* r) {
- std::memcpy(o, r, D * 2);
- o += D;
- };
- put(img_mark[0].data());
- for (int i = 0; i < nh; ++i) {
- for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);
- put(img_mark[1].data());
- }
- put(img_mark[2].data());
- return true;
- }
- // the row of a picture's span at position p (null: p is in none)
- const uint16_t* image_row(int64_t p) const {
- if (imgs.empty()) return nullptr;
- auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });
- if (it == imgs.begin()) return nullptr;
- const Image& im = *(it - 1);
- const int64_t r = p - im.start;
- if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;
- return im.rows->data() + (size_t) r * g.dim;
- }
- // token tk's embedding row at position p: inside a picture's span, the picture's
- const uint16_t* emb_row(int64_t p, int tk) const {
- const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;
- return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;
- }
- bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }
- // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)
- int32_t ctok(int64_t p, int tk) const {
- if (tk == img_tok && image_row(p) != nullptr) return -1;
- return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;
- }
- // a chunk's image rows (route()'s bits); false: none
- bool vl_bits(int64_t p0, int T, VlBits& b) const {
- if (imgs.empty()) return false;
- std::memset(&b, 0, sizeof b);
- bool any = false;
- for (int t = 0; t < T && t < kVlRows; ++t)
- if (in_image(p0 + t)) {
- b.w[t >> 5] |= 1u << (t & 31);
- any = true;
- }
- return any;
- }
-
- // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]
- // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a
- // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,
- // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.
- void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;
- const size_t per = (size_t) cols * (hd + hd / 32);
- b.clear();
- b.reads.reserve((size_t) T * cols);
- const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
- const Tensor* tw = pk.get(P + "weight");
- const Tensor* ts = pk.get(P + "scale");
- const int fmt = eg_fmt[(size_t) li];
- const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)
- const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
- const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)
- std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
- int64_t hits = 0;
- {
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- for (int t = 0; t < T; ++t) {
- const int64_t p = p0 + t;
- uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;
- uint8_t* scales = rows + (size_t) cols * hd;
- if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)
- std::memset(rows, 0, per);
- continue;
- }
- int64_t prod[8];
- bool blocked = false;
- for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)
- blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;
- const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];
- prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];
- }
- int64_t rolling = prod[0];
- for (int n = 1; n < pk.eg_ngram; ++n) {
- rolling ^= prod[n];
- for (int h = 0; h < pk.eg_heads; ++h) {
- const int c = (n - 1) * pk.eg_heads + h;
- const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];
- uint8_t* rd = rows + (size_t) c * hd;
- uint8_t* sd = scales + (size_t) c * (hd / 32);
- const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;
- if (const uint8_t* hit = eg_cache.find(key)) {
- std::memcpy(rd, hit, (size_t) hd);
- std::memcpy(sd, hit + hd, (size_t) hd / 32);
- ++hits;
- continue;
- }
- const auto fi = first.find(key);
- if (fi != first.end()) {
- b.dups.push_back({fi->second, {rd, sd}});
- continue;
- }
- first.emplace(key, (int) b.reads.size());
- EgRead r;
- r.file = tw->file;
- r.dst = rd;
- r.scales = sd;
- r.key = key;
- if (mx8) {
- r.mx8 = true;
- r.n = (uint32_t) (hd / 32 * 33);
- r.off = tw->off + (uint64_t) id * r.n;
- } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)
- r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);
- r.off = tw->off + (uint64_t) id * r.n;
- } else {
- r.n = (uint32_t) hd;
- r.off = tw->off + (uint64_t) id * hd;
- r.file2 = ts->file;
- r.n2 = (uint32_t) (hd / 32);
- r.off2 = ts->off + (uint64_t) id * (hd / 32);
- }
- b.reads.push_back(r);
- }
- }
- }
- }
- eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();
- eg_hits_n += hits + (int64_t) b.dups.size();
- eg_reads_n += (int64_t) b.reads.size();
- eg_io.submit(b);
- }
- // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)
- void eg_finish(EgBatch& b) {
- if (b.post.load(std::memory_order_acquire) == 2) return;
- const int64_t t0 = now_ns();
- eg_io.wait(b);
- int want = 0;
- if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {
- const int hd = pk.eg_hd;
- for (const auto& d : b.dups) {
- const EgRead& r = b.reads[(size_t) d.first];
- std::memcpy(d.second.first, r.dst, (size_t) hd);
- std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
- }
- if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite
- for (size_t i = 0; i < b.reads.size(); ++i) {
- const EgRead& r = b.reads[i];
- for (int k = 0; k < hd / 32; ++k) {
- uint16_t hb;
- std::memcpy(&hb, r.dst + (size_t) k * 18, 2);
- if ((hb & 0x7c00) == 0x7c00)
- std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),
- r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);
- }
- }
- }
- {
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- for (size_t i = 0; i < b.reads.size(); ++i) {
- const EgRead& r = b.reads[i];
- if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)
- std::memset(r.dst, 0, (size_t) hd);
- std::memset(r.scales, 0, (size_t) hd / 32);
- continue;
- }
- if (uint8_t* c = eg_cache.put(r.key)) {
- std::memcpy(c, r.dst, (size_t) hd);
- std::memcpy(c + hd, r.scales, (size_t) hd / 32);
- }
- }
- }
- eg_io_ns += b.t1.load() - b.t0;
- b.live = false;
- b.post.store(2, std::memory_order_release);
- } else {
- while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();
- }
- eg_wait_ns += now_ns() - t0;
- }
- void eg_stats() {
- st.eg_rows = eg_rows_n.load();
- st.eg_hits = eg_hits_n.load();
- st.eg_reads = eg_reads_n.load();
- st.eg_io_ms = (double) eg_io_ns.load() / 1e6;
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- st.eg_cached = (int64_t) eg_cache.size();
- st.eg_cache_cap = (int64_t) eg_cache.capacity();
- }
- // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)
- void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {
- EgBatch b;
- eg_issue(p0, T, only, dst, b);
- eg_finish(b);
- }
-
- // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its
- // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,
- // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's
- // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down
- // pieces; the down bands). yd[slot] gets each row's expert output.
- struct Ent {
- int e = 0, n = 0, slot0 = 0;
- int tok[kc::kExl3CpuMaxN] = {};
- };
- void run_nodes(int n, const std::function<void(int)>* f) {
- for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);
- pools[0]->help();
- for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();
- }
-
- // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each
- // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),
- // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits
- // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds
- // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever
- // number of entries take it, and each job its band of h straight into the down's int8 input.
- // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)
- void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
- if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);
- constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
- const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);
- const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);
- const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)
- const float lim = g.lim;
- static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");
- { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at
- // once, so the SSD reads them with large requests side by side while the first entries compute
- // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost
- // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
- std::vector<std::pair<const void*, size_t>> want;
- for (const Ent& en : ents)
- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
- os::prefetch_cold(want);
- }
- // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token
- // index past the window (none on the decode or chunk paths) gets a row of its own after them
- uint8_t* xq = (uint8_t*) c_xh.data();
- uint8_t* hq = xq + (size_t) kWin * xa;
- uint8_t* xq_extra = hq + (size_t) kSlots * ha;
- auto x_act = [&](const Ent& en, int i) -> const void* {
- const int t = en.tok[i];
- return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;
- };
- {
- bool done[kWin] = {};
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i) {
- const int t = en.tok[i];
- if (t < kWin && done[t]) continue;
- if (t < kWin) done[t] = true;
- gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));
- }
- }
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];
- float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = x_act(en, i);
- gp[i] = yg[i];
- up[i] = yu[i];
- }
- // (part X's rows, from where they are: a copy's part on node X, or the file's)
- gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);
- gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);
- for (int i = 0; i < en.n; ++i) {
- float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;
- for (int r = r0; r < r1; ++r) {
- float gv = yg[i][r - r0], uv = yu[i][r - r0];
- if (lim > 0.0f) {
- gv = std::min(gv, lim);
- uv = std::min(std::max(uv, -lim), lim);
- }
- h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;
- }
- if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);
- }
- };
- run_nodes(ne * GB, f);
- if (!band_q)
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i)
- gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float* yp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = hq + (size_t) (en.slot0 + i) * ha;
- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;
- }
- gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);
- };
- run_nodes(ne * DB, f);
- }
- // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input
- // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same
-#if defined(STRATA_METAL)
- // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD
- // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN
- // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the
- // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per
- // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).
- static bool la_enabled() { return la_depth() > 0; }
- static int la_depth() {
- static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)
- return std::min(d, 2);
- }
- struct LaPool {
- std::mutex m;
- std::condition_variable cv;
- std::deque<std::pair<const void*, size_t>> q;
- LaPool() {
- const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;
- for (int i = 0; i < nth; ++i)
- std::thread([this] {
- for (;;) {
- std::pair<const void*, size_t> r;
- {
- std::unique_lock<std::mutex> lk(m);
- cv.wait(lk, [&] { return !q.empty(); });
- r = q.front();
- q.pop_front();
- while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)
- }
- const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;
- const size_t len = r.second + ((uintptr_t) r.first - a);
- std::vector<unsigned char> vec((len + 16383) / 16384);
- bool cold = true;
- if (mincore((void*) a, len, (char*) vec.data()) == 0) {
- cold = false;
- for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }
- }
- if (cold) os::prefetch({r});
- }
- }).detach();
- }
- void push(const void* p, size_t n) {
- { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }
- cv.notify_one();
- }
- };
- std::mutex la_mu;
- std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed
- long la_hit = 0, la_all = 0, la_issued = 0;
- void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {
- if (l.nm != 1) return;
- static LaPool* pool = nullptr;
- static bool reg = false;
- if (pool == nullptr) {
- la_guess.assign(L.size(), {});
- pool = new LaPool();
- }
- if (std::getenv("DS4_CPU_LOG") && !reg) {
- reg = true;
- static Impl* self = this;
- std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });
- }
- for (const Ent& en : ents) {
- ++la_all;
- const auto& gs = la_guess[(size_t) l.il];
- if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;
- }
- if (la_n <= 0) return;
- cudaEventSynchronize(ev2);
- static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;
- for (int a = 1; a <= la_n; ++a) {
- Layer& nx = L[(size_t) (l.il + a)];
- std::vector<int> cpu;
- for (int r = 0; r < nx.k_exp; ++r) {
- const int e = h_idx_a[(a - 1) * kPicks + r];
- if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);
- }
- la_guess[nx.il] = cpu;
- for (int e : cpu)
- for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }
- }
- }
-#endif
- void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
- constexpr int RB = 64; // rows a job
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
- gg::IqCpu fm;
- std::string ferr;
- if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||
- fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {
- static std::atomic<bool> said{false};
- if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());
- for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);
- return;
- }
- static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;
- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }
- ++n_calls;
- double tp0 = now_ms();
- const float lim = g.lim;
- { // (os::prefetch, as above)
- std::vector<std::pair<const void*, size_t>> want;
- for (const Ent& en : ents)
- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
- if (std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)
- static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;
- if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }
- for (const auto& w : want) {
- const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;
- unsigned char vec[64];
- const size_t len = std::min<size_t>(w.second, 64 * 16384);
- bool cold = false;
- if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;
- const double t0 = now_ms();
- os::prefetch({w});
- (cold ? cold_ms : warm_ms) += now_ms() - t0;
- (cold ? cold_n : warm_n)++;
- }
- } else if (want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {
- // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of
- // all the call's experts at once keep the SSD busy)
- std::function<void(int)> pf[4];
- for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };
- run_nodes((int) want.size(), pf);
- } else
- os::prefetch(want);
- }
- double tp1 = now_ms(); t_pre += tp1 - tp0;
- // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of
- // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or
- // chunk paths), then (x no longer needed) the slot's h
- uint8_t* aq = (uint8_t*) c_xh.data();
- auto x_act = [&](const Ent& en, int i) -> uint8_t* {
- const int t = en.tok[i];
- return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;
- };
- {
- bool done[kWin] = {};
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i) {
- const int t = en.tok[i];
- if (t < kWin && done[t]) continue;
- if (t < kWin) done[t] = true;
- gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
- }
- }
- double tq1 = now_ms(); t_q1 += tq1 - tp1;
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float* hp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = x_act(en, i);
- hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)
- }
- // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any
- // block size of the type)
- gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,
- r0 - X * FFh, r1 - X * FFh, lim);
- };
- run_nodes(ne * GB, f);
- double tr1 = now_ms(); t_r1 += tr1 - tq1;
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i)
- gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
- aq + (size_t) (kWin + en.slot0 + i) * E * 4);
- double tq2 = now_ms(); t_q2 += tq2 - tr1;
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
- const void* hp[gg::kMxCpuMaxN];
- float* yp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;
- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)
- }
- gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
- };
- run_nodes(ne * DB, f);
- t_r2 += now_ms() - tq2;
- }
-
- // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
- struct StoreHold {
- Impl* m = nullptr;
- int layer = 0;
- std::vector<int> es;
- std::vector<char> ok;
- StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
- if (!m_->store) return;
- m = m_;
- layer = l.il;
- for (const Ent& en : ents)
- if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
- m->store->ensure(layer, es.data(), (int) es.size(), ok);
- }
- ~StoreHold() {
- if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
- }
- };
- void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
- const int* ord = nullptr) {
- if (ldx == 0) ldx = (size_t) g.dim;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- StoreHold hold(this, l, ents);
- return cpu_experts_mx(l, x, ents, ldx, ord);
- }
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
- const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
- const int ne = (int) ents.size();
- const float lim = g.lim;
- auto part = [&](int e, int X, int r) {
- const uint8_t* b = l.hh[(size_t) e][(size_t) X];
- kc::Exl3Part p;
- p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);
- p.suh = (const uint16_t*) (b + l.hl.suh[r]);
- p.svh = (const uint16_t*) (b + l.hl.svh[r]);
- p.k = l.hl.k[r];
- p.n = l.hl.n[r];
- p.K = l.lay.K[r];
- return p;
- };
- auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };
- auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };
- auto acc = [&](int X, int sl, int r, int q) {
- return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;
- };
- auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };
- auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };
- auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / (2 * GQ))];
- const int r = (j / GQ) % 2, q = j % GQ;
- const kc::Exl3Part p = part(en.e, X, r);
- const float* xp[kc::kExl3CpuMaxN];
- float* ap[kc::kExl3CpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- const float* xt = x + (size_t) en.tok[i] * ldx;
- for (int b = 2 * q; b < 2 * q + 2; ++b)
- xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);
- xp[i] = xh(X, sl, r);
- ap[i] = acc(X, sl, r, q);
- }
- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);
- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);
- };
- run_nodes(ne * 2 * GQ, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / GB)];
- const int b = j % GB;
- const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- float y[2][128], a[128], hb[128];
- for (int r = 0; r < 2; ++r) {
- for (int c = 0; c < 128; ++c) a[c] = 0.0f;
- for (int q = 0; q < GQ; ++q) {
- const float* aq = acc(X, sl, r, q) + b * 128;
- for (int c = 0; c < 128; ++c) a[c] += aq[c];
- }
- double xs = 0.0;
- for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];
- kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);
- }
- for (int c = 0; c < 128; ++c) {
- float gv = y[0][c], uv = y[1][c];
- if (lim > 0.0f) {
- gv = std::min(gv, lim);
- uv = std::min(std::max(uv, -lim), lim);
- }
- hb[c] = gv / (1.0f + std::exp(-gv)) * uv;
- }
- const int gb = X * GB + b;
- xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);
- }
- };
- run_nodes(ne * GB, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / DQ)];
- const int q = j % DQ;
- const kc::Exl3Part p = part(en.e, X, 2);
- const float* xp[kc::kExl3CpuMaxN];
- float* ap[kc::kExl3CpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- xp[i] = xhd(en.slot0 + i);
- ap[i] = accd(X, en.slot0 + i, q);
- }
- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);
- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);
- };
- run_nodes(ne * DQ, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / DB)];
- const int b = j % DB;
- const kc::Exl3Part pd = part(en.e, X, 2);
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- float a[128];
- for (int c = 0; c < 128; ++c) a[c] = 0.0f;
- for (int q = 0; q < DQ; ++q) {
- const float* aq = accd(X, sl, q) + b * 128;
- for (int c = 0; c < 128; ++c) a[c] += aq[c];
- }
- double xs = 0.0;
- for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];
- kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);
- }
- };
- run_nodes(ne * DB, f);
- }
-
- // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it
- // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs
- // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's
- // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:
- // the CPU lane reads it from the file again.
- void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot
- for (;;) {
- RamSwap rs;
- {
- std::unique_lock<std::mutex> lk(sw_mu);
- ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });
- if (quit.load()) return;
- rs = ram_queue.front();
- ram_queue.pop_front();
- }
- const Layer& l = L[(size_t) rs.layer];
- if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {
- for (int r = 0; r < 3; ++r) // (one copy of it in RAM)
- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],
- mx_part_bytes(l, r), false);
- } else {
- rs.in = -1; // (failed: the slot goes back)
- }
- std::lock_guard<std::mutex> lk(sw_mu);
- ram_ready.push_back(rs);
- }
- }
-
- void cache_loop(SwapBufs& B) {
- for (;;) {
- Swap sw;
- {
- std::unique_lock<std::mutex> lk(sw_mu);
- sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });
- if (quit.load()) return;
- sw = sw_queue.front();
- sw_queue.pop_front();
- }
- Layer& l = L[(size_t) sw.layer];
- if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)
- swap_in_mx(l, sw, B);
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_ready.push_back(sw);
- continue;
- }
- parts_to_blob(l, l.hh[(size_t) sw.in], B.a);
- if (l.nm == 1) {
- const Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t cs = B.st[(size_t) l.dev];
- cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);
- if (!file_experts)
- cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,
- cudaMemcpyDeviceToHost, cs);
- cudaStreamSynchronize(cs);
- } else { // a group: each member's part in, each member's part out
- for (int m = 0; m < l.nm; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = B.st[(size_t) x.dev];
- blob_to_half(l, B.a, m, B.h);
- cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);
- cudaStreamSynchronize(cs);
- if (file_experts) continue;
- cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,
- cudaMemcpyDeviceToHost, cs);
- cudaStreamSynchronize(cs);
- half_to_blob(l, B.h, m, B.b);
- }
- }
- if (!file_experts)
- for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);
- if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte
- std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);
- bool ok = true;
- for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)
- const int e = pass == 0 ? sw.in : sw.out;
- reg.resize(l.region_bytes[(size_t) e]);
- pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());
- if (pass == 0) {
- repack(l, reg.data(), blob.data());
- std::vector<uint8_t> dev(l.blob);
- if (l.nm == 1)
- cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);
- else
- for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)
- std::vector<uint8_t> h(member(l, m).hblob);
- cudaSetDevice(devs[(size_t) member(l, m).dev].id);
- cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,
- cudaMemcpyDeviceToHost);
- half_to_blob(l, h.data(), m, dev.data());
- }
- for (int r = 0; r < 3; ++r)
- ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&
- std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&
- std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;
- } else {
- for (int X = 0; X < NH; ++X) {
- repack_part(l, reg.data(), X, part.data());
- for (int r = 0; r < 3; ++r) {
- const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];
- ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&
- std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&
- std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;
- }
- }
- }
- }
- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,
- ok ? "both match the model file" : "MISMATCH");
- }
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_ready.push_back(sw);
- }
- }
-
- // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);
- // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)
- // from the layer's spare RAM slot, which it is copied into from VRAM here
- void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {
- uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)
- const std::vector<cudaStream_t>& copy_st = B.st;
- bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till
- // this swap's switch)
- for (int m = 0; m < l.nm && ok; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = copy_st[(size_t) x.dev];
- if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);
- ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),
- cudaMemcpyHostToDevice, cs) == cudaSuccess &&
- cudaStreamSynchronize(cs) == cudaSuccess;
- }
- if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot
- bool got = ok;
- for (int m = 0; m < l.nm && got; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = copy_st[(size_t) x.dev];
- got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),
- unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&
- cudaStreamSynchronize(cs) == cudaSuccess;
- if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);
- }
- if (got) blob_to_ram(l, sw_b, sw.ram);
- else sw.ram = -1; // (back to the file's pages then)
- }
- if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte
- std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);
- for (int m = 0; m < l.nm && ok; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),
- cudaMemcpyDeviceToHost);
- if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());
- }
- ok = ok && read_mx_file(l, sw.in, file.data());
- for (int r = 0; r < 3 && ok; ++r)
- ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
- bool ok_out = true;
- if (sw.ram >= 0) {
- ok_out = read_mx_file(l, sw.out, file.data());
- ram_to_blob(l, sw.ram, dev.data());
- for (int r = 0; r < 3 && ok_out; ++r)
- ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
- }
- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,
- sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");
- } else if (!ok) {
- std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);
- }
- }
-
- void cache_tick(int T) {
- cache_tick_swaps(T);
- store_sync();
- store_watch();
- }
- // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time
- // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free
- void store_watch() {
- if (!store) return;
- static int tick = 0;
- if (++tick % 16 != 0) return;
- const uint64_t avail = ram_avail();
- if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;
- const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));
- const size_t n = store->shrink_chunks(chunks);
- if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());
- }
- void cache_tick_swaps(int T) {
- if (!o.expert_cache || cache_ths.empty()) return;
- std::vector<Swap> ready;
- std::vector<RamSwap> rready;
- {
- std::lock_guard<std::mutex> lk(sw_mu);
- ready.swap(sw_ready);
- rready.swap(ram_ready);
- }
- for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)
- --ram_inflight;
- Layer& l = L[(size_t) rs.layer];
- if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {
- ram_free.push_back(rs.slot);
- continue;
- }
- l.ram[(size_t) rs.in] = rs.slot;
- if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {
- ram_free.push_back(l.ram[(size_t) rs.out]);
- l.ram[(size_t) rs.out] = -1;
- }
- ++st.ram_moves;
- }
- for (const Swap& sw : ready) {
- Layer& l = L[(size_t) sw.layer];
- const int old_slot = l.slot[(size_t) sw.out];
- const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];
- if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)
- const int s_in = l.ram[(size_t) sw.in];
- l.ram[(size_t) sw.in] = -1;
- if (sw.ram >= 0) {
- l.ram[(size_t) sw.out] = sw.ram;
- copy_spare[(size_t) l.il] = s_in;
- } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)
- --ram_held;
- if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;
- }
- } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)
- ram_free.push_back(l.ram[(size_t) sw.in]);
- l.ram[(size_t) sw.in] = -1;
- }
- l.slot[(size_t) sw.in] = l.spare_vram;
- if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)
- l.zc[(size_t) sw.in] = 0;
- store->unmark(l.il, sw.in);
- }
- if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
- l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
- l.host[(size_t) sw.in] = nullptr;
- l.slot[(size_t) sw.out] = -1;
- if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)
- l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);
- } else if (file_experts) { // (back to its pages in the file)
- l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};
- l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];
- } else {
- l.hh[(size_t) sw.out] = l.spare_host;
- l.host[(size_t) sw.out] = l.spare_host[0];
- l.spare_host = old_parts;
- }
- l.spare_vram = old_slot;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- const Dev& dm = devs[(size_t) x.dev];
- l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));
- l.tab_pin[2 * m + 1] = 0ull;
- cudaSetDevice(dm.id); // (stream order: before the next window)
- cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);
- cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);
- }
- l.swap_pending = false;
- --sw_inflight;
- ++st.swaps;
- }
- decay_tokens += T;
- if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens
- decay_tokens = 0;
- for (Layer& l : L)
- for (float& h : l.heat) h *= c_decay;
- }
- bool queued = false, rqueued = false;
- for (Layer& l : L) {
- if (l.cpu_hits.empty()) continue;
- std::vector<int> hits;
- hits.swap(l.cpu_hits);
- int best = -1, vin = -1;
- for (int e : hits)
- if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;
- if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in
- int cold = -1;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;
- if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {
- l.swap_pending = true;
- ++sw_inflight;
- vin = best;
- Swap sw{l.il, best, cold};
- if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_queue.push_back(sw);
- queued = true;
- }
- }
- if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;
- int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)
- for (int e : hits)
- if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&
- (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))
- rb = e;
- if (rb < 0) continue;
- int rc = -1;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;
- const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)
- if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;
- RamSwap rs;
- rs.layer = l.il;
- rs.in = rb;
- rs.out = room ? -1 : rc;
- rs.slot = ram_free.back();
- ram_free.pop_back();
- ++ram_inflight;
- std::lock_guard<std::mutex> lk(sw_mu);
- ram_queue.push_back(rs);
- rqueued = true;
- }
- if (queued) sw_cv.notify_all();
- if (rqueued) ram_cv.notify_one();
- }
-
- // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes
- void prefetch_loop() {
- std::vector<std::pair<const void*, size_t>> b;
- for (;;) {
- {
- std::unique_lock<std::mutex> lk(pf_mu);
- pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });
- if (quit.load(std::memory_order_relaxed)) return;
- b.swap(pf_q);
- }
- os::prefetch_cold(b); // (the resident ones would only cost the page walk)
- b.clear();
- }
- }
- // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)
- static bool file_read(const Layer& l, int e) {
- return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);
- }
- // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4
- // experts read in place, some of them from the files now (none: nothing to read early - null)
- const Layer* ahead_layer(const Layer& l0, int a) const {
- const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;
- if (j >= end || j >= (int) L.size()) return nullptr;
- const Layer& n = L[(size_t) j];
- if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||
- (int) n.slot.size() < n.n_exp)
- return nullptr;
- for (int e = 0; e < n.n_exp; ++e)
- if (file_read(n, e)) return &n;
- return nullptr;
- }
- // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the
- // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts
- std::vector<std::pair<const void*, size_t>> pf_want;
- void ahead_ask(int li, const Req& r, unsigned seq, int T) {
- pf_want.clear();
- for (int a = 1; a <= 2; ++a) {
- const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;
- if (m <= 0 || j >= (int) L.size()) continue;
- const Layer& n = L[(size_t) j];
- uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;
- uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;
- if (ahead_for[(size_t) j] != seq + (unsigned) a) {
- std::memset(fl, 0, 1024);
- std::memset(rk, 0, 1024);
- ahead_for[(size_t) j] = seq + (unsigned) a;
- }
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < m; ++i) {
- const int e = r.ahead[t][(a - 1) * kAhead1 + i];
- if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;
- fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);
- if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
- if ((fl[e] & 8) || !file_read(n, e)) continue;
- fl[e] |= 8;
- if (store) store->prefetch(j, e);
- else
- for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
- ++st.ahead_pf;
- }
- }
- if (pf_want.empty()) return;
- {
- std::lock_guard<std::mutex> lk(pf_mu);
- pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());
- }
- pf_cv.notify_one();
- }
-
- // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's
- // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.
- void serve_lane() {
- unsigned next = 1;
- auto last = std::chrono::steady_clock::now();
- std::vector<Ent> ents;
- while (!quit.load(std::memory_order_relaxed)) {
- Req& r = reqs[next % kRing];
- if (*(volatile unsigned*) &r.seq != next) {
- cpu_relax();
- if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))
- std::this_thread::sleep_for(std::chrono::microseconds(20));
- continue;
- }
- std::atomic_thread_fence(std::memory_order_acquire);
- const int li = layer_of[next % kRing].load(std::memory_order_acquire);
- Resp& rp = resps[next % kRing];
- const int T = std::max(1, std::min(kWin, r.T));
- for (int t = 0; t < T; ++t) rp.cpu[t] = 0;
- if (li >= 0 && li < (int) L.size()) {
- Layer& l = L[(size_t) li];
- uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
- ents.clear();
- if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)
- // lookahead: the guesses made for this layer (when they are for this request), counted on decode
- const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
- ? ahead_fl.data() + (size_t) li * 1024 : nullptr;
- // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row
- float wv[kWin * kPicks];
- int tok_of[kWin * kPicks];
- int nslots = 0;
- std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
- if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids
- // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the
- // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's
- // predecessors (the request carries the guesses for the layers after it)
- static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;
- uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};
- std::fwrite(hd, 1, 2, trace_f);
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < l.k_exp; ++i) {
- const int16_t v = (int16_t) r.ids[t][i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- if (with_ahead) {
- const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};
- std::fwrite(na, 1, 2, trace_f);
- for (int t = 0; t < T; ++t) {
- for (int i = 0; i < na[0]; ++i) {
- const int16_t v = (int16_t) r.ahead[t][i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- for (int i = 0; i < na[1]; ++i) {
- const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- }
- }
- }
- const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
- for (int t = 0; t < T; ++t) {
- float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)
- if (dsk)
- for (int i = 0; i < l.k_exp; ++i)
- if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];
- for (int i = 0; i < l.k_exp; ++i) {
- const int e = r.ids[t][i];
- if (e < 0 || e >= l.n_exp) continue;
- ++pr[e];
- l.heat[(size_t) e] += 1.0f;
- if (fl != nullptr) {
- ++st.ahead_picks;
- st.ahead_hit_k += fl[e] & 1;
- st.ahead_hit += (fl[e] & 6) != 0;
- st.ahead_hit2 += (fl[e] >> 2) & 1;
- }
- if (l.slot[(size_t) e] < 0) {
- l.cpu_hits.push_back(e);
- if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot
- store->touch(li, e);
- ++st.store_zc;
- continue;
- }
- if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
- ++st.skipped;
- continue;
- }
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;
- else if (fl != nullptr) {
- ++st.ahead_cold;
- st.ahead_cold_hit += (fl[e] >> 3) & 1;
- }
- picks.push_back({e, t * kPicks + i});
- }
- }
- }
- if (fl != nullptr) { // the guesses by rank: how many were picked
- const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;
- for (int e = 0; e < l.n_exp && e < 1024; ++e) {
- if (rk[e] == 0 || rk[e] > 16) continue;
- bool hit = false;
- for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;
- ++st.ahead_rank_n[rk[e] - 1];
- st.ahead_rank_hit[rk[e] - 1] += hit;
- }
- }
- if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);
- std::sort(picks.begin(), picks.end());
- for (size_t a = 0; a < picks.size();) {
- Ent en;
- en.e = picks[a].first;
- en.slot0 = nslots;
- en.n = 0;
- while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {
- const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;
- en.tok[en.n++] = t;
- tok_of[nslots] = t;
- wv[nslots] = r.w[t][i];
- ++nslots;
- ++a;
- }
- ents.push_back(en);
- }
- // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for
- // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)
- int ord[kWin * kPicks];
- const int* po = nullptr;
- if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&
- ents.size() <= (size_t) (kWin * kPicks)) {
- const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;
- int n = 0;
- for (int pass = 0; pass < 2; ++pass)
- for (size_t q = 0; q < ents.size(); ++q) {
- const int e = ents[q].e;
- const bool early = !file_read(l, e) || (fa[e] & 8) != 0;
- if (early == (pass == 0)) ord[n++] = (int) q;
- }
- po = ord;
- }
- if (!ents.empty()) {
- const double tc = now_ms();
- cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);
- const int D = g.dim;
- for (int t = 0; t < T; ++t) {
- bool any = false;
- for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;
- if (!any) continue;
- float* o = rp.part[t];
- for (int e = 0; e < D; ++e) o[e] = 0.0f;
- for (int sl = 0; sl < nslots; ++sl) {
- if (tok_of[sl] != t) continue;
- const float* y = c_yd.data() + (size_t) sl * D;
- for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];
- }
- rp.cpu[t] = 1;
- }
- st.cpu_ms += now_ms() - tc;
- st.cpu_experts += nslots;
- }
- }
- std::atomic_thread_fence(std::memory_order_release);
- *(volatile unsigned*) &rp.seq = next;
- ++next;
- last = std::chrono::steady_clock::now();
- }
- }
-
- // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's
- static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {
- bool g = false;
- for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;
- return g ? gg::mv(j, n, s) : x3::mv(j, n, s);
- }
- bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {
- x3::MvJob j[x3::kMaxMvJobs];
- int n = 0;
- for (const x3::MvJob& x : jobs) j[n++] = x;
- if (!mv_any(j, n, d.st)) {
- err = "exl3 mv failed";
- return false;
- }
- return true;
- }
- bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {
- for (int a = 0; a < n; a += x3::kMaxMvJobs)
- if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {
- err = "exl3 mv failed";
- return false;
- }
- return true;
- }
- // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs
- static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,
- float alpha = 1.0f) {
- x3::MvJob j;
- j.m = &m;
- j.xf = x;
- j.y = y;
- j.alpha = alpha;
- j.ncol = T;
- j.xf_cs = x_cs;
- j.y_cs = y_cs;
- return j;
- }
-
- // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:
- // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm
- // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x
- // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),
- // the trunk's head, the markov bias token by token (greedy) and the confidence head.
- bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {
- if (g.n_mtp == 0 || n <= 0) return true;
- if (n > kWin * kPicks) {
- err = "seeding the drafter with " + std::to_string(n) + " rows";
- return false;
- }
- Dev& d = devs[(size_t) d_dev];
- cudaSetDevice(d.id);
- const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;
- if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps
- for (Dev& e : devs) {
- cudaSetDevice(e.id);
- CK(cudaStreamSynchronize(e.st));
- }
- cudaSetDevice(d.id);
- const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};
- std::fwrite(hdr, 8, 4, dump_d);
- std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);
- std::fflush(dump_d);
- }
- for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)
- const int na = std::min(kWin, n - a);
- CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,
- cudaMemcpyHostToDevice, d.st));
- if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;
- }
- CK(cudaEventRecord(ev_seed, d.st));
- Finish fm;
- fm.w = d_main_norm;
- fm.eps = g.eps;
- finish(d.ed, n, D, D, fm, d.st);
- for (int i = 0; i < g.n_mtp; ++i) {
- Layer& l = L[(size_t) (g.nl + i)];
- if (l.dev != d_dev) {
- err = "the DSpark blocks are split across devices";
- return false;
- }
- for (int a = 0; a < n; a += kWin) {
- const int na = std::min(kWin, n - a);
- if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) (p0 + a);
- fk.freqs = d.freqs_win;
- fk.quant = 1;
- fk.dst = d.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(d.kvr, na, HD, HD, fk, d.st);
- ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);
- }
- }
- return true;
- }
-
- bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,
- int top_k, float min_p, uint64_t seed, uint64_t counter) {
- const double t0 = now_ms();
- drafts_q = temperature > 0.0f && sample_drafts;
- dq_temp = temperature;
- dq_top_p = top_p;
- dq_top_k = top_k;
- dq_min_p = min_p;
- dq_seed = seed;
- dq_counter = counter;
- const bool ok = draft_(y, out, conf, err);
- if (!ok) drafts_q = false;
- if (max_drafts > 0 && (int) out.size() > max_drafts) {
- out.resize((size_t) max_drafts);
- conf.resize((size_t) max_drafts);
- }
- st.draft_ms += now_ms() - t0;
- ++st.drafts_rounds;
- return ok;
- }
- bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {
- out.clear();
- conf.clear();
- const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;
- const int64_t p0 = pos;
- if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;
- int tk[kWin];
- tk[0] = y;
- for (int t = 1; t < T; ++t) tk[t] = g.d_noise;
- Dev& d = devs[(size_t) d_dev];
- cudaSetDevice(d.id);
- for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, d.st);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float hh[4] = {0, 0, 0, 0};
- cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, d.st);
- WinPlan dp; // (window attention alone: the mailbox seqs)
- dp.async = !sync_decode && T <= async_max_t;
- for (int i = 0; i < g.n_mtp && dp.async; ++i) {
- const int il = g.nl + i;
- dp.seq[il] = ++seq_issued;
- layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);
- }
- const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;
- if (dump_d != nullptr && h_dump == nullptr &&
- (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))
- return false;
- for (int i = 0; i < g.n_mtp; ++i) {
- Layer& l = L[(size_t) (g.nl + i)];
- if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)
- ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);
- CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));
- }
- if (!layer(l, 0, p0, T, dp, err)) return false;
- if (dump_d != nullptr) {
- cudaSetDevice(d.id);
- CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));
- }
- }
- cudaSetDevice(d.id);
- hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)
- Dev& hd = devs[(size_t) head_dev];
- // (through the host to the head's device, and to the head group's other members - the drafter's device can be
- // the head group's lead itself, a group of every GPU)
- if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {
- CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));
- CK(cudaEventRecord(d.ev_out, d.st));
- }
- if (d_dev != head_dev) {
- cudaSetDevice(hd.id);
- CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));
- CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));
- }
- // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)
- const std::vector<int>& hg = groups[(size_t) hd.grp].mem;
- const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)
- for (int m = (int) hg.size() - 1; m >= 0; --m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- if (m > 0) {
- CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));
- CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));
- } else
- CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));
- Finish fn;
- fn.w = e.d_norm;
- fn.eps = g.eps;
- finish(e.ao, T, D, D, fn, e.st);
- if (!head_share(e, m, e.ao, T, err, !mk)) return false;
- }
- h_idx[0] = y;
- if (mk) {
- // the markov steps over the group: each member its rows of the bias and its share's argmax, every member
- // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,
- // each step ~1/nm of their 66 MB and no gather of the shares
- const int nm = (int) hg.size();
- const unsigned seq = ++mk_seq;
- for (int m = 0; m < nm; ++m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));
- CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));
- }
- for (int i = 0; i < T; ++i)
- for (int m = 0; m < nm; ++m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;
- markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);
- argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);
- markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,
- e.st);
- }
- cudaSetDevice(hd.id);
- } else {
- CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));
- strata::dsv41k::SamplerParams sp;
- sp.greedy = true;
- sp.temperature = 0.0f;
- if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)
- sp.greedy = false;
- sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);
- sp.top_p = dq_top_p;
- sp.top_k = dq_top_k;
- sp.min_p = dq_min_p;
- sp.seed = dq_seed;
- }
- for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
- markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
- if (drafts_q) {
- sp.counter = dq_counter + (uint64_t) i;
- strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);
- } else
- strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);
- }
- }
- confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
- int tok[kWin + 1];
- float cf[kWin];
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];
- for (int i = 0; i < T; ++i) cf[i] = h_w[i];
- if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden
- std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream
- CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));
- cudaSetDevice(d.id);
- CK(cudaStreamSynchronize(d.st));
- const int64_t hdr[4] = {'D', p0, y, T};
- std::fwrite(hdr, 8, 4, dump_d);
- std::fwrite(tok + 1, 4, (size_t) T, dump_d);
- std::fwrite(cf, 4, (size_t) T, dump_d);
- std::fwrite(xs.data(), 4, xs.size(), dump_d);
- for (int i = 0; i < g.n_mtp; ++i) {
- std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);
- std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);
- }
- std::fflush(dump_d);
- cudaSetDevice(hd.id);
- }
- for (int i = 0; i < T; ++i) {
- out.push_back(tok[i + 1]);
- conf.push_back(cf[i]);
- }
- return true;
- }
-
- int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,
- const float* bias) {
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- strata::dsv41k::SamplerParams sp;
- sp.temperature = temperature;
- sp.greedy = temperature <= 0.0f;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- sp.logit_bias = bias;
- strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
- int t = -1;
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- return t;
- }
-
- // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.
- // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own
- // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution
- // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the
- // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.
- bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
- uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {
- const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);
- const int T = n + 1, V = g.vocab;
- const int64_t p0 = pos;
- int tk[kWin];
- tk[0] = y;
- for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];
- spec_win = true;
- const double tv = now_ms();
- const bool ok = forward(tk, T, T, err, 0);
- st.verify_ms += now_ms() - tv;
- spec_win = false;
- if (!ok) return false;
- Dev& hd = devs[(size_t) head_dev];
- cudaSetDevice(hd.id);
- int m = 0;
- if (temperature <= 0.0f) {
- strata::dsv41k::SamplerParams sp;
- sp.greedy = true;
- sp.temperature = 0.0f;
- strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
- next = h_idx[m];
- } else {
- // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)
- // (never d itself): each row a token distributed as the model's own draw at that position, p through the
- // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,
- // never kept); past the first rejection the rows do not count
- const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&
- dq_min_p == min_p && dq_seed == seed && dq_counter == counter;
- if (!own) { // the drafter's argmax: q a point at each draft
- for (int i = 0; i < n; ++i) {
- h_specq[(size_t) i].n = 1;
- h_specq[(size_t) i].ids[0] = drafts[(size_t) i];
- h_specq[(size_t) i].p[0] = 1.0;
- }
- CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),
- cudaMemcpyHostToDevice, hd.st));
- }
- for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];
- CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));
- strata::dsv41k::SamplerParams sp;
- sp.greedy = false;
- sp.temperature = temperature;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
- next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);
- }
- drafts_q = false;
- rollback(p0, T, m + 1);
- if (!seed_drafter(p0, m + 1, 0, err)) return false;
- accepted = m;
- st.drafts_offered += n;
- st.drafts_accepted += m;
- for (int i = 0; i < n && i <= m && i < 8; ++i) {
- ++st.draft_reached[i];
- if (i < m) ++st.draft_kept[i];
- }
- return true;
- }
-
- // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone
- // (the compressed KV and the index keys past them are written again when those positions come)
- void rollback(int64_t p0, int T, int keep) {
- if (keep >= T) return;
- // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor
- // parallel, a millisecond of the host's before the next draft)
- std::vector<RingBatch> rb(devs.size());
- for (int il = 0; il < g.nl; ++il)
- for (int m = 0; m < L[(size_t) il].nm; ++m) {
- Layer& l = member(L[(size_t) il], m);
- RingBatch& b = rb[(size_t) l.dev];
- if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)
- cudaSetDevice(devs[(size_t) l.dev].id);
- ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);
- continue;
- }
- b.ring[b.n] = l.win;
- b.bak[b.n++] = l.ring_bak;
- }
- for (size_t di = 0; di < devs.size(); ++di)
- if (rb[di].n > 0) {
- cudaSetDevice(devs[di].id);
- ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);
- }
- for (int il = 0; il < g.nl; ++il)
- for (int m = 0; m < L[(size_t) il].nm; ++m) {
- Layer& l = member(L[(size_t) il], m);
- Dev& d = devs[(size_t) l.dev];
- if (l.c_snap != nullptr) {
- cudaSetDevice(d.id);
- const size_t ns = (size_t) l.ratio * g.hd * 4;
- cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);
- cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);
- compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,
- g.hd, l.c_norm, g.eps, d.latent, d.st);
- }
- }
- pos = p0 + keep;
- toks.resize((size_t) pos);
- ctoks.resize((size_t) pos);
- }
-
- // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
- bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }
-
-
- // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
- // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
- // place, the CPU lane's streamed in batches)
- bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {
- Layer& l = member(L0, mi);
- Dev& d = devs[(size_t) l.dev];
- Chunk& c = *chunks[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- prof_mark(d, 0);
- const int nm = L0.nm;
- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
- const ArGroup& ab = groups[(size_t) L0.grp].arb;
- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
- // Engram, in slabs of 256 tokens
- if (l.eg >= 0) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- const size_t per = (size_t) cols * (ehd + ehd / 32);
- const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)
- uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;
- CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.eg_ev[slot], s));
- for (int s0 = 0; s0 < T; s0 += 256) {
- const int ns = std::min(256, T - s0);
- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
- x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
- if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
- engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
- }
- }
- // attention
- if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
- dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
- hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
- hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
- dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
- dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);
- dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);
- dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);
- dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);
- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
- return false;
- if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
- if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
- dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);
- dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
- // the window rows: the ring's (positions p0 - win ..) then the chunk's
- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = c.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(c.kvr, T, HD, HD, fk, s);
- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
- if (snap) {
- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
- }
- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
- const uint8_t* ckv = nullptr;
- const long long ss = (long long) o.max_context + 64;
- if (l.ratio > 0) {
- Src& sr = srcs[(size_t) l.src];
- const int R = l.ratio;
- const int64_t g_first = p0 / R, g_end = (p0 + T) / R;
- const int ng = (int) std::max<int64_t>(0, g_end - g_first);
- const float* latent = c.latent;
- if (l.kv_src) {
- if (R == 1) {
- Finish fc;
- fc.w = l.c_norm;
- fc.eps = g.eps;
- finish(c.ckr, T, HD, HD, fc, s);
- latent = c.ckr;
- } else {
- if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)
- const int64_t ps = p0 + wp.snap_t;
- for (int sl = 0; sl < R; ++sl) {
- const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;
- const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;
- const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;
- CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
- }
- }
- compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);
- }
- }
- const int clen_max = (int) ((p0 + T) / R);
- if (l.ix_src) {
- Src& ks = srcs[(size_t) l.ik_src];
- if (l.owns_k && ng > 0) {
- x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);
- if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;
- Finish fi;
- fi.w = l.ik_norm;
- fi.eps = g.eps;
- fi.rd = g.rd;
- fi.pos = (int) (g_first * R);
- fi.pos_step = R;
- fi.freqs = freqs;
- fi.quant = 3;
- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- fi.pdst_stride = (long long) ik_rb();
- finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
- }
- if (clen_max > 0) {
- if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
- return false;
- for (int a0 = 0; a0 < T; a0 += c.AI) {
- const int na = std::min(c.AI, T - a0);
- const int64_t pa = p0 + a0;
- if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,
- g.ix_heads * g.ix_hd, 0.0f, err))
- return false;
- Finish fi;
- fi.rd = g.rd;
- fi.pos = (int) pa;
- fi.vpp = g.ix_heads;
- fi.freqs = freqs;
- fi.quant = 3;
- finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
- Rows lim;
- lim.gen = 1;
- lim.p0 = pa;
- lim.R = R;
- lim.kcap = g.ix_topk;
- lim.ccap = g.cand_topk;
- lim.blk = std::max(1, g.cand_blk);
- const int nlast = (int) ((pa + na) / R); // the slab's widest query
- int* tk = c.topk + (size_t) a0 * topk_stride();
- int* cd = c.cand + (size_t) a0 * cand_stride();
- const float* iw = c.iw + (size_t) a0 * 64;
- if (nlast <= 0) continue;
- if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {
- Rows cl = lim;
- cl.gen = 2;
- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,
- cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);
- Rows tr = lim;
- tr.gen = 3;
- topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);
- } else {
- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,
- lim, c.scores, ss, s);
- if (l.cand_src) {
- Rows cr = lim;
- cr.kcap = g.cand_topk;
- cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,
- (int) cand_stride(), s);
- }
- topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);
- }
- }
- }
- }
- if (l.kv_src && ng > 0) {
- Finish fc;
- fc.rd = g.rd;
- fc.pos = (int) (g_first * R);
- fc.pos_step = R;
- fc.freqs = freqs;
- fc.quant = 2;
- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
- fc.pdst_stride = (long long) ckv_rb();
- finish(const_cast<float*>(latent), ng, HD, HD, fc, s);
- if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once
- const int slot = wp.k % 2; // they took the chunk before last's)
- if (wp.k >= 2)
- for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);
- Xfer x;
- x.src[0] = fc.pdst;
- x.dst[0] = sr.stage(slot);
- x.bytes[0] = (size_t) ng * ckv_rb();
- x.n = 1;
- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();
- x.bytes[1] = (size_t) ng * ik_rb();
- x.n = 2;
- }
- xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);
- }
- }
- ckv = sr.ckv[(size_t) l.dev];
- }
- // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
- if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
- for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
- const int na = std::min(c.A, T - a0);
- if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;
- Finish fr;
- fr.rd = g.rd;
- fr.pos = (int) (p0 + a0);
- fr.vpp = NHL;
- fr.freqs = freqs;
- finish(c.q, na * NHL, HD, HD, fr, s);
- if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);
- AttnRows ar;
- ar.gen = 1;
- ar.p0 = p0;
- ar.t0 = a0;
- ar.win = g.win;
- ar.R = l.ratio;
- ar.kcap = g.ix_topk;
- sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
- na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
- if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);
- Finish fo;
- fo.rd = g.rd;
- fo.pos = (int) (p0 + a0);
- fo.vpp = NHL;
- fo.freqs = freqs;
- fo.inverse = 1;
- fo.quant = 1;
- finish(c.o, na * NHL, HD, HD, fo, s);
- if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);
- x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
- const int per = HQ / OG;
- for (int gi = 0; gi < OG; ++gi)
- if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,
- OG * g.o_lora, 0.0f, err))
- return false;
- x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
- if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
- if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);
- }
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);
- hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)
- prof_mark(d, 1);
- // the MoE
- if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
- hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);
- hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);
- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
- gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)
- for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)
- const int nr = std::min(kVlRows, T - r0);
- VlBits vb;
- const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);
- route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,
- c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);
- }
- const int n_vs = L0.n_slots + (int) L0.host_list.size();
- CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)
- moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);
- CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));
- // the shared expert into acc while the host waits for the bounds
- if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))
- return false;
- swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);
- x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);
- if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;
- prof_mark(d, 2);
- CK(cudaStreamSynchronize(s));
- std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);
- if (dump_r != nullptr && mi == 0) {
- const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;
- std::vector<int> ri(np), rt(nk);
- std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);
- CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
- cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};
- std::fwrite(hdr, 8, 8, dump_r);
- std::fwrite(ri.data(), 4, np, dump_r);
- std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);
- std::fwrite(b.data(), 4, b.size(), dump_r);
- std::fwrite(rt.data(), 4, nk, dump_r);
- std::fwrite(gl.data(), 4, gl.size(), dump_r);
- std::fwrite(gb.data(), 4, gb.size(), dump_r);
- std::fflush(dump_r);
- }
- if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids
- std::vector<int> ri((size_t) T * kPicks);
- CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};
- std::fwrite(hd, 1, 3, trace_f);
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < l.k_exp; ++i) {
- const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- }
- if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e) {
- const int v = L0.vs_pin[e];
- pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);
- }
- st.experts += (int64_t) T * l.k_exp;
- }
- if (c.flow) {
- if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 3);
- return true;
- }
- // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones
- // in shares between them (the device works while the copies run)
- const int di = l.dev, P = Chunk::kParts;
- int nb = 0;
- for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;
- int v_done = 0;
- for (int bi = 0; bi < nb; ++bi) {
- const Batch& bt = S.b[S.kp];
- const int half = (int) (S.kp % P);
- CK(cudaStreamWaitEvent(s, c.filled[half], 0));
- const int v0 = L0.n_slots + bt.j0;
- if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,
- b.data(), v0, v0 + bt.n, T, err))
- return false;
- CK(cudaEventRecord(c.freed[half], s));
- ++S.kp;
- if (!copy_ahead(d, c, S, di, err)) return false;
- const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);
- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;
- v_done = v_to;
- }
- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 3);
- return true;
- }
-
- // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)
- void plan_streams() {
- for (Layer& l : L) {
- if (l.mtp || l.vs_pin == nullptr) continue;
- l.host_list.clear();
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);
- for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)
- Layer& x = member(l, m);
- int j = 0;
- for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;
- }
- }
- }
-
- // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows
- Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {
- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- void* dv[4] = {c.h, c.pre, c.topk, c.cand};
- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
- const size_t nb[4] = {nh, np, nk, nc};
- Xfer x;
- x.n = 4;
- for (int i = 0; i < 4; ++i) {
- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
- x.dst[i] = put ? (void*) hv[i] : dv[i];
- x.bytes[i] = nb[i];
- }
- return x;
- }
-
- // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the
- // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows
- // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across
- // layers and chunks
- struct Run {
- std::vector<int64_t> p0;
- std::vector<int> T;
- std::vector<unsigned> seq;
- int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
- };
- // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
- // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
- // layers and back before the next - so a layer's experts are read from the files once for the whole run (the
- // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk
- bool run_dev_layers(int di, const Run& run, std::string& err) {
- Dev& d = devs[(size_t) di];
- Chunk& c = *chunks[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc, nk = (int) run.T.size();
- Streamer S; // (none: the flow streams)
- const size_t hb = chunk_hand_bytes();
- // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while
- // the one before runs (slot u % 2, once unit u - 2's copy of it is done)
- std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
- for (int il : c.fl_layers)
- if (L[(size_t) il].eg >= 0)
- for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});
- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
- auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
- if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
- const int li = egu[u].first, k = egu[u].second;
- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);
- };
- size_t eg_u = 0;
- std::future<void> eg_next;
- if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);
- for (size_t li = 0; li < c.fl_layers.size(); ++li) {
- const int il = c.fl_layers[li];
- Layer& L0 = L[(size_t) il];
- const bool last_l = li + 1 == c.fl_layers.size();
- for (int k = 0; k < nk; ++k) {
- const int64_t p0 = run.p0[(size_t) k];
- const int T = run.T[(size_t) k], slot = k % 2;
- const bool last_k = k + 1 == nk;
- WinPlan wp;
- wp.par = true;
- wp.async = true;
- wp.win = run.seq[(size_t) k];
- wp.k = k;
- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
- if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way
- const double te = now_ms();
- eg_next.get();
- st.engram_ms += now_ms() - te;
- wp.eg_slot = (int) (eg_u % 2);
- if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);
- }
- if (li == 0) { // the chunk's input: its embedding rows
- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
- for (int t = 0; t < T; ++t)
- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
- (size_t) D * 2);
- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.emb_ev[slot], s));
- embed(c.emb, T, c.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = c.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(c.pre, T, s);
- } else { // or its state after the layer before
- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
- for (int i = 0; i < x.n; ++i)
- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));
- }
- if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (it != g.d_targets.end()) {
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int nr = std::min(T, tap_cap);
- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
- CK(cudaStreamWaitEvent(s, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
- (size_t) nr, cudaMemcpyDeviceToHost, s));
- }
- }
- if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;
- if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
- if (!last_l) { // the chunk's state kept for the next layer
- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
- for (int i = 0; i < x.n; ++i)
- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));
- continue;
- }
- // through every layer: the chunk's end, and the logits
- if ((size_t) k < chunk_done.size()) {
- CK(cudaEventRecord(chunk_done[(size_t) k], s));
- chunk_rec[(size_t) k].store(1, std::memory_order_release);
- }
- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
- for (int r0 = 0; r0 < T; r0 += kWin) {
- const int nr = std::min(kWin, T - r0);
- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, nr, err)) return false;
- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaStreamSynchronize(s));
- }
- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
- const int t0 = T - n_logits;
- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
- }
- }
- lm_layers.store((int) li + 1, std::memory_order_release);
- }
- return true;
- }
- bool run_dev_chunks(int di, const Run& run, std::string& err) {
- Dev& d = devs[(size_t) di];
- Chunk& c = *chunks[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc, nk = (int) run.T.size();
- Streamer S;
- const int B = std::max(1, c.ring_n / Chunk::kParts);
- if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)
- CK(cudaStreamSynchronize(s));
- CK(cudaStreamSynchronize(c.cs));
- c.items.clear();
- c.base = c.n_read = c.n_dma = c.n_comp = 0;
- c.n_file = 0;
- std::fill(c.stage_item.begin(), c.stage_item.end(), -1);
- c.fl_layers.clear();
- for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)
- for (int m = 0; m < L[(size_t) il].nm; ++m)
- if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
- c.fl_T = run.T;
- c.spec_k = c.spec_il = -1;
- c.wait_ms = 0;
- c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
- c.lc_il = -1;
- c.lc_used = 0;
- c.lc_first = c.lc_chk = 0;
- c.io_d0 = c.io->direct_bytes();
- c.io_b0 = c.io->buffered_bytes();
- if (!c.fl_layers.empty()) {
- flow_spec(c, 0, c.fl_layers[0], run.T[0]);
- if (!flow_pump(c, err)) return false;
- }
- if (c.lm) return run_dev_layers(di, run, err);
- }
- for (int k = 0; k < nk && !c.flow; ++k)
- for (int il = 0; il < g.nl; ++il) {
- const Layer& L0 = L[(size_t) il];
- bool here = false;
- for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
- if (!here) continue;
- for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)
- S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});
- }
- if (!S.b.empty() && c.ring == nullptr) {
- err = "no slots to stream experts into on CUDA" + std::to_string(d.id);
- return false;
- }
- std::atomic<bool> ra_stop{false};
- std::thread ra;
- if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });
- struct Join { // (every way out of the run: the read-ahead ends before S does)
- std::atomic<bool>& stop;
- std::thread& th;
- ~Join() {
- stop.store(true);
- if (th.joinable()) th.join();
- }
- } ra_join{ra_stop, ra};
- if (!copy_ahead(d, c, S, di, err)) return false;
- // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once
- // chunk k - 2's copy of it is done)
- std::vector<int> egs;
- for (const Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m)
- if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);
- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
- auto read_eg = [this, &c, &run, &egs, eg_per](int k) {
- if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);
- for (int li : egs)
- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);
- };
- std::future<void> eg_next;
- if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
- for (int k = 0; k < nk; ++k) {
- const int64_t p0 = run.p0[(size_t) k];
- const int T = run.T[(size_t) k], slot = k % 2;
- const bool last_k = k + 1 == nk;
- if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way
- const double te = now_ms();
- eg_next.get();
- if (d.mi == 0) st.engram_ms += now_ms() - te;
- if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);
- }
- WinPlan wp;
- wp.par = true;
- wp.async = true;
- wp.win = run.seq[(size_t) k];
- wp.k = k;
- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
- bool started = false;
- std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
- Layer& L0 = L[(size_t) il];
- int mi = -1;
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == di) mi = m;
- if (mi < 0) continue;
- if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off
- started = true;
- if (il == 0) {
- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
- for (int t = 0; t < T; ++t)
- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
- (size_t) D * 2);
- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.emb_ev[slot], s));
- embed(c.emb, T, c.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = c.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(c.pre, T, s);
- } else {
- Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
- xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);
- set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);
- }
- }
- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
- if (sidx < 0 || got[(size_t) sidx]) continue;
- got[(size_t) sidx] = 1;
- Src& sr = srcs[(size_t) sidx];
- if (L[(size_t) sr.layer].grp == L0.grp) continue;
- const int64_t g_first = p0 / sr.ratio;
- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
- Xfer x;
- if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {
- x.src[x.n] = sr.stage(slot);
- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
- x.bytes[x.n++] = (size_t) ng * ckv_rb();
- }
- if (ng > 0 && sr.ik[(size_t) di] != nullptr) {
- x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();
- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
- x.bytes[x.n++] = (size_t) ng * ik_rb();
- }
- if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);
- set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)
- }
- if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (it != g.d_targets.end()) {
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int nr = std::min(T, tap_cap);
- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
- CK(cudaStreamWaitEvent(s, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
- (size_t) nr, cudaMemcpyDeviceToHost, s));
- }
- }
- if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;
- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
- if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
- const int nnext = L[(size_t) il + 1].nm;
- if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);
- xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);
- }
- }
- if (di == head_dev && (size_t) k < chunk_done.size()) {
- CK(cudaEventRecord(chunk_done[(size_t) k], s));
- chunk_rec[(size_t) k].store(1, std::memory_order_release);
- }
- if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits
- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
- for (int r0 = 0; r0 < T; r0 += kWin) {
- const int nr = std::min(kWin, T - r0);
- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, nr, err)) return false;
- if (di != head_dev) continue;
- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaStreamSynchronize(s));
- }
- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
- const int t0 = T - n_logits;
- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
- }
- }
- }
- return true;
- }
-
- // ---- a prompt's path: the flow or windows of 16
- // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a
- // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in
- // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the
- // flow, as before)
- void calibrate_paths() {
- if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;
- if (const char* v = std::getenv("DS4_PATH_PICK"))
- if (std::atoi(v) == 0) return;
- int di = -1;
- for (size_t i = 0; i < chunks.size() && di < 0; ++i)
- if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;
- if (di < 0) return;
- Chunk& c = *chunks[(size_t) di];
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- const double t0 = now_ms();
- // the DMA: staging -> the ring (once to warm up, once timed)
- const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});
- double ms = 0;
- for (int rep = 0; rep < 2; ++rep) {
- const double a = now_ms();
- if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||
- cudaStreamSynchronize(c.cs) != cudaSuccess) {
- cudaGetLastError();
- return;
- }
- ms = now_ms() - a;
- }
- pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);
- // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are
- auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {
- const double a = now_ms();
- left.store((int) jobs.size());
- c.io->push(jobs.data(), (int) jobs.size());
- while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
- return now_ms() - a;
- };
- {
- const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;
- std::vector<uint8_t> src(part * 8, 1);
- std::vector<IoQueue::Job> jobs(8);
- std::atomic<int> left{0};
- std::atomic<bool> bad{false};
- for (int i = 0; i < 8; ++i) {
- jobs[(size_t) i].mem = src.data() + (size_t) i * part;
- jobs[(size_t) i].dst = c.stage + (size_t) i * part;
- jobs[(size_t) i].len = part;
- jobs[(size_t) i].left = &left;
- jobs[(size_t) i].bad = &bad;
- }
- run_jobs(jobs, left);
- const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);
- pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);
- }
- // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:
- // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane
- // three times (from the disk, then in RAM: the faster of two)
- int best = -1, most = 0;
- for (int il = 0; il < g.nl; ++il) {
- int n = 0;
- for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;
- if (n > most) most = n, best = il;
- }
- if (best < 0) return;
- const Layer& l = L[(size_t) best];
- std::vector<int> cold;
- const int n_read = std::min(8, c.S), n_cpu = 8;
- for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {
- if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;
- bool res = false;
- for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);
- if (!res) cold.push_back(e);
- }
- if ((int) cold.size() >= n_read + n_cpu) {
- std::vector<IoQueue::Job> jobs;
- std::atomic<int> left{0};
- std::atomic<bool> bad{false};
- for (int i = 0; i < n_read; ++i) {
- uint8_t* sb = c.stage + (size_t) i * c.stage_stride;
- for (int r = 0; r < 3; ++r) {
- IoQueue::Job j;
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
- j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];
- j.len = mx_part_bytes(l, r);
- j.dst = sb;
- j.left = &left;
- j.bad = &bad;
- j.map = pk.at(l.mx_file[r], j.off);
- jobs.push_back(j);
- sb += flow_region(r);
- }
- }
- const double rd = run_jobs(jobs, left);
- if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);
- }
- { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM
- const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;
- std::vector<Ent> ents;
- for (int i = 0; i < n_cpu; ++i) {
- Ent en;
- en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;
- if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)
- for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)
- if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;
- if (en.e < 0) break;
- en.n = 1;
- en.slot0 = i;
- en.tok[0] = 0;
- ents.push_back(en);
- }
- if (ents.empty()) return;
- std::vector<float> x((size_t) g.dim);
- for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);
- const double bytes = (double) ents.size() * (double) l.blob;
- double a = now_ms();
- cpu_experts(l, x.data(), ents, (size_t) g.dim);
- const double t_first = now_ms() - a;
- double t_warm = 1e30;
- for (int rep = 0; rep < 2; ++rep) {
- a = now_ms();
- cpu_experts(l, x.data(), ents, (size_t) g.dim);
- t_warm = std::min(t_warm, now_ms() - a);
- }
- pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);
- if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);
- }
- pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;
- if (o.verbose) {
- char dk[32] = "not measured", cc[32] = "not measured";
- if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);
- if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);
- std::fprintf(stderr,
- "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "
- "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",
- pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);
- }
- }
- // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's
- // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)
- double pm_rate(const Layer& l, int& n_cold) const {
- const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0, cold = 0;
- n_cold = 0;
- for (int e = 0; e < l.n_exp; ++e) {
- sum += pr[e];
- if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;
- }
- if (n_cold == 0) return 0.0;
- const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;
- return pm.m * (double) l.k_exp * share / n_cold;
- }
- // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))
- double pm_picks(int n) const {
- double d = 0;
- for (int il = 0; il < g.nl; ++il) {
- int nc = 0;
- const double a = pm_rate(L[(size_t) il], nc);
- d += nc * (1.0 - std::exp(-a * n));
- }
- return d;
- }
- // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation
- // that (bisection over its log; a running mean of the log)
- void pm_learn(int n, double picked) {
- if (!pm.on || n < 2 || picked < 1) return;
- const double m0 = pm.m;
- double lo = std::log(1e-3), hi = std::log(1e3);
- for (int it = 0; it < 40; ++it) {
- pm.m = std::exp(0.5 * (lo + hi));
- (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);
- }
- const double obs = 0.5 * (lo + hi);
- pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);
- ++pm.n_m;
- }
- // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's
- // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -
- // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:
- // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at
- // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
- // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
- // faster (by half, with less than kPickRes in RAM); said with --verbose
- bool pick_path(int n) {
- pm.last = -1;
- if (!pm.on || n >= kPickMax) return true;
- double f_res = 0, f_cold = 0, w = 0;
- const int nw = n / kWin, rem = n % kWin;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- int nc = 0;
- const double a = pm_rate(l, nc);
- if (nc == 0) continue;
- int sampled = 0, res = 0, seen = 0;
- const int step = std::max(1, nc / 32);
- for (int e = 0; e < l.n_exp; ++e) {
- if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;
- bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;
- if (!r) {
- r = true;
- for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);
- }
- ++sampled;
- res += r;
- }
- const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;
- auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };
- f_res += share * u(n) * b;
- f_cold += (1.0 - share) * u(n) * b;
- w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;
- }
- const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;
- const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);
- const double t_flow = f_res / r_fr + f_cold / r_fc;
- const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;
- const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;
- // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the
- // fixed costs of every layer on every GPU again, which a pass of the flow pays once)
- const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;
- // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane
- // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would
- // cost the most; windows win where they are all in RAM)
- const double res_share = f_res / std::max(1.0, f_res + f_cold);
- const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));
- pm.last = flow ? 1 : 0;
- pm.last_cls = cls;
- pm.last_nw = n_win;
- pm.last_pred = flow ? t_flow : t_win;
- if (o.verbose)
- std::fprintf(stderr,
- "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "
- "experts' %.1f GB in RAM)\n",
- n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,
- 100.0 * res_share, (f_res + f_cold) / 1e9);
- return flow;
- }
- // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more
- static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }
- // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs
- // is mostly the fixed costs a window pays again and again)
- double pm_k(int c) const {
- for (int dc = 0; dc < 4; ++dc)
- for (int cc : {c - dc, c + dc})
- if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];
- return 1.0;
- }
- // n tokens through windows of 16, the last one's logits_rows rows of logits
- bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {
- logits_all = false;
- for (int a = 0; a < n; a += kWin) {
- const int T = std::min(kWin, n - a);
- if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;
- }
- return true;
- }
- // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken
- // / predicted), windows' time a window - running means
- void path_done(double ms) {
- if (pm.last < 0 || pm.last_pred <= 0) return;
- if (pm.last == 0) {
- const double w = ms / 1000.0 / std::max(1, pm.last_nw);
- pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;
- ++pm.win_n;
- } else {
- const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));
- double& k = pm.k[0][pm.last_cls];
- int64_t& n = pm.kn[0][pm.last_cls];
- k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;
- ++n;
- }
- pm.last = -1;
- }
- // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the
- // first would read from the files again more than the chunks' states moved to host memory and back at every layer
- // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,
- // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB
- // sets it) - at least one expert, or not layer-major
- bool lm_begin(const Run& run) {
- static const char* env = std::getenv("DS4_LAYER_MAJOR");
- if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;
- Chunk& c = *chunks[0];
- if (!c.flow) return false;
- size_t need = 0, n_exp = 0, blob_max = 0;
- double file_b = 0;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- size_t ne = 0;
- for (int e : l.host_list)
- if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;
- const size_t stride = (l.blob + 63) & ~(size_t) 63;
- need = std::max(need, ne * stride);
- n_exp = std::max(n_exp, (size_t) l.n_exp);
- blob_max = std::max(blob_max, stride);
- file_b += (double) ne * (double) l.blob;
- }
- int64_t rows = 0;
- for (int T : run.T) rows += T;
- const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;
- const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;
- if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;
- size_t lc_bytes = need;
- if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);
- else {
- const uint64_t avail = ram_avail();
- if (avail != ~uint64_t{0})
- lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);
- }
- if (lc_bytes < blob_max) return false;
- void* st_p = nullptr;
- c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;
- if (!c.lm_pinned) {
- cudaGetLastError();
- st_p = os::anon_alloc(state_b, -1, 1);
- if (st_p == nullptr) return false;
- }
- c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);
- if (c.lc == nullptr) {
- if (c.lm_pinned) cudaFreeHost(st_p);
- else os::anon_free(st_p, state_b);
- return false;
- }
- c.lm_state = (uint8_t*) st_p;
- c.lm_state_bytes = state_b;
- c.lc_bytes = lc_bytes;
- c.lc_of.assign(n_exp, -1);
- c.lc_writer.assign(n_exp, -1);
- c.lm = true;
- lm_layers.store(0);
- return true;
- }
- void lm_end() {
- for (size_t di = 0; di < chunks.size(); ++di) {
- Chunk& c = *chunks[di];
- if (!c.lm) continue;
- cudaSetDevice(devs[di].id);
- cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's
- cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)
- for (size_t i = c.base; i < c.n_read; ++i)
- while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
- if (c.lm_pinned) cudaFreeHost(c.lm_state);
- else os::anon_free(c.lm_state, c.lm_state_bytes);
- os::anon_free(c.lc, c.lc_bytes);
- c.lm_state = c.lc = nullptr;
- c.lm_state_bytes = c.lc_bytes = 0;
- c.lm = c.lm_pinned = false;
- }
- }
-
- // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
- // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
- bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
- if (n < 1) return true;
- if (pos + n > o.max_context) {
- err = "the context is full";
- return false;
- }
- for (int t = 0; t < n; ++t)
- if (tk[t] < 0 || tk[t] >= g.vocab) {
- err = "token " + std::to_string(tk[t]) + " out of range";
- return false;
- }
- const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);
- Run run;
- for (int a = 0; a < n; a += per) {
- run.p0.push_back(pos + a);
- run.T.push_back(std::min(per, n - a));
- run.seq.push_back(++win_seq);
- }
- logits_all = logits_rows > kWin;
- if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
- for (int t = 0; t < n; ++t) {
- toks.push_back(tk[t]);
- ctoks.push_back(ctok(pos + t, tk[t]));
- }
- plan_streams();
- if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way
- run.snap_pos = pos + n - 1;
- for (int il = 0; il < g.nl; ++il) {
- Layer& l = L[(size_t) il];
- Dev& d = devs[(size_t) l.dev];
- if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;
- if (l.c_kv != nullptr && l.snap_kv == nullptr &&
- (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))
- return false;
- }
- }
- const int T_last = run.T.back();
- n_logits = logits_all ? 0 : std::min(logits_rows, T_last);
- const double tl = now_ms();
- { // each chunk's end on the head device, for the progress reports
- cudaSetDevice(devs[(size_t) head_dev].id);
- while (chunk_done.size() < run.T.size()) {
- cudaEvent_t e = nullptr;
- CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
- chunk_done.push_back(e);
- }
- chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);
- for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);
- }
- prof_chunk = prof_on;
- const bool lm = lm_begin(run);
- struct LmEnd { // (every way out: its host memory freed once the device is done with it)
- Impl* p;
- bool on;
- ~LmEnd() {
- if (on) p->lm_end();
- }
- } lm_end_{this, lm};
- std::vector<std::string> errs(devs.size());
- for (size_t di = 0; di < devs.size(); ++di)
- workers[di]->submit([this, di, &run, &errs] {
- std::string e;
- if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
- });
- size_t reported = 0;
- for (;;) {
- bool all = true;
- for (auto& w : workers) all = all && w->idle();
- if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
- const int nd = lm_layers.load(std::memory_order_acquire);
- if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
- } else if (progress) { // chunks through every layer (the last layer's device past their event)
- while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&
- cudaEventQuery(chunk_done[reported]) == cudaSuccess)
- ++reported;
- if (reported > 0 && reported < run.T.size())
- progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);
- }
- if (all) break;
- std::this_thread::sleep_for(std::chrono::milliseconds(50));
- }
- for (const std::string& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- st.layer_ms += now_ms() - tl;
- eg_stats();
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- const double th = now_ms();
- CK(cudaStreamSynchronize(d.st));
- st.head_ms += now_ms() - th;
- if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed
- for (Dev& dv : devs) {
- cudaSetDevice(dv.id);
- CK(cudaStreamSynchronize(dv.st));
- }
- cudaSetDevice(d.id);
- prof_collect();
- std::fprintf(stderr,
- "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "
- "(with the waits for streamed ones), between layers %.0f ms\n",
- n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);
- for (double& v : cprof_ms) v = 0;
- prof_chunk = false;
- }
- if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {
- const double ms = now_ms() - tl;
- for (size_t di = 0; di < chunks.size(); ++di) {
- const Chunk& c = *chunks[di];
- if (!c.flow) continue;
- const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;
- const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;
- const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)
- char lcs[160] = "";
- if (c.lm)
- std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "
- "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);
- std::fprintf(stderr,
- "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "
- "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "
- "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "
- "%.0f ms for reads\n",
- devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),
- (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,
- (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,
- (unsigned long long) c.it_spec_unused, c.wait_ms);
- }
- }
- if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)
- bool tp = false;
- for (const Layer& l : L) tp = tp || l.nm > 1;
- double picked = 0;
- for (const auto& cp : chunks)
- if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;
- if (!tp) pm_learn(n, picked);
- }
- if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)
- pos += n;
- st.tokens += n;
- cache_tick(n);
- const int nr = std::min(T_last, tap_cap);
- if (!seed_drafter(pos - nr, nr, 0, err)) return false;
- if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -
- // the next feed seeds them again from its own rows, past this position)
- Check& ckp = ck[snap_slot & 1];
- ckp.pos = run.snap_pos;
- ckp.win.resize(L.size());
- ckp.comp.resize(L.size() * 2);
- for (size_t i = 0; i < L.size(); ++i) {
- Layer& l = L[i];
- Dev& dv = devs[(size_t) l.dev];
- cudaSetDevice(dv.id);
- const bool kept = !l.mtp && l.snap_win != nullptr;
- ckp.win[i].resize((size_t) g.win * win_rb());
- CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),
- cudaMemcpyDeviceToHost, dv.st));
- if (l.c_kv) {
- ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);
- ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
- CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,
- ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));
- CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,
- ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));
- }
- }
- for (Dev& dv : devs) {
- cudaSetDevice(dv.id);
- CK(cudaStreamSynchronize(dv.st));
- }
- }
- return true;
- }
-
- // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)
- void plan_window(int64_t p0, int T, WinPlan& wp) {
- wp.par = par;
- wp.async = par || (!sync_decode && T <= async_max_t);
- wp.vl = 0;
- for (int t = 0; t < T && !imgs.empty(); ++t)
- if (in_image(p0 + t)) wp.vl |= 1u << t;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- if (l.ratio > 0) {
- const int R = l.ratio;
- int n[kWin], k[kWin], clen_max = 0;
- for (int t = 0; t < T; ++t) {
- n[t] = (int) ((p0 + t + 1) / R);
- k[t] = std::min(g.ix_topk, n[t]);
- clen_max = std::max(clen_max, n[t]);
- }
- if (l.ix_src) {
- for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];
- if (clen_max > 0 && l.cand_src)
- for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);
- for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];
- }
- for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];
- }
- if (wp.async) {
- wp.seq[il] = ++seq_issued;
- layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);
- }
- }
- }
-
- int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }
- // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's
- // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows
- void head_cols(int nm, int mi, int& v0, int& vn) const {
- const Tensor* tw = pk.get("head.weight");
- if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {
- vn = g.vocab / nm;
- v0 = mi * vn;
- } else
- split128(g.vocab, nm, mi, v0, vn);
- }
- // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's
- // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two
- // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather
- // false: each member keeps its share (the drafter's markov steps over the group, mk_split)
- bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {
- const int V = g.vocab, D = g.dim, nm = head_nm();
- int v0 = 0, vn = 0, Vmax = 0, z = 0;
- head_cols(nm, mi, v0, vn);
- head_cols(nm, 0, z, Vmax);
- Group& G = groups[(size_t) d.grp];
- float* out = mi == 0 ? d.logits : d.logits_part;
- const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;
- cudaSetDevice(d.id);
- // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4
- // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)
- const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head)); // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)
- for (int c = 0; c < rows; c += per)
- if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))
- return false;
- if (nm == 1 || !gather) return true;
- const unsigned seq = ++d.head_seq;
- const int slot = (int) (seq % 2);
- auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };
- auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };
- if (mi > 0) {
- if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);
- Xfer x;
- x.n = 1;
- x.src[0] = d.logits_part;
- x.dst[0] = stage(mi);
- x.bytes[0] = (size_t) rows * vn * 4;
- xfer_put(x, flag(mi), seq, d.xcnt, d.st);
- return true;
- }
- for (int m = 1; m < nm; ++m) {
- int m0 = 0, mn = 0;
- head_cols(nm, m, m0, mn);
- Xfer x;
- x.n = 1;
- x.src[0] = stage(m);
- x.dst[0] = d.logits_part;
- x.bytes[0] = (size_t) rows * mn * 4;
- xfer_get(x, flag(m), seq, d.st);
- CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,
- (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));
- }
- set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);
- return true;
- }
-
- // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)
- bool head_rows_dev(int di, int T, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- const int D = g.dim, H = g.hc;
- if (n_logits <= 0) return true;
- const int t0 = T - n_logits;
- hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);
- return head_share(d, d.mi, d.x, n_logits, err);
- }
-
- // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read
- // h_taps)
- bool tap(Dev& d, int il, int T, std::string& err) {
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (g.n_mtp == 0 || it == g.d_targets.end()) return true;
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int D = g.dim;
- cudaSetDevice(d.id);
- hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);
- CK(cudaStreamWaitEvent(d.st, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,
- cudaMemcpyDeviceToHost, d.st));
- return true;
- }
-
- bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {
-#if defined(STRATA_METAL)
- struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)
- bool on;
- BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }
- ~BusyAll() { if (on) ds4_busy_end(); }
- } busy_all;
-#endif
- if (T < 1 || T > kWin) {
- err = "a window of " + std::to_string(T) + " tokens";
- return false;
- }
- if (pos + T > o.max_context) {
- err = "the context is full";
- return false;
- }
- for (int t = 0; t < T; ++t)
- if (tk[t] < 0 || tk[t] >= g.vocab) {
- err = "token " + std::to_string(tk[t]) + " out of range";
- return false;
- }
- const int64_t p0 = pos;
- for (int t = 0; t < T; ++t) {
- toks.push_back(tk[t]);
- ctoks.push_back(ctok(pos + t, tk[t]));
- }
- // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is
- // left of their reads by then is the window's Engram time
- const double te = now_ms();
- const int64_t eg_w0 = eg_wait_ns.load();
- for (int li = 0; li < pk.eg_layers; ++li) {
- EgBatch& b = *eg_win[(size_t) li];
- if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)
- eg_issue(p0, T, li, nullptr, b);
- }
- st.engram_ms += now_ms() - te;
- const int D = g.dim;
- for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)
- std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);
- plan_window(p0, T, wplan);
- prof_window = prof_on && T == 1;
- n_logits = std::min(logits_rows, T);
- const double tl = now_ms();
- if (par) {
- if (!forward_par(p0, T, err)) return false;
- } else if (!forward_serial(p0, T, err)) return false;
- st.layer_ms += now_ms() - tl;
- for (auto& b : eg_win)
- if (b->live) eg_finish(*b);
- st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;
- eg_stats();
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- const double th = now_ms();
- CK(cudaStreamSynchronize(d.st));
- st.head_ms += now_ms() - th;
- prof_collect();
- pos += T;
- st.tokens += T;
- cache_tick(T);
- if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;
- return true;
- }
-
- // one thread: the layers device after device, the hand-offs through pinned memory and events
- void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer
- ds4_metal_flush();
- const float* hp = (const float*) strata::gpu::host_view(dev);
- size_t nf = 0, first = 0; float mx = 0.0f;
- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }
- std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);
- }
- void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains
- ds4_metal_flush();
- const float* hp = (const float*) strata::gpu::host_view(d.h);
- float mx = 0.0f; size_t nf = 0;
- const size_t n = (size_t) T * g.hc * g.dim;
- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }
- std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);
- }
- bool forward_serial(int64_t p0, int T, std::string& err) {
- const int D = g.dim, H = g.hc;
- int cur = L[0].dev;
- if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();
- {
- Dev& d = devs[(size_t) cur];
- cudaSetDevice(d.id);
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, d.st);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, d.st);
- if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);
- }
- std::vector<float> dump;
- if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
- for (int il = 0; il < g.nl; ++il) {
- Layer& l = L[(size_t) il];
- if (l.dev != cur) {
- Dev& a = devs[(size_t) cur];
- Dev& b = devs[(size_t) l.dev];
- cudaSetDevice(a.id);
- const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- uint8_t* bb = a.bounce;
- CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));
- CK(cudaEventRecord(a.ev_out, a.st));
- cudaSetDevice(b.id);
- CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));
- CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));
- cur = l.dev;
- }
- if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
- if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);
- if (!layer(l, 0, p0, T, wplan, err)) return false;
- if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
- Dev& dd = devs[(size_t) l.dev];
- std::vector<float> hv((size_t) T * H * D);
- cudaSetDevice(dd.id);
- CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));
- CK(cudaStreamSynchronize(dd.st));
- for (int t = 0; t < T; ++t)
- std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,
- (size_t) H * D * 4);
- }
- }
- if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);
- if (cur != head_dev) {
- err = "the head is not on the last layer's GPU";
- return false;
- }
- return head_rows_dev(head_dev, T, err);
- }
-
- // threads: each device's thread enqueues its layers (a group's members each their share) at once, the
- // devices waiting on each other through host-mapped flags (the window's seq)
- bool forward_par(int64_t p0, int T, std::string& err) {
- wplan.win = ++win_seq;
- std::vector<std::string> errs(devs.size());
- for (size_t di = 0; di < devs.size(); ++di)
- workers[di]->submit([this, di, p0, T, &errs] {
- std::string e;
- if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
- });
- for (auto& w : workers) w->wait();
- for (const std::string& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- return true;
- }
-
- // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer
- Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {
- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- void* dv[4] = {d.h, d.pre, d.topk, d.cand};
- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
- const size_t nb[4] = {nh, np, nk, nc};
- Xfer x;
- x.n = 4;
- for (int i = 0; i < 4; ++i) {
- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
- x.dst[i] = put ? (void*) hv[i] : dv[i];
- x.bytes[i] = nb[i];
- }
- return x;
- }
-
- bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc;
- bool started = false;
- std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
- Layer& L0 = L[(size_t) il];
- int mi = -1;
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == di) mi = m;
- if (mi < 0) continue;
- if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
- started = true;
- if (il == 0) {
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, s);
- } else {
- const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
- xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);
- }
- }
- // the compressed KV / index keys another group's source layer wrote this window
- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
- if (sidx < 0 || got[(size_t) sidx]) continue;
- got[(size_t) sidx] = 1;
- Src& sr = srcs[(size_t) sidx];
- if (L[(size_t) sr.layer].grp == L0.grp) continue;
- const int64_t g_first = p0 / sr.ratio;
- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
- if (ng == 0) continue;
- Xfer x;
- if (sr.ckv[(size_t) di] != nullptr) {
- x.src[x.n] = sr.x_stage;
- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
- x.bytes[x.n++] = (size_t) ng * ckv_rb();
- }
- if (sr.ik[(size_t) di] != nullptr) {
- x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();
- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
- x.bytes[x.n++] = (size_t) ng * ik_rb();
- }
- if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);
- }
- if (mi == 0 && !tap(d, il, T, err)) return false;
- if (!layer(L0, mi, p0, T, wp, err)) return false;
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);
- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
- if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);
- }
- if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);
- return true;
- }
-
- // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,
- // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it
- bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,
- size_t bytes, int from, std::string& err) {
- bool any = false;
- for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);
- if (!any) return true;
- Dev& d = devs[(size_t) from];
- CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));
- CK(cudaEventRecord(ev, d.st));
- for (size_t od = 0; od < devs.size(); ++od) {
- if (copies[od] == nullptr || (int) od == from) continue;
- Dev& o2 = devs[od];
- cudaSetDevice(o2.id);
- CK(cudaStreamWaitEvent(o2.st, ev, 0));
- CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));
- }
- cudaSetDevice(d.id);
- return true;
- }
-
- // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;
- // the partial sums all-reduced across the group)
- bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {
- Layer& l = member(L0, mi);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int nm = L0.nm;
- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
- const ArGroup& ag = groups[(size_t) L0.grp].ar;
- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
- if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's
- ds4_metal_flush();
- int shown = 0;
- for (int e = 0; e < l.n_exp && shown < 3; ++e) {
- if (l.slot[(size_t) e] < 0) continue;
- std::vector<uint8_t> want(l.blob, 0);
- read_mx_file(l, e, want.data());
- const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));
- size_t bad = 0, first = 0;
- for (size_t i = 0; i < l.blob; ++i)
- if (want[i] != have[i]) { if (!bad) first = i; ++bad; }
- std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);
- ++shown;
- }
- }
- if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);
- if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
- std::vector<float> hv((size_t) T * H * D);
- cudaSetDevice(d.id);
- ds4_metal_flush();
- {
- const float* hvp = (const float*) strata::gpu::host_view(d.h);
- if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);
- }
- int nf = 0, nzv = 0;
- float mx = 0.0f;
- size_t firstnan = 0, lastnan = 0;
- for (size_t i = 0; i < hv.size(); ++i) {
- const float v = hv[i];
- if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }
- if (v != 0.0f) ++nzv;
- if (std::fabs(v) > mx) mx = std::fabs(v);
- }
- std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",
- l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],
- hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);
- }
- prof_mark(d, 0);
- // Engram
- if (l.eg >= 0) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- const size_t per = (size_t) cols * (ehd + ehd / 32);
- eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)
- uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;
- for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]
- CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,
- cudaMemcpyHostToDevice, s));
- uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;
- for (int t = 0; t < T; ++t)
- CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
- (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
- dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);
- dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);
- if (std::getenv("DS4_DBG_EG")) {
- dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
- const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
- for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)
- if (!std::isfinite(xv[i])) {
- const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;
- const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;
- const uint8_t* hv = src + t * per + c * ehd;
- std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);
- for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);
- std::fprintf(stderr, "\n");
- break;
- }
- }
- if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
- if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
- dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);
- dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);
- engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
- dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
- }
- dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));
- // attention
- hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
- dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);
- dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);
- dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);
- dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);
- hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
- dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);
- dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
- {
- x3::MvJob j[4];
- int nj = 0;
- j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);
- j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);
- if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);
- if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);
- if (!mv_n(d, j, nj, err)) return false;
- }
- dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
- dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(d.qr, T, g.q_lora, g.q_lora, fq, s);
- dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
- if (NHL > 0) {
- if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;
- Finish fr;
- fr.rd = g.rd;
- fr.pos = (int) p0;
- fr.vpp = NHL;
- fr.freqs = freqs;
- finish(d.q, T * NHL, HD, HD, fr, s);
- }
- if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);
- // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark
- // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of
- // that ring and to the whole block, and leaves the ring as it was
- ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = d.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(d.kvr, T, HD, HD, fk, s);
- if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
- dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);
- dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)
- AttnRows ar;
- for (int t = 0; t < T; ++t) {
- if (l.mtp) {
- ar.win0[t] = 0;
- ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);
- ar.nsel[t] = T;
- continue;
- }
- const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)
- ar.win0[t] = first;
- ar.nwin[t] = g.win + t - first + 1;
- ar.nsel[t] = 0;
- }
- const void* ckv = nullptr;
- int ckv_fmt = kKvFp4;
- if (l.ratio > 0) {
- Src& sr = srcs[(size_t) l.src];
- const int R = l.ratio;
- // the groups this window completes: g_first .. g_first + ng - 1
- const int64_t g_first = p0 / R; // the first group whose last position is >= p0
- const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1
- const int ng = (int) std::max<int64_t>(0, g_end - g_first);
- if (l.kv_src) {
- if (R == 1) {
- Finish fc;
- fc.w = l.c_norm;
- fc.eps = g.eps;
- finish(d.ckr, T, HD, HD, fc, s);
- CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));
- } else {
- if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected
- const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;
- CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
- }
- compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
- dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);
- dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);
- }
- }
- Rows lim;
- int clen_max = 0;
- for (int t = 0; t < T; ++t) {
- lim.n[t] = (int) ((p0 + t + 1) / R);
- lim.k[t] = std::min(g.ix_topk, lim.n[t]);
- clen_max = std::max(clen_max, lim.n[t]);
- }
- if (l.ix_src) {
- Src& ks = srcs[(size_t) l.ik_src];
- if (l.owns_k && ng > 0) {
- if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;
- Finish fi;
- fi.w = l.ik_norm;
- fi.eps = g.eps;
- fi.rd = g.rd;
- fi.pos = (int) (g_first * R);
- fi.pos_step = R;
- fi.freqs = freqs;
- fi.quant = 3;
- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- fi.pdst_stride = (long long) ik_rb();
- finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
- if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),
- (size_t) ng * ik_rb(), l.dev, err))
- return false;
- }
- if (clen_max > 0) {
- if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;
- Finish fi;
- fi.rd = g.rd;
- fi.pos = (int) p0;
- fi.vpp = g.ix_heads;
- fi.freqs = freqs;
- fi.quant = 3;
- finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
- // (iw rows are 64 apart)
- gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,
- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);
- const long long ss = (long long) o.max_context + 64;
- bool restrict_ = false;
- for (int t = 0; t < T; ++t)
- restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;
- if (l.uses_cand && restrict_) {
- Rows cl; // the candidate positions of each query: its blocks x blk
- int nmax = 0;
- for (int t = 0; t < T; ++t) {
- cl.n[t] = lim.n[t];
- cl.k[t] = wp.cand[l.il][t];
- nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);
- }
- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,
- (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);
- Rows tr;
- for (int t = 0; t < T; ++t) {
- tr.n[t] = wp.cand[l.il][t] * g.cand_blk;
- tr.k[t] = lim.k[t];
- }
- topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);
- } else {
- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,
- lim, d.scores, ss, s);
- if (l.cand_src) {
- Rows cr;
- for (int t = 0; t < T; ++t) {
- cr.n[t] = lim.n[t];
- cr.k[t] = g.cand_topk;
- }
- cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,
- (int) cand_stride(), s);
- }
- topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);
- }
- }
- }
- if (l.kv_src && ng > 0) {
- Finish fc;
- fc.rd = g.rd;
- fc.pos = (int) (g_first * R);
- fc.pos_step = R;
- fc.freqs = freqs;
- fc.quant = 2;
- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
- fc.pdst_stride = (long long) ckv_rb();
- finish(d.latent, ng, HD, HD, fc, s);
- if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),
- (size_t) ng * ckv_rb(), l.dev, err))
- return false;
- if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers
- Xfer x;
- x.src[0] = fc.pdst;
- x.dst[0] = sr.x_stage;
- x.bytes[0] = (size_t) ng * ckv_rb();
- x.n = 1;
- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- x.dst[1] = sr.x_stage + x_cap() * ckv_rb();
- x.bytes[1] = (size_t) ng * ik_rb();
- x.n = 2;
- }
- xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
- }
- }
- if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);
- dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);
- for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
- ckv = sr.ckv[(size_t) l.dev];
- }
- if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection
- ckv = d.ext + (size_t) g.win * HD;
- ckv_fmt = kKvBf16;
- iota_rows(d.topk, T, T, (int) topk_stride(), s);
- }
- if (NHL > 0) {
- sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,
- 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);
- dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);
- Finish fo;
- fo.rd = g.rd;
- fo.pos = (int) p0;
- fo.vpp = NHL;
- fo.freqs = freqs;
- fo.inverse = 1;
- fo.quant = 1;
- finish(d.o, T * NHL, HD, HD, fo, s);
- dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);
- x3::MvJob j[16];
- const int per = HQ / OG;
- for (int gi = 0; gi < OG; ++gi)
- j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);
- if (!mv_n(d, j, OG, err)) return false;
- if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;
- } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)
- dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);
- if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
- hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
- dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- prof_mark(d, 1);
- // the MoE
- hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);
- hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
- dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);
- dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);
- dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);
- dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
- if (wp.async) {
- // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members
- // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the
- // router made), and waits for the answer only before the experts' sum
- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
- const unsigned sq = wp.seq[l.il];
- const int slot = (int) (sq % kRing);
- const bool lead = mi == 0;
- int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below
- for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {
- const Layer* nx = ahead_layer(L0, a);
- if (nx == nullptr) break;
- const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);
- float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;
- gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN
- // weights instead: the same hit rate)
- route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);
- na[a - 1] = m;
- }
- route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,
- d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],
- na[1]);
- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;
- swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);
- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;
- prof_mark(d, 2);
- const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;
- if (T == 1 && moe_token_path) {
- x3::MoeArgs ma;
- ma.plan_ptr = d.plan_ptr;
- ma.plan_w = d.rw;
- ma.cpu_part = lead ? resps[slot].part[0] : nullptr;
- ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;
- ma.lay = lay;
- ma.k = l.k_exp;
- ma.n_embd = D;
- ma.n_ff = FF;
- ma.limit = g.lim;
- const bool mx = lay.fmt == gg::kFmtMXFP4;
- if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)
- zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);
- if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
- else x3::moe_gate_up(ma, d.x, d.xg, s);
- prof_mark(d, 3);
- if (lead) wait_seq(&resps[slot].seq, sq, s);
- prof_mark(d, 4);
- if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
- else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
- } else { // each distinct expert read once for the window's tokens
- gf::MoeWin mw;
- mw.plan_ptr = d.plan_ptr;
- mw.T = T;
- mw.k = l.k_exp;
- mw.grp_ptr = d.grp_ptr;
- mw.grp_start = d.grp_start;
- mw.grp_ent = d.grp_ent;
- mw.n_groups = d.n_groups;
- gf::moe_win_group(mw, s);
- x3::WinArgs wa;
- wa.grp_ptr = d.grp_ptr;
- wa.grp_start = d.grp_start;
- wa.grp_ent = d.grp_ent;
- wa.n_groups = d.n_groups;
- wa.plan_ptr = d.plan_ptr;
- wa.plan_w = d.rw;
- wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;
- wa.cpu_stride = kMaxDim;
- wa.cpu_flag = lead ? resps[slot].cpu : nullptr;
- wa.wait_ptr = lead ? &resps[slot].seq : nullptr;
- wa.wait_val = sq;
- wa.x = d.x;
- wa.x_stride = (size_t) D;
- wa.h = d.xg;
- wa.part = d.ed;
- wa.sh_out = d.sh_out;
- wa.out = d.moe_out;
- wa.out_stride = (size_t) D;
- wa.lay = lay;
- wa.T = T;
- wa.k = l.k_exp;
- wa.n_embd = D;
- wa.n_ff = FF;
- wa.limit = g.lim;
- if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {
- err = "exl3 win_experts failed";
- return false;
- }
- }
- if (lead) st.experts += (int64_t) T * l.k_exp;
- if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 5);
- return true;
- }
- if (nm > 1) {
- err = "a tensor-parallel layer on the host-routed path";
- return false;
- }
- // (glog rows are 1024 apart: route reads them so)
- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
- VlBits vb = {};
- vb.w[0] = l.mtp ? 0u : wp.vl;
- route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,
- vb.w[0] != 0 ? &vb : nullptr);
- CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaEventRecord(d.ev, s));
- // the shared expert while the host looks at the picks
- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;
-#if defined(STRATA_METAL)
- int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer
- if (T == 1 && la_enabled()) {
- for (int a = 1; a <= la_depth(); ++a) {
- const int il = l.il + a;
- if (il >= g.nl) break;
- const Layer& nx = L[(size_t) il];
- if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;
- gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);
- route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,
- d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);
- CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));
- la_n = a;
- }
- if (la_n > 0) CK(cudaEventRecord(d.ev2, s));
- }
-#endif
- if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);
- std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }
- swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);
- if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);
- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;
- const double tw = now_ms();
- CK(cudaEventSynchronize(d.ev));
- st.gpu_wait_ms += now_ms() - tw;
- dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);
- dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);
- dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);
- dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);
- // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the
- // CPU lane's entries
- Combine cm;
- Gather ga;
- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];
- int nvex = 0, nrows = 0;
- std::vector<Ent> ents;
- std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)
- std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)
- for (int t = 0; t < T; ++t) {
- cm.n[t] = 0;
- cm.cpu[t] = 0;
- for (int i = 0; i < l.k_exp; ++i) {
- const int e = h_idx[t * kPicks + i];
- if (e < 0 || e >= l.n_exp) {
- err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";
- return false;
- }
- ++pr[e];
- l.heat[(size_t) e] += 1.0f;
- if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
- per[(size_t) e].push_back({t, i});
- }
- }
- int nslots = 0;
- for (int e = 0; e < l.n_exp; ++e) {
- const auto& pe = per[(size_t) e];
- if (pe.empty()) continue;
-#if defined(STRATA_METAL)
- const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));
-#else
- const bool on_gpu = l.slot[(size_t) e] >= 0;
-#endif
- if (on_gpu) {
- vex[nvex] = e;
- vrow0[nvex] = nrows;
- vn[nvex] = (int) pe.size();
- for (const auto& tp : pe) {
- ga.src[nrows] = tp.first;
- const int t = tp.first;
- cm.row[t][cm.n[t]] = nrows;
- cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];
- ++cm.n[t];
- ++nrows;
- }
- ++nvex;
- } else {
- for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {
- Ent en;
- en.e = e;
- en.slot0 = nslots;
- en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);
- for (int i = 0; i < en.n; ++i) {
- en.tok[i] = pe[a + (size_t) i].first;
- cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});
- cm.cpu[en.tok[i]] = 1;
- }
- nslots += en.n;
- ents.push_back(en);
- }
- }
- }
- st.experts += (int64_t) T * l.k_exp;
- st.cpu_experts += nslots;
-#if defined(STRATA_METAL)
- if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);
-#endif
- if (nrows > 0) {
- gather_rows(d.x, ga, nrows, D, d.xe, s);
- dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);
- { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }
- dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }
- x3::MvJob j[2 * kWin * kPicks];
- int nj = 0;
- for (int i = 0; i < nvex; ++i) {
- const auto& vm = l.vm[(size_t) vex[i]];
- j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
- j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
- }
- if (!mv_n(d, j, nj, err)) return false;
- dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);
- dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);
- swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);
- dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);
- nj = 0;
- for (int i = 0; i < nvex; ++i)
- j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,
- vn[i], g.ff, D);
- if (!mv_n(d, j, nj, err)) return false;
- dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);
- }
- if (!ents.empty()) {
- const double tc = now_ms();
-#if defined(STRATA_METAL)
- ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
- ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)
-#endif
- static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
- if (!skip_cpu) cpu_experts(l, h_x, ents);
- else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)
-#if defined(STRATA_METAL)
- ds4_busy_end();
-#endif
- std::fill(h_part, h_part + (size_t) T * D, 0.0f);
- for (int sl = 0; sl < nslots; ++sl) {
- float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
- const float wv = cpu_rows[(size_t) sl].second;
- const float* y = c_yd.data() + (size_t) sl * D;
- for (int e = 0; e < D; ++e) o[e] += wv * y[e];
- }
- { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;
- ms += now_ms() - tc; ++calls; exps += (long) ents.size();
- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }
- (void) 0; (void) ms; }
- st.cpu_ms += now_ms() - tc;
- dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);
- CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));
- }
- combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);
- dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);
- if (std::getenv("DS4_DBG_MOE")) {
- float hp = 0.0f;
- for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));
- std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);
- dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);
- dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);
- }
- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)
- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
- return true;
- }
-
- int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- strata::dsv41k::SamplerParams sp;
- sp.temperature = temperature;
- sp.greedy = temperature <= 0.0f;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
- sp, d.tok, d.st);
- int t = -1;
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- return t;
- }
-
- bool logits(std::vector<float>& out) {
- if (logits_all) {
- out = chunk_logits;
- return true;
- }
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- out.resize((size_t) std::max(1, n_logits) * g.vocab);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
- }
-
- void save_profile() const {
- if (o.profile.empty()) return;
- const std::string tmp = o.profile + ".tmp";
- std::FILE* f = std::fopen(tmp.c_str(), "wb");
- if (f == nullptr) return;
- const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();
- std::fclose(f);
- // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile
- // never moved on and only .tmp files were left)
- if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());
- }
-
- std::vector<std::string> info_lines() const {
- std::vector<std::string> v;
- int nh = 0;
- for (const Layer& l : L) nh += l.n_exp - l.n_vram;
- char b[512];
- // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)
- std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "
- "cpu_experts=%d ram_experts=%d engine=ds4-0.1",
- (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);
- v.push_back(b);
- if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)
- std::snprintf(b, sizeof b,
- "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "
- "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "
- "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);
- v.push_back(b);
- std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),
- pagefile_hint);
- v.push_back(b);
- }
- if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor
- std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,
- pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);
- v.push_back(b);
- }
- if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)
- std::snprintf(b, sizeof b,
- "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "
- "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",
- vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,
- (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),
- (long long) (vt.device_bytes() >> 20));
- v.push_back(b);
- } else if (!vision_err.empty()) { // (the server offers no pictures and says why)
- std::string why = vision_err;
- for (char& ch : why)
- if (ch == '\n' || ch == '\r') ch = ' ';
- v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");
- v.push_back("INFO vision=off");
- }
- return v;
- }
-};
-
-Model::Model() : p_(new Impl) {}
-Model::~Model() = default;
-bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }
-bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }
-bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
- Impl& p = *p_;
- p.pm.last = -1;
- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);
- if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
- const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
- const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
- : p.feed_windows(tokens, n, logits_rows, err);
- if (ok) p.path_done(now_ms() - t0);
- return ok;
- }
- return p.feed_windows(tokens, n, logits_rows, err);
-}
-bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
- Impl& p = *p_;
- p.pm.last = -1;
- const double t0 = now_ms();
- if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
- const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
- if (ok) p.path_done(now_ms() - t0);
- return ok;
- }
- if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;
- p.checkpoint(slot);
- const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);
- if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)
- return ok;
-}
-int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
-void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
-int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
- return p_->sample(t, tp, tk, mp, seed, counter);
-}
-bool Model::logits(std::vector<float>& out) { return p_->logits(out); }
-int Model::image_token() const { return p_->img_tok; }
-int Model::dim() const { return p_->g.dim; }
-bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {
- return p_->encode_image(rgb, width, height, span, err);
-}
-void Model::set_images(std::vector<Image> images) {
- std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });
- p_->imgs = std::move(images);
-}
-int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }
-bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,
- float mp, uint64_t seed, uint64_t counter) {
- return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);
-}
-bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,
- int& accepted, int& next, std::string& err) {
- return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);
-}
-int64_t Model::pos() const { return p_->pos; }
-int64_t Model::max_context() const { return p_->o.max_context; }
-int Model::vocab() const { return p_->g.vocab; }
-const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
-void Model::reset() { p_->reset(); }
-void Model::checkpoint(int slot) { p_->checkpoint(slot); }
-bool Model::restore(int64_t limit) { return p_->restore(limit); }
-int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
-void Model::save_profile() const { p_->save_profile(); }
-const Stats& Model::stats() const {
- Stats& st = p_->st;
- if (p_->store) { // (the store keeps its own counters: copied in when asked)
- const ExpertStore& x = *p_->store;
- st.store_hits = x.stats().hits.load();
- st.store_misses = x.stats().misses.load();
- st.store_prefetched = x.stats().prefetched.load();
- st.store_pf_used = x.stats().prefetch_used.load();
- st.store_pf_wasted = x.stats().prefetch_wasted.load();
- st.store_evictions = x.stats().evictions.load();
- st.store_bytes = x.stats().bytes_read.load();
- st.store_fails = x.stats().fails.load();
- st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
- st.store_busy_ms = x.busy_ms();
- st.store_zc_visible = (int64_t) x.visible_count();
- st.store_slots = (int64_t) x.slots_allocated();
- st.store_slots_max = (int64_t) x.slots_max();
- }
- return st;
-}
-std::string Model::describe() const { return p_->describe(); }
-std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
-
-} // namespace strata::ds4
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).
+//
+// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):
+// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables
+// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)
+// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus
+// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the
+// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b
+// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])
+// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert
+// h = post * moe + comb^T h; pre = f_pre
+// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
+#include "ds4_model.hpp"
+#include "ds4_engram_io.hpp"
+
+#include "ds4_gguf.cuh"
+#include "ds4_gguf_cpu.hpp"
+#include "ds4_kernels.cuh"
+#include "ds4_os.hpp"
+#include "ds4_pool.hpp"
+#include "ds4_store.hpp"
+#include "ds4_vision.cuh"
+#include "strata/dsv41/exl3_cpu.hpp"
+#include "strata/dsv41/exl3.hpp"
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/dsv41/moe_win.hpp"
+#include "strata/dsv41/sampler.hpp"
+
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+
+#if defined(STRATA_METAL)
+extern "C" void ds4_busy_begin();
+extern "C" void ds4_busy_end();
+#include <sys/sysctl.h>
+#include <mach/mach.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <condition_variable>
+#include <deque>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <map>
+#include <mutex>
+#include <numeric>
+#include <sstream>
+#include <thread>
+#include <unordered_map>
+
+#if defined(_MSC_VER)
+#define strtok_r strtok_s
+#endif
+
+namespace strata::ds4 {
+namespace x3 = strata::dsv41k::exl3;
+namespace kc = strata::dsv41k::cpu;
+namespace gf = strata::dsv41k::glmf;
+namespace gg = strata::ds4::gguf;
+
+#define CK(call) \
+ do { \
+ cudaError_t e_ = (call); \
+ if (e_ != cudaSuccess) { \
+ err = std::string(#call) + ": " + cudaGetErrorString(e_); \
+ return false; \
+ } \
+ } while (0)
+
+namespace {
+
+double now_ms() {
+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+
+size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }
+
+// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:
+// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):
+// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before
+void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {
+ const int b = whole / 128, q = b / np, r = b % np;
+ auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };
+ int at = 0;
+ for (int i = 0; i < p; ++i) at += blocks(i);
+ off = 128 * at;
+ n = 128 * blocks(p);
+}
+
+std::string read_file(const std::string& p) {
+ std::ifstream f(p);
+ std::stringstream ss;
+ ss << f.rdbuf();
+ return ss.str();
+}
+std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
+ std::vector<int> v;
+ std::stringstream ss(s);
+ std::string part;
+ while (std::getline(ss, part, ',')) {
+ if (part.empty() || part == "\n") continue;
+ const size_t dash = part.find('-');
+ const int a = std::atoi(part.c_str());
+ const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);
+ for (int c = a; c <= b; ++c) v.push_back(c);
+ }
+ return v;
+}
+// each NUMA node's physical cores (the first logical CPU of each core), in order
+std::vector<std::vector<int>> node_cores() {
+#if defined(_WIN32)
+ static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)
+ const char* v = std::getenv("DS4_CPU_SMT");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ std::vector<std::vector<int>> nodes = os::node_cores(smt);
+#else
+ std::vector<std::vector<int>> nodes;
+ for (int n = 0; n < 64; ++n) {
+ const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");
+ if (cl.empty()) break;
+ std::vector<int> cores;
+ for (int c : cpu_list(cl)) {
+ const std::vector<int> sib = cpu_list(
+ read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));
+ if (sib.empty() || sib[0] == c) cores.push_back(c);
+ }
+ if (!cores.empty()) nodes.push_back(cores);
+ }
+#endif
+ if (nodes.empty()) {
+ std::vector<int> all;
+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);
+ nodes.push_back(all);
+ }
+ return nodes;
+}
+// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages
+void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }
+void pin_self(int cpu) { os::pin_self(cpu); }
+// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root
+// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known
+uint64_t ram_avail() {
+#if defined(_WIN32)
+ return os::ram_avail();
+#endif
+ uint64_t avail = ~uint64_t{0};
+ {
+ std::istringstream mi(read_file("/proc/meminfo"));
+ std::string k;
+ uint64_t kb = 0;
+ while (mi >> k >> kb) {
+ if (k == "MemAvailable:") avail = kb << 10;
+ mi.ignore(64, '\n');
+ }
+ }
+ std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."
+ const size_t at = cg.find("0::");
+ if (at == std::string::npos) return avail;
+ cg = cg.substr(at + 3);
+ cg = cg.substr(0, cg.find('\n'));
+ for (;;) {
+ const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);
+ const std::string mx = read_file(dir + "/memory.max");
+ if (!mx.empty() && mx.compare(0, 3, "max") != 0) {
+ const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);
+ const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);
+ uint64_t file = 0;
+ std::istringstream st(read_file(dir + "/memory.stat"));
+ std::string k;
+ uint64_t v = 0;
+ while (st >> k >> v)
+ if (k == "file") file = v;
+ const uint64_t used = cur > file ? cur - file : 0;
+ avail = std::min(avail, lim > used ? lim - used : 0);
+ }
+ if (cg.empty() || cg == "/") break;
+ const size_t sl = cg.rfind('/');
+ cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);
+ }
+ return avail;
+}
+
+float bf16_to_f32(uint16_t b) {
+ uint32_t u = (uint32_t) b << 16;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+float f16_to_f32(uint16_t h) {
+ const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
+ uint32_t u;
+ if (e == 0) {
+ if (m == 0) u = s;
+ else { // subnormal
+ int ee = -1;
+ uint32_t mm = m;
+ do {
+ ++ee;
+ mm <<= 1;
+ } while ((mm & 0x400) == 0);
+ u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;
+ }
+ } else if (e == 31) u = s | 0x7f800000u | m << 13;
+ else u = s | (e - 15 + 127) << 23 | m << 13;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+
+// ------------------------------------------------------------------ the pack
+struct Tensor {
+ std::string dtype;
+ int file = -1;
+ uint64_t off = 0, bytes = 0;
+ std::vector<int64_t> shape;
+};
+
+struct Pack {
+ std::string dir, model_dir;
+ std::unordered_map<std::string, std::string> kv;
+ std::vector<std::string> files;
+ std::vector<os::fd_t> fds;
+ std::unordered_map<std::string, Tensor> t;
+ int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;
+ std::vector<int32_t> eg_map;
+ std::vector<int64_t> eg_mult, eg_primes, eg_off;
+ // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the
+ // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)
+ bool gguf = false;
+
+ std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)
+ std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)
+ std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)
+
+ ~Pack() {
+ for (auto& m : maps) os::unmap(m.first, m.second);
+ for (os::fd_t fd : fds) os::close_fd(fd);
+ for (os::fd_t fd : dfds) os::close_fd(fd);
+ for (os::fd_t fd : pfds) os::close_fd(fd);
+ }
+ // file i mapped whole, read-only (once; not thread-safe: called while loading)
+ const uint8_t* map(int file) {
+ if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});
+ auto& m = maps[(size_t) file];
+ if (m.first == nullptr) {
+ const uint64_t size = os::file_size(fds[(size_t) file]);
+ void* p = os::map_ro(fds[(size_t) file], size);
+ if (p == nullptr) return nullptr;
+ m = {(uint8_t*) p, (size_t) size};
+ }
+ return m.first;
+ }
+
+ bool load(const std::string& d, std::string& err) {
+ dir = d;
+ {
+ std::ifstream f(d + "/ds4.txt");
+ if (!f) {
+ err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";
+ return false;
+ }
+ std::string line;
+ while (std::getline(f, line)) {
+ const size_t sp = line.find(' ');
+ if (sp == std::string::npos) continue;
+ kv[line.substr(0, sp)] = line.substr(sp + 1);
+ }
+ }
+ model_dir = str("model_dir");
+ std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");
+ if (f == nullptr) {
+ err = d + "/tensors.txt: not found";
+ return false;
+ }
+ std::vector<char> buf(1 << 16);
+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
+ char* s = buf.data();
+ if (s[0] == 'f') { // file <i> <name>
+ char name[4096];
+ int i = 0;
+ if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
+ if ((int) files.size() <= i) files.resize((size_t) i + 1);
+ files[(size_t) i] = name;
+ }
+ continue;
+ }
+ if (s[0] != 't') continue;
+ char* save = nullptr;
+ ::strtok_r(s, " \n", &save);
+ const char* name = ::strtok_r(nullptr, " \n", &save);
+ const char* dt = ::strtok_r(nullptr, " \n", &save);
+ const char* fi = ::strtok_r(nullptr, " \n", &save);
+ const char* off = ::strtok_r(nullptr, " \n", &save);
+ const char* nb = ::strtok_r(nullptr, " \n", &save);
+ if (name == nullptr || nb == nullptr) continue;
+ Tensor x;
+ x.dtype = dt;
+ x.file = std::atoi(fi);
+ x.off = std::strtoull(off, nullptr, 10);
+ x.bytes = std::strtoull(nb, nullptr, 10);
+ while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));
+ t.emplace(name, std::move(x));
+ }
+ std::fclose(f);
+ for (const std::string& fn : files) {
+ const std::string p = model_dir + "/" + fn;
+ const os::fd_t fd = os::open_ro(p);
+ if (fd == os::kBadFd) {
+ err = p + ": cannot open";
+ return false;
+ }
+ fds.push_back(fd);
+ }
+ gguf = str("format") == "gguf";
+ if (gguf)
+ for (size_t i = 0; i < fds.size(); ++i) {
+ if (map((int) i) == nullptr) {
+ err = model_dir + "/" + files[i] + ": cannot map";
+ return false;
+ }
+ dfds.push_back(os::open_direct(model_dir + "/" + files[i]));
+ pfds.push_back(os::open_par(model_dir + "/" + files[i]));
+ }
+ // the Engram tables
+ std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");
+ if (e == nullptr) {
+ err = d + "/engram.bin: not found";
+ return false;
+ }
+ char magic[4];
+ int32_t hdr[6];
+ bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&
+ std::fread(hdr, 4, 6, e) == 6;
+ if (ok) {
+ eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];
+ const int cols = (eg_ngram - 1) * eg_heads;
+ eg_map.resize((size_t) eg_vocab);
+ eg_mult.resize((size_t) eg_layers * eg_ngram);
+ eg_primes.resize((size_t) eg_layers * cols);
+ eg_off.resize((size_t) eg_layers * cols);
+ ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&
+ std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&
+ std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&
+ std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();
+ }
+ std::fclose(e);
+ if (!ok) {
+ err = d + "/engram.bin: unreadable";
+ return false;
+ }
+ return true;
+ }
+
+ std::string str(const std::string& k, const std::string& def = "") const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : it->second;
+ }
+ double num(const std::string& k, double def) const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : std::atof(it->second.c_str());
+ }
+ std::vector<int> list(const std::string& k) const {
+ std::vector<int> v;
+ std::stringstream ss(str(k));
+ std::string x;
+ while (std::getline(ss, x, ','))
+ if (!x.empty()) v.push_back(std::atoi(x.c_str()));
+ return v;
+ }
+ const Tensor* get(const std::string& n) const {
+ auto it = t.find(n);
+ return it == t.end() ? nullptr : &it->second;
+ }
+ bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)
+ if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)
+ return os::pread_par(pfds[(size_t) file], dst, n, off);
+ return os::pread_full(fds[(size_t) file], dst, n, off);
+ }
+ bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }
+ const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)
+};
+
+// ------------------------------------------------------------------ geometry
+struct Geo {
+ int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,
+ k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,
+ cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;
+ float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;
+ double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;
+ std::vector<int> ratios, kv_src, ix_src, eg_layers;
+ // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks
+ int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;
+ std::vector<int> d_targets;
+
+ void from(const Pack& p) {
+ vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);
+ nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);
+ hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);
+ q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);
+ o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);
+ k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);
+ win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);
+ sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);
+ ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);
+ cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);
+ cand_blk = (int) p.num("candidate_block_size", 0);
+ orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);
+ eos = (int) p.num("eos_token_id", 1);
+ eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);
+ route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);
+ rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);
+ factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);
+ beta_slow = p.num("rope_scaling.beta_slow", 1);
+ ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");
+ ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");
+ d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);
+ d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);
+ d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");
+ // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)
+ n_mtp = d_block > 0 && (int) ratios.size() > nl &&
+ (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)
+ ? (int) ratios.size() - nl : 0;
+ }
+};
+
+// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them
+std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {
+ std::vector<float> f((size_t) dim / 2);
+ for (int i = 0; i < dim / 2; ++i) {
+ const float e = (float) (2 * i) / (float) dim;
+ f[(size_t) i] = 1.0f / std::pow((float) base, e);
+ }
+ if (original > 0) {
+ auto corr = [&](double rot) {
+ return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)
+ };
+ const double low = std::max(std::floor(corr(beta_fast)), 0.0);
+ const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);
+ const float den = (float) std::max(high - low, 1e-3);
+ for (int i = 0; i < dim / 2; ++i) {
+ float ramp = ((float) i - (float) low) / den;
+ ramp = std::min(1.0f, std::max(0.0f, ramp));
+ const float smooth = 1.0f - ramp;
+ f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;
+ }
+ }
+ return f;
+}
+
+// ------------------------------------------------------------------ devices, layers
+struct Dev {
+ int id = 0;
+ cudaStream_t st = nullptr;
+ cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)
+ int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)
+ float* rw_a = nullptr;
+ size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;
+ std::vector<void*> allocs;
+ // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)
+ float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,
+ *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,
+ *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,
+ *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,
+ *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,
+ *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,
+ *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,
+ *ed = nullptr;
+ float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;
+ unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)
+ uint8_t* eg_rows = nullptr;
+ uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows
+ AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)
+ // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the
+ // driver's staging, ~1.7 ms a copy): h | pre | topk | cand
+ uint8_t* bounce = nullptr;
+ cudaEvent_t ev_out = nullptr;
+ float* tap = nullptr; // a window's stream means before a DSpark target layer
+ // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's
+ // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window
+ // MoE's grouping scratch
+ int grp = 0, mi = 0;
+ unsigned ar_seq = 0;
+ unsigned* xcnt = nullptr;
+ // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
+ // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
+ x3::Mat head;
+ float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;
+ unsigned head_seq = 0;
+ // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the
+ // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)
+ uint16_t *mk_share = nullptr, *mk_erow = nullptr;
+ int* mk_ints = nullptr;
+ int mk_v0 = 0, mk_vn = 0;
+ uint8_t* hand = nullptr;
+ unsigned* hand_flag = nullptr;
+ unsigned long long* grp_ptr = nullptr;
+ int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+};
+
+// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of
+// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS
+struct Chunk {
+ int C = 0, A = 0, AI = 0;
+ float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,
+ *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,
+ *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,
+ *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,
+ *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;
+ uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,
+ *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;
+ size_t w16_elems = 0;
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;
+ uint8_t* eg_rows = nullptr;
+ int* bounds_pin = nullptr; // pinned: the sort's bounds, read back
+ // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,
+ // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)
+ static constexpr int kParts = 4;
+ uint8_t* ring = nullptr;
+ size_t ring_stride = 0;
+ int ring_n = 0;
+ cudaStream_t cs = nullptr;
+ cudaEvent_t filled[kParts] = {}, freed[kParts] = {};
+ // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of
+ // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,
+ // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s
+ uint8_t* stage = nullptr;
+ size_t stage_stride = 0;
+ std::unique_ptr<Pool> readers;
+ cublasHandle_t blas = nullptr;
+ // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each
+ // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)
+ uint8_t* hand[2] = {nullptr, nullptr};
+ unsigned* hand_flag[2] = {nullptr, nullptr};
+ unsigned* hand_ack = nullptr;
+ unsigned arb_seq = 0; // the group's chunk all-reduces
+ // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)
+ uint16_t* emb_host[2] = {nullptr, nullptr};
+ uint8_t* eg_host[2] = {nullptr, nullptr};
+ cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};
+ double stream_bytes = 0; // the experts streamed in, in all
+ // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a
+ // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the
+ // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once
+ // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io
+ // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits
+ // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot
+ // i % ring_n; the compute takes the items as they land, in runs of what has.
+ struct Item {
+ int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)
+ int mi = 0; // the device's member index in the layer's group
+ bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA
+ const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read
+ const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)
+ uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there
+ int64_t after = -1; // (layer-major) the items up to this one read first
+ std::atomic<int> left{0}; // parts still being read
+ std::atomic<bool> bad{false};
+ };
+ bool flow = false;
+ bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring
+ int dev = -1; // the device (index into devs) this chunk state belongs to
+ std::unique_ptr<IoQueue> io;
+ std::deque<Item> items; // items[0] is item `base`; the indices below are absolute
+ size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;
+ uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)
+ int S = 0; // staging slots, stage_stride apart
+ std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)
+ std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use
+ int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
+ size_t spec_begin = 0;
+ std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
+ int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
+ int *bc_pin = nullptr, *bc_dev = nullptr;
+ uint64_t n_batch = 0;
+ double wait_ms = 0; // the compute waiting for a read
+ uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;
+ uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started
+ // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through
+ // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in
+ // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for
+ // its later chunks, so each is read once a run
+ bool lm = false, lm_pinned = false;
+ uint8_t* lm_state = nullptr;
+ size_t lm_state_bytes = 0;
+ uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side
+ size_t lc_bytes = 0;
+ int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all
+ size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read
+ std::vector<int> lc_of; // expert -> its slot (-1: none)
+ std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it
+ uint64_t it_lc = 0; // items taken from the layer cache
+};
+
+struct Layer {
+ int il = 0, dev = 0, ratio = 0;
+ // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the
+ // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert
+ // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)
+ int grp = 0, nm = 1, mi = 0;
+ std::vector<Layer> mates;
+ x3::ExpertLayout hlay;
+ size_t hblob = 0;
+ int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)
+ // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more
+ // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their
+ // attention adds nothing to the sum
+ int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;
+ // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's
+ // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member
+ int* vs_pin = nullptr;
+ int* vs_dev = nullptr;
+ std::vector<int> host_list;
+ std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."
+ int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)
+ bool mtp = false; // a DSpark block (index stage = il - n_layers)
+ bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;
+ int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)
+ int ik_src = -1; // the kv source whose index keys its indexer reads
+ int eg = -1; // engram layer index, or -1
+ x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;
+ float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,
+ *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,
+ *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,
+ *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)
+ uint16_t* i_wproj = nullptr;
+ float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)
+ uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)
+ float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)
+ // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row
+ uint8_t* snap_win = nullptr;
+ float *snap_kv = nullptr, *snap_sc = nullptr;
+ // routed experts
+ x3::ExpertLayout lay;
+ size_t blob = 0; // bytes of one expert's blob
+ int n_vram = 0;
+ bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's
+ uint8_t* vram_base = nullptr;
+ std::vector<int> slot; // expert -> VRAM slot or -1
+ std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
+ std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)
+ // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),
+ // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)
+ struct Half {
+ size_t trellis[3], suh[3], svh[3], bytes;
+ int k[3], n[3];
+ } hl;
+ std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes
+ // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it
+ // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)
+ int n_slots = 0, spare_vram = -1;
+ std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};
+ std::vector<float> heat;
+ std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked
+ bool swap_pending = false;
+ unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes
+ // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and
+ // the window's raw rows (kv, then score) to replay the accepted ones
+ uint8_t* ring_bak = nullptr;
+ float *c_snap = nullptr, *c_rows = nullptr;
+ unsigned long long* vram_tab = nullptr; // device [n_exp]
+ std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe
+ unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it
+ std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
+ std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
+ std::vector<uint64_t> region_bytes;
+ std::vector<int> region_file;
+ size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region
+ size_t piece_bytes[3][3];
+ size_t dense_bytes = 0;
+ // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +
+ // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane
+ int mx_file[3] = {-1, -1, -1};
+ uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};
+};
+
+struct Src {
+ int layer = 0, ratio = 1;
+ int64_t cap = 0;
+ // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)
+ std::vector<uint8_t*> ckv, ik;
+ uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices
+ cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;
+ // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows
+ // (ckv rows, then ik rows at kWin * hd) and its flag
+ bool remote = false;
+ uint8_t* x_stage = nullptr;
+ unsigned* x_flag = nullptr;
+ // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements
+ // [slot][device] (32 apart); readers: the devices outside the source's group that read it
+ uint8_t* x_stage1 = nullptr;
+ unsigned* x_flag1 = nullptr;
+ unsigned* x_ack = nullptr;
+ std::vector<int> readers;
+ uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }
+ unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }
+};
+
+// a host thread that enqueues one device's work (threads mode)
+struct Worker {
+ std::thread th;
+ std::mutex mu;
+ std::condition_variable cv;
+ std::function<void()> job;
+ bool busy = false, quit = false;
+ void start() {
+ th = std::thread([this] {
+ for (;;) {
+ std::function<void()> j;
+ {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return quit || job != nullptr; });
+ if (quit) return;
+ j = std::move(job);
+ job = nullptr;
+ }
+ j();
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ busy = false;
+ }
+ cv.notify_all();
+ }
+ });
+ }
+ void submit(std::function<void()> f) {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ job = std::move(f);
+ busy = true;
+ }
+ cv.notify_all();
+ }
+ void wait() {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return !busy; });
+ }
+ bool idle() {
+ std::lock_guard<std::mutex> lk(mu);
+ return !busy;
+ }
+ void stop() {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ quit = true;
+ }
+ cv.notify_all();
+ if (th.joinable()) th.join();
+ }
+};
+
+struct Group {
+ std::vector<int> mem; // device indices, member order (member 0 the lead)
+ ArGroup ar, arb; // a window's all-reduces, a chunk's
+ // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the
+ // lead's acknowledgements (host-mapped)
+ float* lg_stage = nullptr;
+ unsigned *lg_flag = nullptr, *lg_ack = nullptr;
+};
+
+constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers
+
+// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the
+// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate
+// blocks as each layer sees them, the window's hand-off seq
+struct WinPlan {
+ bool par = false, async = false;
+ unsigned win = 0;
+ int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2
+ int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)
+ int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)
+ unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)
+ unsigned seq[kMaxL] = {};
+ int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};
+};
+
+} // namespace
+
+// ------------------------------------------------------------------ the model
+struct Model::Impl {
+ Options o;
+ Pack pk;
+ Geo g;
+ std::vector<Dev> devs;
+ std::vector<Layer> L;
+ std::vector<Src> srcs;
+ std::vector<Group> groups;
+ std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device
+ bool par = false; // threads mode (the windows: decode)
+ bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked
+ unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)
+ // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows
+ int chunk = 0, chunk_min = 512;
+ static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)
+ // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk
+ // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers
+ // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token
+ static constexpr int kMaxChunk = 4 * kVlRows;
+ std::vector<std::unique_ptr<Chunk>> chunks; // per device
+ size_t eg_cap = kWin; // tokens h_eg holds
+ int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
+ bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
+ std::vector<float> chunk_logits;
+ std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
+ std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
+ std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
+ // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
+ // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took
+ struct PathModel {
+ bool on = false;
+ double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)
+ // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and
+ // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)
+ double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};
+ int64_t kn[2][4] = {};
+ double win_s = 0; // windows: seconds a window took, so far
+ int64_t win_n = 0;
+ int last_cls = 0, last_nw = 0; // the last pick's size class, its windows
+ double m = 1; // the picks' spread, against the profile's
+ int64_t n_m = 0;
+ int last = -1; // the last pick: 1 the flow, 0 windows
+ double last_pred = 0; // its prediction (s, before its k)
+ } pm;
+ static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)
+ static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)
+ std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through
+ WinPlan wplan;
+ int head_dev = 0;
+ x3::Mat head;
+ float* norm_w = nullptr; // on the head device
+ // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the
+ // confidence head (on the head device: the drafts' logits come from the trunk's head)
+ x3::Mat d_main_proj;
+ float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
+ uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
+ // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
+ // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
+ // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
+ bool mk_split = false;
+ uint16_t* h_mk_embed = nullptr;
+ unsigned* mk_slots = nullptr;
+ unsigned mk_seq = 0;
+ int d_dev = 0; // the drafter's first block's device
+ bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)
+ float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps
+ cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it
+ float* d_confv = nullptr; // on the head device
+ int* d_ints = nullptr; // on the head device: drafts in, picks out
+ float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;
+ // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for
+ // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()
+ // checks it is its own)
+ strata::dsv41k::SpecList* d_specq = nullptr;
+ std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);
+ bool drafts_q = false;
+ float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;
+ int dq_top_k = 0;
+ uint64_t dq_seed = 0, dq_counter = 0;
+ // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the
+ // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5
+ // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the
+ // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs
+ // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)
+ bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;
+ float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;
+ int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;
+ std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;
+ float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)
+ uint16_t* h_dump_ring = nullptr;
+
+ uint16_t* embed_pinned = nullptr;
+ std::vector<float> freqs_win, freqs_comp;
+ std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores
+ std::vector<std::vector<int>> cores; // each node's physical cores
+ int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)
+ // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps
+ // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)
+ bool file_experts = false;
+ uint64_t warm_bytes = 0;
+ static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)
+ uint8_t* file_part(const Layer& l, int e) const {
+ return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];
+ }
+ // host side of the CPU lane, a window's rows
+ float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;
+ int* h_idx = nullptr;
+ int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks
+ uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales
+ // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one
+ // an Engram layer: asked for when the window's tokens are known, collected right before that layer)
+ EgReader eg_io;
+ EgRowCache eg_cache;
+ std::mutex eg_cache_mu;
+ std::vector<std::unique_ptr<EgBatch>> eg_win;
+ // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1
+ // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8
+ // codes with their E8M0 scales in a tensor of their own (see eg_issue)
+ std::vector<uint8_t> eg_fmt;
+ std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
+ uint16_t* h_emb = nullptr; // the window's embedding rows
+ std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
+ // the state
+ int64_t pos = 0;
+ std::vector<int32_t> toks, ctoks;
+ struct Check { // the state at a position: every window ring, the compressors' partial groups
+ int64_t pos = -1;
+ std::vector<std::vector<uint8_t>> win;
+ std::vector<std::vector<float>> comp;
+ };
+ Check ck[2]; // the end of the last prompt, and the position before it
+ std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
+ Stats st;
+ int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
+ // (plan_window's: the windows' plans are made in order)
+ int n_logits = 0; // the last forward's logit rows
+ // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread
+ // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum
+ Req* reqs = nullptr;
+ Resp* resps = nullptr;
+ unsigned seq_issued = 0;
+ std::atomic<int> layer_of[kRing];
+ std::thread service;
+ std::atomic<bool> quit{false};
+#if defined(STRATA_METAL)
+ // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the
+ // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)
+ bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;
+#else
+ bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
+#endif
+ // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
+ // before the wait for the CPU lane - the same bits, no faster)
+ bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
+ int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this
+ // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same
+ // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files
+ // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next
+ // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the
+ // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's
+ // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).
+ int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;
+ int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;
+ // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)
+ bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;
+ std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;
+ // 8 asked for (prefetch)
+ std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)
+ std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for
+ std::thread pf_th;
+ std::mutex pf_mu;
+ std::condition_variable pf_cv;
+ std::vector<std::pair<const void*, size_t>> pf_q;
+ // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster
+ // on one V100 - more for a PC to try: each its staging buffers and
+ // copy streams), switched between windows
+ struct Swap {
+ int layer = 0, in = -1, out = -1;
+ int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM
+ };
+ static constexpr int kMaxSwaps = 8;
+ // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay
+ // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)
+ static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }
+ const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),
+ c_decay = env_f("DS4_CACHE_DECAY", 0.9f);
+ const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));
+ // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the
+ // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the
+ // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the
+ // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in
+ // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4
+ // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.
+ struct RamSwap {
+ int layer = 0, in = -1, out = -1, slot = -1;
+ };
+ std::vector<uint8_t*> ram_slots, ram_chunks;
+ std::vector<size_t> ram_chunk_bytes;
+ bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)
+ size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers
+ std::vector<int> ram_free;
+ std::deque<RamSwap> ram_queue;
+ std::vector<RamSwap> ram_ready;
+ std::condition_variable ram_cv;
+ std::thread ram_th;
+ int ram_inflight = 0, ram_held = 0;
+ bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;
+ // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a
+ // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route
+ // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every
+ // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the
+ // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left
+ // out still counts as used (the expert cache, the profile).
+ const float disk_skip_w = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");
+ return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;
+ }();
+ const int disk_skip_from = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
+ return v != nullptr ? std::max(0, std::atoi(v)) : 0;
+ }();
+ // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
+ std::unique_ptr<ExpertStore> store;
+ uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token
+ size_t zc_stride = 0;
+ bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves
+ bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r)
+ if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;
+ return true;
+ }
+ return !file_experts || os::in_ram(file_part(l, e), l.blob);
+ }
+ // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads
+ // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between
+ // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of
+ // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'
+ // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes
+ // the RAM place of the one it takes in)
+ // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private
+ // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever
+ // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the
+ // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds
+ // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the
+ // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold
+ // them (the log's WARNING, a NOTE and INFO for the server)
+ double commit_kept = 0, pagefile_hint = 0;
+ double commit_margin() const {
+ const os::Commit c = os::commit();
+ return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);
+ }
+ // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)
+ double commit_room() const {
+ const os::Commit c = os::commit();
+ double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;
+ if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);
+ return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());
+ }
+ // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)
+ bool commit_charged(const Dev& d) const {
+ if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;
+#if defined(_WIN32)
+ int v = 0;
+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return false;
+ }
+ return v != 0;
+#else
+ (void) d;
+ return false;
+#endif
+ }
+ bool copy_mode = false;
+ std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts
+ std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes
+ std::vector<int> copy_spare; // layer -> its spare slot (-1: none)
+ std::mutex sw_mu;
+ std::condition_variable sw_cv;
+ std::deque<Swap> sw_queue;
+ std::vector<Swap> sw_ready;
+ struct SwapBufs {
+ uint8_t *a = nullptr, *b = nullptr, *h = nullptr;
+ std::vector<cudaStream_t> st; // a copy stream a device
+ };
+ std::vector<SwapBufs> sw_bufs;
+ std::vector<std::thread> cache_ths;
+ int sw_inflight = 0;
+ int64_t decay_tokens = 0;
+ bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;
+ std::vector<float> host_logits;
+ std::vector<void*> host_allocs;
+ std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;
+ // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or
+ // chunk path (tests: where the two part)
+ std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;
+ // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records
+ // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)
+ std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;
+ std::vector<int> dump_s_layers = [] {
+ std::vector<int> v;
+ const char* e = std::getenv("DS4_DUMP_S_LAYERS");
+ std::string str = e ? e : "0,1,2,3";
+ size_t a = 0;
+ while (a < str.size()) {
+ size_t b = str.find(',', a);
+ if (b == std::string::npos) b = str.size();
+ if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));
+ a = b + 1;
+ }
+ return v;
+ }();
+ // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),
+ // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the
+ // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias
+ // [n_exp] (F32)
+ std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;
+ // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,
+ // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]
+ // (F16) and y rows [rows dim] (F32)
+ int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;
+ std::FILE* dump_y = [] {
+ const char* v = std::getenv("DS4_DUMP_Y");
+ const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;
+ return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;
+ }();
+ std::mutex dump_mu;
+ // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its
+ // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU
+ // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it
+ bool prof_on = std::getenv("DS4_PROF") != nullptr;
+ std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;
+ struct ProfMark {
+ const Dev* d;
+ cudaEvent_t ev;
+ int stage;
+ };
+ std::mutex prof_mu;
+ std::vector<ProfMark> prof_marks;
+ std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on
+ bool prof_window = false; // (decode windows only: T == 1)
+ // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
+ // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
+ bool prof_chunk = false;
+ double cprof_ms[4] = {};
+ void prof_mark(const Dev& d, int stage) {
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ int cur = 0;
+ cudaGetDevice(&cur);
+ cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")
+ cudaEvent_t e = nullptr;
+ std::vector<cudaEvent_t>& pool = prof_pool[d.id];
+ if (pool.empty()) cudaEventCreate(&e);
+ else {
+ e = pool.back();
+ pool.pop_back();
+ }
+ cudaEventRecord(e, d.st);
+ cudaSetDevice(cur);
+ prof_marks.push_back({&d, e, stage});
+ }
+ void prof_collect() {
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ std::vector<std::pair<const Dev*, cudaEvent_t>> last;
+ for (const ProfMark& m : prof_marks) {
+ auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
+ if (it != last.end()) {
+ float ms = 0.0f;
+ cudaSetDevice(m.d->id);
+ if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
+ (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
+ it->second = m.ev;
+ } else {
+ last.push_back({m.d, m.ev});
+ }
+ }
+ for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);
+ prof_marks.clear();
+ if (!prof_chunk) ++st.prof_tokens;
+ }
+ void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
+ std::vector<float> v((size_t) T * g.hc * g.dim);
+ cudaSetDevice(d.id);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
+ cudaDeviceSynchronize();
+#endif
+ cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[3] = {il, p0, T};
+ std::fwrite(hdr, 8, 3, dump_l);
+ std::fwrite(v.data(), 4, v.size(), dump_l);
+ std::fflush(dump_l);
+ }
+
+ bool dump_s_on(int il) const {
+ if (dump_s == nullptr) return false;
+ for (int x : dump_s_layers)
+ if (x == il) return true;
+ return false;
+ }
+ bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)
+ static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();
+ return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;
+ }
+ void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {
+ if (!dump_s_on(il) || !dump_s_name(name)) return;
+ std::vector<uint8_t> v(nbytes);
+ cudaSetDevice(d.id);
+#if defined(STRATA_METAL)
+ ds4_metal_flush();
+ cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)
+#endif
+ cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ dump_sub_host(name, il, p0, T, v.data(), nbytes);
+ }
+ void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {
+ if (!dump_s_on(il) || !dump_s_name(name)) return;
+ std::lock_guard<std::mutex> lk(dump_mu);
+ char nm[16] = {};
+ std::strncpy(nm, name, 15);
+ const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};
+ std::fwrite(nm, 1, 16, dump_s);
+ std::fwrite(hdr, 8, 4, dump_s);
+ std::fwrite(host, 1, nbytes, dump_s);
+ std::fflush(dump_s);
+ }
+
+ ~Impl() {
+ eg_io.stop(); // (its threads write into h_eg)
+ for (auto& w : workers) w->stop();
+ for (size_t i = 0; i < chunks.size(); ++i)
+ if (chunks[i] && chunks[i]->blas) {
+ cudaSetDevice(devs[i].id);
+ cublasDestroy(chunks[i]->blas);
+ }
+ quit.store(true);
+ if (service.joinable()) service.join();
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ }
+ pf_cv.notify_all();
+ if (pf_th.joinable()) pf_th.join();
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ }
+ sw_cv.notify_all();
+ ram_cv.notify_all();
+ for (std::thread& t : cache_ths)
+ if (t.joinable()) t.join();
+ if (ram_th.joinable()) ram_th.join();
+ for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ for (void* p : d.allocs) cudaFree(p);
+ if (d.st) cudaStreamDestroy(d.st);
+ if (d.ev) cudaEventDestroy(d.ev);
+ }
+ for (void* p : host_allocs) cudaFreeHost(p);
+ for (Layer& l : L)
+ for (uint8_t* b : l.host)
+ if (b != nullptr) {
+ (void) b; // host blobs live in one mapping per layer, unmapped below
+ }
+ for (auto& m : host_maps) {
+ if (chunk > 0) cudaHostUnregister(m.first);
+ os::anon_free(m.first, m.second);
+ }
+ }
+ std::vector<std::pair<void*, size_t>> host_maps;
+
+ template <class T>
+ bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {
+ void* v = nullptr;
+ cudaSetDevice(d.id);
+ const size_t b = align256(n * sizeof(T));
+ if (cudaMalloc(&v, b) != cudaSuccess) {
+ err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st
+ // does not order against - a zeroing still queued there could wipe the weights just copied
+ if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {
+ cudaFree(v);
+ err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ if (acct) *acct += b;
+ else d.scratch += b;
+ p = (T*) v;
+ return true;
+ }
+ static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ // bytes of a VRAM slot of l's experts (a member's part of each, in a group)
+ static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }
+ // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64
+ // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share
+ int head_members(int nm) const {
+ int na = nm;
+ while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;
+ return na;
+ }
+ // host-mapped memory (the same address on the host and every device), zeroed
+ template <class T>
+ bool hmap(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";
+ return false;
+ }
+ std::memset(v, 0, n * sizeof(T) + 64);
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+ template <class T>
+ bool halloc(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {
+ err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";
+ return false;
+ }
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+
+ // ---- weights
+ bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {
+ std::vector<uint8_t> tmp(t.bytes);
+ if (!pk.read(t, tmp.data())) {
+ err = "read failed";
+ return false;
+ }
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
+ return true;
+ }
+ // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs
+ bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);
+ std::vector<uint8_t> w((size_t) n * rb);
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";
+ return false;
+ }
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = gg::kFmtGG;
+ m.K = ggt;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = (int) gg::type_row_bytes(ggt, k);
+ return true;
+ }
+ // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
+ // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
+ // standalone [n][k] matrix
+ bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ const bool q8 = t.dtype == "Q8_0";
+ // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)
+ const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;
+ if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);
+ if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)
+ auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };
+ std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +
+ std::to_string(np) + " parts";
+ return false;
+ }
+ const size_t rb = bytes_of((size_t) k);
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = bytes_of((size_t) wd);
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r)
+ std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = q8 ? (int) bytes_of((size_t) k) : k;
+ return true;
+ }
+ // a Q8_0 tensor's values as F32 (host)
+ static std::vector<float> q8_values(const uint8_t* q, size_t n) {
+ std::vector<float> v(n);
+ for (size_t b = 0; b < n / 32; ++b) {
+ const uint8_t* p = q + b * gg::kQ8Bytes;
+ const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));
+ for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];
+ }
+ return v;
+ }
+ // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone
+ // matrix of its own (the rotations' 128-blocks stay whole)
+ // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix
+ // splits evenly, its groups only as wide as that goes)
+ bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
+ bool cols = true, int rot = 0) {
+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))
+ return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
+ const Tensor* tr = pk.get(pre + ".trellis");
+ const Tensor* su = pk.get(pre + ".suh");
+ const Tensor* sv = pk.get(pre + ".svh");
+ if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {
+ err = pre + ": not an EXL3 matrix in the pack";
+ return false;
+ }
+ if (pk.get(pre + ".mul1") == nullptr) {
+ err = pre + ": only the mul1 codebook is covered";
+ return false;
+ }
+ m.k = (int) tr->shape[0] * 16;
+ m.n = (int) tr->shape[1] * 16;
+ m.K = (int) tr->shape[2] / 16;
+ m.cb = 2;
+ m.ld = 0;
+ if (np > 1) {
+ const int whole = cols ? m.n : m.k;
+ int at = 0, w = 0;
+ if (whole % 128 != 0 || whole / 128 < np) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +
+ std::to_string(np) + " parts of 128-blocks";
+ return false;
+ }
+ split128(whole, np, part, at, w, rot);
+ std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);
+ if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));
+ std::vector<uint8_t> t1, u1, v1;
+ if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)
+ const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;
+ t1.resize((size_t) (m.k / 16) * nth * tile);
+ for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)
+ std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);
+ u1 = u0;
+ v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));
+ m.n = w;
+ } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)
+ const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;
+ t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));
+ u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));
+ v1 = v0;
+ m.k = w;
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||
+ !dalloc(d, dv, v1.size() / 2, err, acct))
+ return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||
+ !dalloc(d, dv, sv->bytes / 2, err, acct))
+ return false;
+ if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ if (!x3::supported(m)) {
+ err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +
+ " not covered by the EXL3 kernels";
+ return false;
+ }
+ return true;
+ }
+ // a float tensor (F32 / F16 / BF16) as F32 on the device
+ bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {
+ const Tensor* t = pk.get(name);
+ if (t == nullptr) {
+ err = name + ": not in the pack";
+ return false;
+ }
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ size_t n = 0;
+ std::vector<float> v;
+ if (t->dtype == "F32") {
+ n = t->bytes / 4;
+ v.resize(n);
+ std::memcpy(v.data(), raw.data(), t->bytes);
+ } else if (t->dtype == "F16" || t->dtype == "BF16") {
+ n = t->bytes / 2;
+ v.resize(n);
+ const uint16_t* s = (const uint16_t*) raw.data();
+ for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);
+ } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)
+ n = t->bytes / gg::kQ8Bytes * 32;
+ v = q8_values(raw.data(), n);
+ } else {
+ err = name + ": dtype " + t->dtype + " is not a float";
+ return false;
+ }
+ if (want != 0 && n != want) {
+ err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);
+ return false;
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));
+ return true;
+ }
+ bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {
+ const Tensor* t = pk.get(name);
+ if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)
+ const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();
+ std::vector<uint16_t> h(n);
+ for (size_t i = 0; i < n; ++i) {
+ float v;
+ if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);
+ else if (t->dtype == "Q8_0") v = q[i];
+ else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);
+ h[i] = strata::kernels::f16_from_f32(v);
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));
+ return true;
+ }
+ if (t == nullptr || t->dtype != "F16") {
+ err = name + ": not an F16 tensor in the pack";
+ return false;
+ }
+ if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;
+ return upload(d, *t, p, err);
+ }
+
+ size_t mat_bytes(const std::string& pre) const {
+ if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
+ size_t b = 0;
+ for (const char* s : {".trellis", ".suh", ".svh"})
+ if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
+ return b;
+ }
+ size_t tensor_bytes(const std::string& name) const {
+ const Tensor* t = pk.get(name);
+ return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
+ }
+
+ // ---- expert regions and blob layout
+ // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every
+ // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
+ // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
+ static int gg_type_of(const std::string& dtype) {
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})
+ if (dtype == gg::type_name(t)) return t;
+ return -1;
+ }
+ bool expert_layout_mx(Layer& l, std::string& err) {
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;
+ const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");
+ const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;
+ const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;
+ const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;
+ if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {
+ err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +
+ "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";
+ return false;
+ }
+ l.mx_file[r] = t0->file;
+ l.mx_off[r] = t0->off;
+ l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");
+ if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||
+ t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {
+ err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";
+ return false;
+ }
+ }
+ l.lay.trellis[r] = off;
+ l.lay.suh[r] = l.lay.svh[r] = off;
+ l.lay.K[r] = ty;
+ off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)
+ }
+ l.lay.fmt = gg::kFmtMXFP4;
+ if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
+ gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||
+ gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {
+ err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +
+ " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";
+ return false;
+ }
+ l.blob = off;
+ // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where
+ // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM
+ for (int r = 0; r < 3; ++r)
+ if (!gg::type_on_device(l.lay.K[r])) {
+ l.cpu_only = true;
+ std::fprintf(stderr,
+ "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "
+ "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",
+ gg::type_name(l.lay.K[r]));
+ break;
+ }
+ return true;
+ }
+ const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
+ if (store) {
+ const int s = store->find(l.il, e);
+ if (s >= 0) return store->mat(s, r);
+ }
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
+ return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
+ : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
+ return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);
+ }
+ size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)
+ return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);
+ }
+ // (copy_mode) matrix r's rows, and the rows of one NUMA part of it
+ int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }
+ size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }
+ // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them
+ const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {
+ if (copy_mode && l.ram[(size_t) e] >= 0)
+ return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ }
+ // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)
+ void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);
+ }
+ }
+ void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);
+ }
+ }
+ // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down
+ // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back
+ bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {
+ if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);
+ else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);
+ return true;
+ }
+ return read_mx_file(l, e, blob);
+ }
+ bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)
+ for (int r = 0; r < 3; ++r)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),
+ blob + l.lay.trellis[r]))
+ return false;
+ return true;
+ }
+ void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);
+ }
+ void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);
+ }
+
+ bool expert_layout(Layer& l, std::string& err) {
+ if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)
+ return expert_layout_mx(l, err);
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ const char* pieces[3] = {".trellis", ".suh", ".svh"};
+ l.region_off.resize((size_t) l.n_exp);
+ l.region_bytes.resize((size_t) l.n_exp);
+ l.region_file.resize((size_t) l.n_exp);
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");
+ if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {
+ err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";
+ return false;
+ }
+ l.lay.K[r] = (int) tr->shape[2] / 16;
+ l.lay.trellis[r] = off;
+ off += align256(tr->bytes);
+ l.lay.suh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);
+ l.lay.svh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);
+ }
+ l.lay.cb = 2;
+ l.blob = off;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const std::string E = P + std::to_string(e) + ".";
+ uint64_t lo = UINT64_MAX, hi = 0;
+ int file = -1;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const Tensor* t = pk.get(E + parts[r] + pieces[q]);
+ if (t == nullptr) {
+ err = E + parts[r] + pieces[q] + ": missing";
+ return false;
+ }
+ if (file < 0) file = t->file;
+ if (t->file != file) {
+ err = E + ": split across files";
+ return false;
+ }
+ lo = std::min(lo, t->off);
+ hi = std::max(hi, t->off + t->bytes);
+ if (e == 0) l.piece_bytes[r][q] = t->bytes;
+ else if (t->bytes != l.piece_bytes[r][q]) {
+ err = E + parts[r] + ": not the shape of expert 0's";
+ return false;
+ }
+ }
+ l.region_off[(size_t) e] = lo;
+ l.region_bytes[(size_t) e] = hi - lo;
+ l.region_file[(size_t) e] = file;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;
+ if (e == 0) l.piece_src[r][q] = src;
+ else if (src != l.piece_src[r][q]) {
+ err = E + ": the pieces lie differently from expert 0's";
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ // the region of expert e (read from the file) -> its blob
+ void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};
+ for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);
+ }
+ }
+
+ // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)
+ void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = region + l.piece_src[r][0];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // ---- loading
+ bool load(const Options& opt, std::string& err) {
+ o = opt;
+ const double t0 = now_ms();
+ if (!pk.load(o.pack, err)) return false;
+ g.from(pk);
+ if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {
+ err = o.pack + "/ds4.txt: incomplete geometry";
+ return false;
+ }
+ if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {
+ err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +
+ std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";
+ return false;
+ }
+ freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);
+ freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);
+
+ // devices
+ int ndev = 0;
+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
+ err = "no CUDA device";
+ return false;
+ }
+ std::vector<int> ids = o.gpus;
+ if (ids.empty())
+ for (int i = 0; i < ndev; ++i) ids.push_back(i);
+ devs.resize(ids.size());
+ for (size_t i = 0; i < ids.size(); ++i) {
+ Dev& d = devs[i];
+ d.id = ids[i];
+ CK(cudaSetDevice(d.id));
+ CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));
+ CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));
+ if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {
+ err = "exl3 scratch on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ CK(cudaMemGetInfo(&d.free0, &d.total));
+ }
+ // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each
+ // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)
+ int tp = o.tp;
+ if (pk.gguf && tp > 1) {
+ int blk = 32;
+ for (const auto& kv : pk.t) {
+ const std::string& n = kv.first;
+ if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||
+ n.compare(n.size() - 9, 9, "w2.weight") != 0)
+ continue;
+ const int ty = gg_type_of(kv.second.dtype);
+ if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
+ }
+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||
+ g.o_groups % tp != 0))
+ --tp;
+ if (tp != o.tp)
+ std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "
+ "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);
+ }
+ // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone
+ // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the
+ // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the
+ // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down
+ // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
+ par = o.threads || tp > 1;
+ prompt_par = par || o.tp > 1;
+#if defined(STRATA_METAL)
+ if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path
+#endif
+ const bool tp_all = tp > 1 && tp == (int) devs.size();
+ for (size_t i = 0; i < devs.size();) {
+ Group G;
+ G.mem.push_back((int) i);
+ size_t j = i + 1;
+ while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))
+ G.mem.push_back((int) j++);
+ if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own
+ for (int m : G.mem) {
+ Group one;
+ one.mem.push_back(m);
+ groups.push_back(one);
+ }
+ } else groups.push_back(G);
+ i = j;
+ }
+ for (size_t gi = 0; gi < groups.size(); ++gi)
+ for (size_t m = 0; m < groups[gi].mem.size(); ++m) {
+ devs[(size_t) groups[gi].mem[m]].grp = (int) gi;
+ devs[(size_t) groups[gi].mem[m]].mi = (int) m;
+ }
+ // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the
+ // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM
+ // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place
+ // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times
+ if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM
+ // after VRAM's are in, by --mmap-experts)
+ else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;
+ else {
+ double exp_b = 0, other_b = 0, room = 0;
+ for (const auto& kt : pk.t) {
+ const std::string& n = kt.first;
+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
+ (!o.dspark && n.rfind("mtp.", 0) == 0))
+ continue;
+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
+ }
+ for (const Dev& d : devs)
+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -
+ (prompt_par ? 1.5 : 0.4) * (1 << 30)) *
+ o.vram_experts;
+ file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
+ }
+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;
+ const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)
+ if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
+ if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
+ if (chunk > 0) {
+ chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));
+ eg_cap = (size_t) chunk;
+ tap_cap = std::max(kWin, 128);
+ } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)
+ std::fprintf(stderr,
+ "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"
+ " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"
+ " 461 / 1010 in chunks of %d) - %s\n",
+ kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,
+ o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"
+ : "leave --tp 1 out, or pass --threads on, for chunks");
+ // the vision tower on the first GPU, before the split: the layers get what it leaves
+ if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,
+ if (o.vision_only) return false; // and the server is told pictures are off - info_lines)
+ std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "
+ "text only\n", err.c_str());
+ vision_err = err.empty() ? "unknown reason" : err;
+ err.clear();
+ vt.release();
+ img_tok = -1;
+ }
+ if (o.vision_only) return true;
+
+ // layers: roles and the sources they read
+ if (!o.dspark) g.n_mtp = 0;
+ L.resize((size_t) (g.nl + g.n_mtp));
+ int last_src = -1;
+ for (int il = 0; il < g.nl + g.n_mtp; ++il) {
+ Layer& l = L[(size_t) il];
+ l.il = il;
+ l.mtp = il >= g.nl;
+ l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
+ l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
+ l.k_exp = l.mtp ? g.d_k : g.k_exp;
+ l.ratio = g.ratios[(size_t) il];
+ if (l.mtp) { // window attention only: no compressor, indexer or Engram
+ if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
+ err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +
+ std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";
+ return false;
+ }
+ if (!expert_layout(l, err)) return false;
+ continue;
+ }
+ l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();
+ l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();
+ l.owns_k = l.kv_src;
+ l.cand_src = il == g.cand_src;
+ l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;
+ if (l.kv_src) {
+ Src s;
+ s.layer = il;
+ s.ratio = l.ratio;
+ s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;
+ srcs.push_back(s);
+ last_src = (int) srcs.size() - 1;
+ }
+ if (l.ratio > 0) {
+ if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {
+ err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +
+ " without a KV source of that ratio before it";
+ return false;
+ }
+ l.src = last_src;
+ l.ik_src = last_src;
+ }
+ auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);
+ if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());
+ if (!expert_layout(l, err)) return false;
+ }
+
+ // the split: each layer's dense weights + its experts, the head on the last GPU
+ for (Layer& l : L) {
+ const std::string& P = l.name;
+ size_t b = 0;
+ if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");
+ for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",
+ "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",
+ "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})
+ b += mat_bytes(P + m);
+ for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));
+ b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");
+ b += (size_t) g.win * win_rb() + (2u << 20);
+ l.dense_bytes = b;
+ }
+ const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);
+ const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;
+ const size_t kv_per_src_dev = 0; // counted after the split
+ (void) kv_per_src_dev;
+ std::vector<double> avail(groups.size(), 0.0);
+ double avail_sum = 0;
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ for (int m : groups[gi].mem)
+ avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));
+ avail_sum += avail[gi];
+ }
+ double need_sum = (double) head_b;
+ for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ {
+ // contiguous layer ranges, each group's share of the total by its free memory
+ size_t d = 0;
+ double cum = 0, target = avail[0] / avail_sum * need_sum;
+ std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers
+ for (Layer& l : L)
+ if (l.mtp) order.push_back(&l);
+ for (Layer& l : L)
+ if (!l.mtp) order.push_back(&l);
+ for (Layer* lp : order) {
+ Layer& l = *lp;
+ const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ if (cum + lb / 2 > target && d + 1 < groups.size()) {
+ ++d;
+ target += avail[d] / avail_sum * need_sum;
+ }
+ l.grp = (int) d;
+ cum += lb;
+ }
+ // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its
+ // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage
+ // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all
+ if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {
+ const int G = (int) groups.size(), NL = g.nl;
+ std::vector<double> cap(groups.size());
+ for (int gi = 0; gi < G; ++gi) {
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ double mn = 1e300;
+ for (int m : groups[(size_t) gi].mem)
+ mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -
+ 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)
+ cap[(size_t) gi] = mn * nm;
+ }
+ for (const Layer& l : L)
+ if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ cap[(size_t) G - 1] -= (double) head_b;
+ std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense
+ for (int il = 0; il < NL; ++il) {
+ pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;
+ pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;
+ }
+ auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);
+ return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;
+ };
+ // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)
+ std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));
+ std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));
+ for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);
+ for (int gi = 1; gi < G; ++gi)
+ for (int b = gi + 1; b <= NL; ++b)
+ for (int a = gi; a < b; ++a) {
+ const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));
+ if (v < best[(size_t) gi][(size_t) b]) {
+ best[(size_t) gi][(size_t) b] = v;
+ cut[(size_t) gi][(size_t) b] = a;
+ }
+ }
+ int b = NL;
+ for (int gi = G - 1; gi >= 0; --gi) {
+ const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;
+ for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;
+ b = a;
+ }
+ }
+ // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)
+ if (const char* sp = std::getenv("DS4_SPLIT")) {
+ std::vector<int> n;
+ for (const char* q = sp; *q;) {
+ n.push_back(std::atoi(q));
+ while (*q && *q != ',') ++q;
+ if (*q == ',') ++q;
+ }
+ int il = 0;
+ for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)
+ for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;
+ for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;
+ }
+ for (Layer& l : L) {
+ l.dev = groups[(size_t) l.grp].mem[0];
+ l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)
+ }
+ head_dev = L[(size_t) g.nl - 1].dev;
+ d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;
+ }
+ // the groups' members: copies of each layer's device state, a member's part of an expert
+ for (Layer& l : L) {
+ l.ff0 = 0;
+ l.ffn = g.ff;
+ l.ah0 = l.og0 = 0;
+ l.ahn = g.heads;
+ l.ogn = g.o_groups;
+ if (l.nm == 1) continue;
+ const int na = head_members(l.nm);
+ if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";
+ return false;
+ }
+ for (int m = 1; m < l.nm; ++m) {
+ Layer c = l;
+ c.mates.clear();
+ c.mi = m;
+ c.dev = groups[(size_t) l.grp].mem[(size_t) m];
+ l.mates.push_back(std::move(c));
+ }
+ // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even
+ // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its
+ // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)
+ const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const int ha = m - (l.nm - na); // (its place among the members with heads)
+ x.ahn = ha < 0 ? 0 : g.heads / na;
+ x.ah0 = ha < 0 ? 0 : ha * x.ahn;
+ x.ogn = ha < 0 ? 0 : g.o_groups / na;
+ x.og0 = ha < 0 ? 0 : ha * x.ogn;
+ if (gguf_x) {
+ x.ffn = g.ff / l.nm;
+ x.ff0 = m * x.ffn;
+ } else
+ split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;
+ x.hlay.K[r] = l.lay.K[r];
+ x.hlay.trellis[r] = off;
+ if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks
+ const int ty = gg::expert_type(l.lay, r);
+ if (k % gg::type_block(ty) != 0) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +
+ " blocks do not split";
+ return false;
+ }
+ x.hlay.suh[r] = x.hlay.svh[r] = off;
+ off += align256((size_t) n * gg::type_row_bytes(ty, k));
+ continue;
+ }
+ off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));
+ x.hlay.suh[r] = off;
+ off += align256((size_t) k * 2);
+ x.hlay.svh[r] = off;
+ off += align256((size_t) n * 2);
+ }
+ x.hlay.cb = l.lay.cb;
+ x.hlay.fmt = l.lay.fmt;
+ x.hblob = off;
+ }
+ }
+ // which devices hold each source's compressed KV / index keys
+ for (Src& s : srcs) {
+ s.ckv.assign(devs.size(), nullptr);
+ s.ik.assign(devs.size(), nullptr);
+ }
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ const Layer& l = member(l0, m);
+ if (l.src >= 0) {
+ srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;
+ }
+ if (l.ix_src && l.ik_src >= 0) {
+ srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;
+ }
+ }
+
+ // per device: the scratch, the caches, the dense weights
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ const int D = g.dim, H = g.hc;
+ const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&
+ dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&
+ dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&
+ dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&
+ dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&
+ dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&
+ dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&
+ dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&
+ dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&
+ dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&
+ dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&
+ dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&
+ dalloc(d, d.rw, W * kPicks, err) &&
+ dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&
+ dalloc(d, d.moe_out, W * D, err) &&
+ dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&
+ dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&
+ dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&
+ dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&
+ dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&
+ dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
+ dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
+ dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
+ if (ok && std::getenv("DS4_DBG_DEVH"))
+ std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);
+ if (!ok) return false;
+ d.attn_scr.max_t = kAttnScratchT;
+ d.attn_scr.cap = (int) topk_stride();
+ if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;
+ if (!g.eg_layers.empty()) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&
+ dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);
+ if (!ok) return false;
+ }
+ if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
+ if (!halloc(d.bounce, bounce_bytes(), err)) return false;
+ if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
+ !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
+ !dalloc(d, d.n_groups, 4, err))
+ return false;
+ if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;
+ if (g.n_mtp > 0 && (int) di == head_dev &&
+ (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))
+ return false;
+ CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));
+ for (Src& s : srcs) {
+ if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)
+ if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))
+ return false;
+ CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));
+ CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));
+ CK(cudaSetDevice(d.id));
+ }
+ if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))
+ return false;
+ if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))
+ return false;
+ }
+ }
+ for (Src& sr : srcs) {
+ if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))
+ return false;
+ if (sr.remote && chunk > 0 &&
+ (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||
+ !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))
+ return false;
+ const int sg = L[(size_t) sr.layer].grp;
+ for (size_t di = 0; di < devs.size(); ++di)
+ if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);
+ }
+ for (Group& G : groups) {
+ G.ar.nm = (int) G.mem.size();
+ G.ar.slots = 4;
+ G.ar.stride = (size_t) kWin * g.dim;
+ if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))
+ return false;
+ // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)
+ if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))
+ return false;
+ if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&
+ (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||
+ !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))
+ return false;
+ G.arb = G.ar; // (a chunk's: two slots of chunk rows)
+ G.arb.slots = 2;
+ G.arb.stride = (size_t) chunk * g.dim;
+ if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))
+ return false;
+ if (chunk > 0 && G.arb.nm > 2 &&
+ (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))
+ return false;
+ }
+ if (g.n_mtp > 0) {
+ if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;
+ CK(cudaSetDevice(devs[(size_t) d_dev].id));
+ CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));
+ CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));
+ }
+ // the dense weights, each device's layers by a thread of its own
+ {
+ std::vector<std::string> errs(devs.size());
+ std::vector<std::thread> th;
+ for (size_t di = 0; di < devs.size(); ++di)
+ th.emplace_back([this, di, &errs] {
+ std::string e;
+ if (!load_dense((int) di, e)) errs[di] = e;
+ });
+ for (auto& t : th) t.join();
+ for (auto& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);
+ // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E
+ // host-mapped, the steps' slots
+ // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)
+ bool mk_rows = g.n_mtp > 0 && head_nm() > 1;
+ for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;
+ if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {
+ const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");
+ if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {
+ err = "the markov embed: not a BF16 [vocab][rank] table";
+ return false;
+ }
+ if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||
+ !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {
+ if (err.empty()) err = "the markov embed: read failed";
+ return false;
+ }
+ devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)
+ mk_split = true;
+ }
+
+ // the embedding table, pinned on the host
+ {
+ const Tensor* t = pk.get("embed.weight");
+ const size_t nv = (size_t) g.vocab * g.dim;
+ const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
+ // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):
+ // each row through ggml's to_float, to the same BF16 the host table holds
+ const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;
+ const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;
+ const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;
+ if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {
+ err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";
+ return false;
+ }
+ if (!halloc(embed_pinned, nv, err)) return false;
+ if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)
+ std::vector<uint8_t> row(erb);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {
+ err = "embed.weight: ggml has no decode for " + t->dtype;
+ return false;
+ }
+ }
+ } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
+ std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);
+ for (size_t c = 0; c < (size_t) g.dim; ++c) {
+ uint32_t u;
+ std::memcpy(&u, &v[c], 4);
+ u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even
+ embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);
+ }
+ }
+ } else if (!pk.read(*t, embed_pinned)) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,
+ "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",
+ (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],
+ (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],
+ (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);
+
+ }
+ // the host side of the CPU lane and the Engram rows
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||
+ !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||
+ !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||
+ !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))
+ return false;
+ if (pk.eg_layers > 0) {
+ // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);
+ // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
+ const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
+ const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
+ int nth = (int) std::min(16u, std::max(4u, hw / 2));
+ if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
+ const uint64_t avail = ram_avail();
+ uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
+ if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);
+ eg_cache.init((size_t) ((mb << 20) / row), row);
+ std::vector<std::string> paths;
+ for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
+ eg_io.start(paths, nth);
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ eg_win.emplace_back(new EgBatch());
+ const std::string id = std::to_string(g.eg_layers[(size_t) li]);
+ const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");
+ // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read
+ // as some other format's bytes - the old code then read the scale tensor of a pack that has none and
+ // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.
+ int fmt;
+ if (tw == nullptr) {
+ std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());
+ std::exit(2);
+ } else if (tw->dtype == "MXFP8") {
+ fmt = 0;
+ } else if (tw->dtype == "Q6_K") {
+ fmt = 1;
+ } else if (tw->dtype == "IQ4_NL") {
+ fmt = 2;
+ } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {
+ fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)
+ } else {
+ std::fprintf(stderr,
+ "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "
+ "(MXFP8, Q6_K, IQ4_NL)\n",
+ id.c_str(), tw->dtype.c_str());
+ std::exit(2);
+ }
+ if (fmt == 1 && pk.eg_hd % 256 != 0) {
+ err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole superblocks of 256)";
+ return false;
+ }
+ if (fmt == 2 && pk.eg_hd % 32 != 0) {
+ err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole blocks of 32)";
+ return false;
+ }
+ eg_fmt.push_back((uint8_t) fmt);
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
+ "their layer; row cache %llu MB (%zu rows)\n",
+ nth, (unsigned long long) mb, eg_cache.capacity());
+ }
+ prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);
+ if (!o.profile.empty()) {
+ std::FILE* f = std::fopen(o.profile.c_str(), "rb");
+ if (f != nullptr) {
+ std::vector<uint32_t> p(prof.size());
+ const size_t n = std::fread(p.data(), 4, p.size(), f);
+ if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());
+ std::fclose(f);
+ }
+ }
+
+ // the chunked prompt path's buffers (before the experts: they take what is left)
+ if (chunk > 0 && !alloc_chunks(err)) return false;
+ // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM
+ // now leaves over
+ if (file_experts) {
+ double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;
+ }
+ for (const Layer& l : L) all += (double) l.blob * l.n_exp;
+ const double need = std::max(0.0, all - room);
+ const uint64_t avail = ram_avail();
+ warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
+ if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
+ if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)
+ std::fprintf(stderr,
+ "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
+ "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
+ need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"
+ : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",
+ avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);
+ }
+ // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in
+ // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)
+ cores = node_cores();
+ NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());
+ if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;
+ if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)
+ {
+ const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;
+ c_xh.resize((size_t) NH * S * 2 * E);
+ c_xsb.resize((size_t) NH * S * 2 * (E / 128));
+ c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));
+ c_xhd.resize(S * FF);
+ c_xsd.resize(S * (FF / 128));
+ c_accd.resize((size_t) NH * S * DQ * (E / NH));
+ c_yd.resize(S * E);
+ }
+ // the experts: VRAM slots in what each device has left, the rest on the host
+ if (!place_experts(err)) return false;
+ if (!load_experts(err)) return false;
+ if (pk.gguf) make_pools();
+ if (chunk > 0) {
+ // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
+ const double tr = now_ms();
+ for (auto& m : host_maps)
+ if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {
+ err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";
+ return false;
+ }
+ if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);
+ if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)
+ size_t mr = 0;
+ for (const Layer& l : L)
+ if (!l.mtp) {
+ for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)
+ mr = std::max(mr, l.blob);
+ for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));
+ }
+ }
+ const char* nr = std::getenv("DS4_READERS");
+ const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);
+ const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;
+ // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the
+ // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the
+ // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)
+ const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ auto& cp = chunks[di];
+ if (cp == nullptr || cp->ring == nullptr) continue;
+ cp->stage_stride = (mr + 4095) & ~(size_t) 4095;
+ // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)
+ bool fl = flow_on;
+ int n_exp_max = 0;
+ for (const Layer& l : L)
+ if (!l.mtp)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ // (a group's members: each its share of the down's blocks in every row)
+ fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&
+ (l.nm == 1 || (flow_tp && (g.ff / l.nm) %
+ std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));
+ n_exp_max = std::max(n_exp_max, l.n_exp);
+ }
+ if (fl) // (each part read unbuffered from its offset rounded down: a page more each)
+ cp->stage_stride = std::max(cp->stage_stride, flow_stride());
+ cp->S = cp->ring_n;
+ if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;
+ cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);
+ if (!fl) continue;
+ cudaSetDevice(devs[di].id);
+ cp->flow = true;
+#if defined(STRATA_METAL)
+ cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;
+#endif
+ cp->dev = (int) di;
+ cp->io = std::make_unique<IoQueue>(n_readers);
+ cp->stage_item.assign((size_t) cp->S, -1);
+ cp->stage_ev.resize((size_t) cp->S);
+ cp->ring_fill.resize((size_t) cp->ring_n);
+ cp->ring_free.resize((size_t) cp->ring_n);
+ for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})
+ for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ // (a region is written again only bc_n batches later - more than a layer's items, so after that
+ // layer's bounds wait, which every copy from it precedes)
+ cp->bc_n = n_exp_max + 2;
+ cp->bc_w = cp->ring_n + 1;
+ if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||
+ !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))
+ return false;
+ }
+ // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one
+ // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each
+ // window, a page fault at a time) - DS4_CHUNK_MIN still overrides
+ bool all_flow = true;
+ for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);
+ if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;
+ }
+ for (Layer& l : L) {
+ if (l.mtp) continue;
+ for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)
+ Layer& x = member(l, m);
+ if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;
+ if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;
+ }
+ }
+ }
+ {
+ void* m = nullptr;
+ if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's requests";
+ return false;
+ }
+ host_allocs.push_back(m);
+ reqs = (Req*) m;
+ if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's answers";
+ return false;
+ }
+ host_allocs.push_back(m);
+ resps = (Resp*) m;
+ std::memset((void*) reqs, 0, sizeof(Req) * kRing);
+ std::memset((void*) resps, 0, sizeof(Resp) * kRing);
+ for (auto& x : layer_of) x.store(-1);
+ ahead_fl.assign(L.size() * 1024, 0);
+ ahead_rank.assign(L.size() * 1024, 0);
+ ahead_for.assign(L.size(), 0);
+ const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;
+ service = std::thread([this, core] {
+ if (core >= 0) pin_self(core);
+ serve_lane();
+ });
+ if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });
+ }
+ if (o.expert_cache) {
+ size_t maxb = 0;
+ for (const Layer& l : L) maxb = std::max(maxb, l.blob);
+ const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));
+ sw_bufs.resize((size_t) nt);
+ for (SwapBufs& B : sw_bufs) {
+ if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStream_t cs = nullptr;
+ CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
+ B.st.push_back(cs);
+ }
+ }
+ for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ }
+ if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)
+ for (size_t di = 0; di < devs.size(); ++di) {
+ workers.push_back(std::make_unique<Worker>());
+ workers.back()->start();
+ }
+ calibrate_paths();
+ init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)
+ preload_store();
+ if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
+ reset();
+ return true;
+ }
+
+
+ // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed
+ // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)
+ size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }
+ size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
+ size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
+
+ // ---- the chunked prompt path
+ size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
+ size_t chunk_hand_bytes() const {
+ return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+
+ bool alloc_chunks(std::string& err) {
+ chunks.resize(devs.size());
+ const int D = g.dim, H = g.hc, HD = g.hd;
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ cudaSetDevice(d.id);
+ chunks[di] = std::make_unique<Chunk>();
+ Chunk& c = *chunks[di];
+ const int nm = (int) groups[(size_t) d.grp].mem.size();
+ const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);
+ int FF = 0, ff0_ = 0;
+ split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
+ const size_t C = (size_t) chunk;
+ c.C = chunk;
+ c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
+ // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
+ c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
+ c.w16_elems = (size_t) 8 << 20;
+ bool has_eg = false, has_first = false;
+ size_t stride = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di && !l.mtp) {
+ has_eg = has_eg || l.eg >= 0;
+ has_first = has_first || l.il == 0;
+ stride = std::max(stride, align256(unit(member(l, m))));
+ }
+ const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&
+ dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&
+ dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&
+ dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&
+ dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&
+ dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&
+ dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&
+ dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&
+ dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&
+ dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&
+ dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&
+ dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&
+ dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&
+ dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&
+ dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&
+ dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
+ dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
+ dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
+ dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
+ halloc(c.bounds_pin, 2048, err);
+ if (!ok) return false;
+ if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
+ if (has_eg) {
+ const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);
+ if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||
+ !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))
+ return false;
+ }
+ if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches
+ const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;
+ c.ring_stride = stride;
+ c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);
+ if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;
+ }
+ CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));
+ for (int i = 0; i < Chunk::kParts; ++i) {
+ CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));
+ }
+ if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {
+ err = "cuBLAS on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ for (int sl = 0; sl < 2; ++sl) {
+ if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;
+ if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;
+ if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;
+ CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));
+ }
+ if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;
+ }
+ return true;
+ }
+
+ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS
+ bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
+ std::string& err) {
+ if (rows <= 0) return true;
+ const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
+ for (int c0 = 0; c0 < m.n; c0 += cols) {
+ const int nc = std::min(cols, m.n - c0);
+ if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
+ else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
+ const float one = 1.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
+ m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx: status " + std::to_string((int) st);
+ return false;
+ }
+ }
+ return true;
+ }
+ // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)
+ bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,
+ float* y, int ldy, std::string& err) {
+ const float zero = 0.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,
+ x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);
+ return false;
+ }
+ return true;
+ }
+ // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale
+ bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {
+ const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
+#if defined(STRATA_METAL)
+ if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass
+ hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);
+ return true;
+ }
+#endif
+ const float one = 1.0f, zero = 0.0f;
+ if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=
+ CUBLAS_STATUS_SUCCESS) {
+ err = "cublasSgemm (mHC mixes)";
+ return false;
+ }
+ hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);
+ return true;
+ }
+
+ // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full
+ // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file
+ // region read into pinned staging)
+ bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,
+ const uint8_t* src = nullptr) {
+ if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into
+ // pinned staging as its blob) or from the file's mapping
+ auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };
+ bool ok = true;
+ if (l.nm == 1) {
+ if (src != nullptr)
+ ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ else
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ } else {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,
+ (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,
+ (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+ const auto& parts = l.hh[(size_t) e];
+ const int D = g.dim, FF = g.ff;
+ auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {
+ return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {
+ return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ bool ok = true;
+ if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets
+ const uint8_t* p = src != nullptr ? src : parts[0];
+ if (l.nm == 1) {
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&
+ cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&
+ cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);
+ } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;
+ ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,
+ nt * tile, nth * tile, (size_t) D / 16) &&
+ cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&
+ cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];
+ ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,
+ (size_t) (fn / 16) * row) &&
+ cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&
+ cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);
+ }
+ } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,
+ (size_t) (k / 16));
+ ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ }
+ } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of
+ // both parts' columns
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)
+ for (int X = 0; X < 2; ++X) {
+ const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);
+ if (a >= b) continue;
+ ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,
+ (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&
+ cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,
+ (size_t) (b - a) * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);
+ }
+ const size_t tile = (size_t) 32 * l.lay.K[2];
+ const int nh = D / 2;
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,
+ (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));
+ ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+
+ // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk
+ // in layer order; the copies run at most kParts batches ahead of the compute
+ struct Batch {
+ int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range
+ };
+ struct Streamer {
+ std::vector<Batch> b;
+ size_t cp = 0, kp = 0; // the next batch to copy, to compute
+ std::atomic<size_t> done{0}; // cp, for read_ahead's thread
+ };
+ // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel
+ // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -
+ // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone
+ void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {
+ static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *
+ 1048576.0;
+ std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied
+ double ahead = 0;
+ for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {
+ const size_t done = S.done.load(std::memory_order_relaxed);
+ while (!issued.empty() && issued.front().first < done) {
+ ahead -= issued.front().second;
+ issued.pop_front();
+ }
+ if (ahead > cap) {
+ std::this_thread::sleep_for(std::chrono::microseconds(500));
+ continue;
+ }
+ const Batch& bt = S.b[q];
+ const Layer& l0 = L[(size_t) bt.il];
+ double nb = 0;
+ if (l0.dev == di)
+ for (int i = 0; i < bt.n; ++i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4) {
+ if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);
+ nb += (double) l0.blob;
+ continue;
+ }
+ os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);
+ nb += (double) l0.region_bytes[(size_t) e];
+ }
+ issued.push_back({q++, nb});
+ ahead += nb;
+ }
+ }
+ bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {
+ const int P = Chunk::kParts;
+ while (S.cp < S.b.size() && S.cp < S.kp + P) {
+ const Batch& bt = S.b[S.cp];
+ const int half = (int) (S.cp % P);
+ const Layer& l0 = L[(size_t) bt.il];
+ int mi = 0;
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di) mi = m;
+ if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));
+ uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots
+ if (c.stage != nullptr) {
+ if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)
+ sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;
+ std::atomic<bool> bad{false};
+ c.readers->run(bt.n, [&](int i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)
+ : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],
+ l0.region_bytes[(size_t) e],
+ sb + (size_t) i * c.stage_stride))
+ bad.store(true);
+ });
+ if (bad.load()) {
+ err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";
+ return false;
+ }
+ }
+ for (int i = 0; i < bt.n; ++i) {
+ uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;
+ if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,
+ sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))
+ return false;
+ c.stream_bytes += (double) unit(member(l0, mi));
+ }
+ CK(cudaEventRecord(c.filled[half], c.cs));
+ ++S.cp;
+ S.done.store(S.cp, std::memory_order_relaxed);
+ }
+ return true;
+ }
+
+ // ---- the flow (see Chunk)
+ // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute
+ // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing
+ static void kick(cudaStream_t s) {
+#if defined(_WIN32) && defined(STRATA_USE_HIP)
+ (void) cudaStreamQuery(s);
+ (void) cudaGetLastError();
+#else
+ (void) s;
+#endif
+ }
+ size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
+ return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
+ }
+ size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }
+ static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }
+ // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from
+ // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of
+ // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole
+ size_t mx_read_off(const Layer& l, int m, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : 0;
+ }
+ size_t mx_read_len(const Layer& l, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : mx_part_bytes(l, r);
+ }
+ // the device's member index in L0's group (0 alone)
+ int flow_mi(const Chunk& c, const Layer& L0) const {
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == c.dev) return m;
+ return 0;
+ }
+ void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {
+ c.items.emplace_back();
+ Chunk::Item& it = c.items.back();
+ it.il = l.il;
+ it.mi = mi;
+ it.k = k;
+ it.e = e;
+ it.v = v;
+ bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)
+ if (!in_ram && store && l.nm == 1) {
+ const int ss = store->find(l.il, e);
+ if (ss >= 0) {
+ store_blob = store->blob(ss);
+ in_ram = true;
+ }
+ }
+ if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)
+ const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];
+ for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);
+ ++c.it_ram;
+ return;
+ }
+ if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)
+ it.ram = store_blob != nullptr ? store_blob
+ : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];
+ bool hit = false;
+ if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after
+ const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;
+ if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again
+ c.lc_il = l.il; // once that layer's items are all read - `after` below)
+ c.lc_used = 0;
+ c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);
+ c.lc_first = i;
+ std::fill(c.lc_of.begin(), c.lc_of.end(), -1);
+ }
+ const int j = c.lc_of[(size_t) e];
+ if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)
+ it.ram = c.lc + (size_t) j * stride;
+ it.after = c.lc_writer[(size_t) j];
+ hit = true;
+ } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)
+ const int jn = c.lc_used++;
+ c.lc_of[(size_t) e] = jn;
+ c.lc_writer[(size_t) jn] = (int64_t) i;
+ it.keep = c.lc + (size_t) jn * stride;
+ it.after = (int64_t) c.lc_first - 1;
+ }
+ }
+ if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)
+ it.direct = true;
+ it.stage = -1;
+ it.left.store(3, std::memory_order_relaxed);
+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
+ return;
+ }
+ it.stage = (int) (c.n_file++ % (uint64_t) c.S);
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);
+ it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);
+ sb += flow_region(r);
+ }
+ it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);
+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
+ }
+ // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance
+ // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the
+ // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -
+ // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is
+ // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer
+ // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the
+ // rest of its picks are streamed in)
+ // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same
+ // profile and the same routing - and each reads only its part of every expert)
+ void flow_spec(Chunk& c, int k, int il, int T) {
+ static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;
+ Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ int* vs = member(l, mi).vs_pin;
+ c.spec_k = k;
+ c.spec_il = il;
+ c.spec_begin = c.base + c.items.size();
+ const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += pr[e];
+ std::vector<std::pair<double, int>> cand;
+ const double picks = (double) l.k_exp * T;
+ // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE
+ // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole
+ // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376
+ // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and
+ // 21 of 1568, the early reads 9-12% faster there)
+ static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;
+ const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;
+ for (int e : l.host_list) {
+ if (!early) break;
+ const double p = sum > 0 ? pr[e] / sum : 0.0;
+ const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));
+ if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {
+ return a.first > b.first;
+ });
+ if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));
+ std::vector<char> is((size_t) l.n_exp, 0);
+ int j = 0;
+ for (const auto& x : cand) {
+ vs[x.second] = l.n_slots + j++;
+ is[(size_t) x.second] = 1;
+ }
+ for (int e : l.host_list)
+ if (!is[(size_t) e]) vs[e] = l.n_slots + j++;
+ for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);
+ c.n_spec = (int) cand.size();
+ c.it_spec += cand.size();
+ }
+ // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones
+ void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {
+ const Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ const int* vs = member(l, mi).vs_pin;
+ for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {
+ const int v = fit(c, i).v;
+ if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;
+ }
+ for (int e : l.host_list) { // (index order: their virtual slots ascending)
+ const int v = vs[e];
+ if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
+ }
+ }
+ // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)
+ bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
+ const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
+ const size_t i = (size_t) (it - c.fl_layers.begin());
+ if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
+ if (k + 1 < (int) c.fl_T.size()) {
+ k2 = k + 1;
+ il2 = il;
+ return true;
+ }
+ if (i + 1 >= c.fl_layers.size()) return false;
+ k2 = 0;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (i + 1 < c.fl_layers.size()) {
+ k2 = k;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {
+ k2 = k + 1;
+ il2 = c.fl_layers[0];
+ return true;
+ }
+ return false;
+ }
+ // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each
+ // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now
+ bool flow_pump(Chunk& c, std::string& err) {
+ const size_t end = c.base + c.items.size();
+ IoQueue::Job jobs[6];
+ while (c.n_read < end) {
+ Chunk::Item& it = fit(c, c.n_read);
+ if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot
+ // written again once the layer before's items are all read (none still copies out of it)
+ bool wait = false;
+ if (it.keep == nullptr)
+ wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;
+ else {
+ while (c.lc_chk < c.n_read &&
+ (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))
+ ++c.lc_chk;
+ wait = c.lc_chk <= (size_t) it.after;
+ }
+ if (wait) break;
+ }
+ if (it.direct) {
+ const size_t R = (size_t) c.ring_n, r = c.n_read % R;
+ if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it
+ if (c.n_comp <= c.n_read - R) break;
+ CK(cudaEventSynchronize(c.ring_free[r]));
+ }
+ const Layer& l = L[(size_t) it.il];
+#if defined(STRATA_METAL)
+ uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)
+#else
+ uint8_t* dst = c.ring + r * c.ring_stride;
+#endif
+ const x3::ExpertLayout& lay = l.lay;
+ int nj = 0;
+ for (int q = 0; q < 3; ++q) {
+ IoQueue::Job& j = jobs[nj++];
+ j = IoQueue::Job{};
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];
+ j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];
+ j.len = mx_part_bytes(l, q);
+ j.dst = dst + lay.trellis[q];
+ j.exact = true;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;
+ j.map = pk.at(l.mx_file[q], j.off);
+ j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;
+ }
+ c.io->push(jobs, nj);
+ } else if (it.stage >= 0) {
+ const int64_t prev = c.stage_item[(size_t) it.stage];
+ if (prev >= 0) {
+ if ((size_t) prev >= c.n_dma) break;
+ const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);
+ if (q == cudaErrorNotReady) break;
+ CK(q);
+ }
+ c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)
+ int nj = 0;
+ for (int r = 0; r < 3; ++r) {
+ if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X) {
+ IoQueue::Job& j = jobs[nj++];
+ j = IoQueue::Job{};
+ j.len = nb;
+ j.dst = sb + (size_t) X * nb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ }
+ sb += flow_region(r);
+ continue;
+ }
+ IoQueue::Job& j = jobs[nj++];
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);
+ j.len = mx_read_len(l, r);
+ j.dst = sb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;
+ j.map = pk.at(l.mx_file[r], j.off);
+ j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;
+ sb += flow_region(r);
+ }
+ c.io->push(jobs, nj);
+ }
+ ++c.n_read;
+ }
+ const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;
+ struct Kick { // (every way out: what was queued is submitted)
+ cudaStream_t s;
+ const size_t &a, b;
+ ~Kick() {
+ if (a != b) kick(s);
+ }
+ } kq{c.cs, c.n_dma, dma0};
+ while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {
+ Chunk::Item& it = fit(c, c.n_dma);
+ if (it.left.load(std::memory_order_acquire) != 0) break;
+ if (it.bad.load(std::memory_order_acquire)) {
+ err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +
+ " from the model's files failed";
+ return false;
+ }
+ const size_t r = c.n_dma % R;
+ if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* dst = c.ring + r * c.ring_stride;
+ const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;
+ if (it.direct) { // (already in the slot)
+ CK(cudaEventRecord(c.ring_fill[r], c.cs));
+ c.stream_bytes += (double) unit(l);
+ ++c.n_dma;
+ continue;
+ }
+ for (int p = 0; p < 2; ++p)
+ CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));
+ if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)
+ const int t2 = gg::expert_type(l.lay, 2);
+ const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);
+ CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,
+ cudaMemcpyHostToDevice, c.cs));
+ } else
+ CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));
+ CK(cudaEventRecord(c.ring_fill[r], c.cs));
+ if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));
+ c.stream_bytes += (double) unit(l);
+ ++c.n_dma;
+ }
+ return true;
+ }
+ // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM
+ // ones in shares between them (the device works while the reads run)
+ bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {
+ cudaStream_t s = d.st;
+ const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)
+ if (c.spec_k != k || c.spec_il != L0.il) {
+ err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";
+ return false;
+ }
+ flow_rest(c, k, L0.il, b);
+ const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
+ int k2 = 0, il2 = 0;
+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);
+ const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
+ int v_done = 0;
+ while (c.n_comp < le) {
+ // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what
+ // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so
+ // the sums come out the same on every run whatever the reads' timing
+ const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});
+ if (!flow_pump(c, err)) return false;
+ if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy
+ const double tw = now_ms();
+ kick(s);
+ if (c.n_read > c.n_dma) c.io->wait_any(500);
+ else {
+ const Chunk::Item& it = fit(c, c.n_read);
+ if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));
+ }
+ c.wait_ms += now_ms() - tw;
+ continue;
+ }
+ const int n = (int) (i1 - i0);
+ int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ ++c.n_batch;
+ // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)
+ for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];
+ bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];
+ if (bc[n] > bc[0]) {
+ CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));
+ CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));
+ if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))
+ return false;
+ }
+ for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));
+ kick(s);
+ c.n_comp = i1;
+ if (!flow_pump(c, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)
+ c.items.pop_front();
+ ++c.base;
+ }
+ return true;
+ }
+
+ // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)
+ // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order
+ bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,
+ const int* b, int i0, int i1, int T, std::string& err) {
+ const int D = g.dim, FF = l.ffn;
+ while (i0 < i1) {
+ int j = i0 + 1;
+ while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;
+ const int a = b[i0], nrows = b[j] - a;
+ int max_rows = 0;
+ for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);
+ if (nrows > 0) {
+ x3::SetArgs sa;
+ sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)
+ sa.stride = stride;
+ sa.lay = l.nm > 1 ? l.hlay : l.lay;
+ sa.n_exp = j - i0;
+ sa.bounds = d_bounds + i0;
+ sa.row0 = a;
+ sa.max_rows = max_rows;
+ sa.hbounds = b + i0;
+ sa.total_rows = nrows;
+ sa.n_embd = D;
+ sa.n_ff = FF;
+ sa.limit = g.lim;
+ if (sa.lay.fmt == gg::kFmtMXFP4) {
+ gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ gg::set_down(sa, c.h16, FF, c.y, D, d.st);
+ } else {
+ x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ x3::set_down(sa, c.h16, FF, c.y, D, d.st);
+ }
+ if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {
+ std::vector<int> bb(b + i0, b + j + 1);
+ std::vector<uint16_t> hv((size_t) nrows * FF);
+ std::vector<float> yv((size_t) nrows * D);
+ CK(cudaStreamSynchronize(d.st));
+ CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&
+ wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};
+ std::fwrite(hdr, 8, 6, dump_y);
+ std::fwrite(bb.data(), 4, bb.size(), dump_y);
+ std::fwrite(hv.data(), 2, hv.size(), dump_y);
+ std::fwrite(yv.data(), 4, yv.size(), dump_y);
+ std::fflush(dump_y);
+ }
+ moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);
+ }
+ i0 = j;
+ }
+ (void) err;
+ return true;
+ }
+
+ bool load_dense(int di, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ size_t* acct = &d.weights;
+ for (Layer& l0 : L)
+ for (int mi = 0; mi < l0.nm; ++mi) {
+ Layer& l = member(l0, mi);
+ if (l.dev != di) continue;
+ // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,
+ // of the shared expert's FFN; the rest whole)
+ const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);
+ const std::string& P = l.name;
+ const std::string A = P + "attn.";
+ bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&
+ mat(d, A + "wkv", l.wkv, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&
+ f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&
+ f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&
+ f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&
+ f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&
+ f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&
+ f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&
+ f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&
+ f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&
+ f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&
+ (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||
+ f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&
+ mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);
+ if (!ok) return false;
+ l.sink += (size_t) l.ah0;
+ for (int gi = 0; gi < l.ogn; ++gi)
+ if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;
+ if (l.kv_src) {
+ const std::string C = A + "compressor.";
+ if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;
+ if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;
+ if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ if (l.ix_src) {
+ const std::string I = A + "indexer.";
+ if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))
+ return false;
+ if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))
+ return false;
+ }
+ if (l.eg >= 0) {
+ if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;
+ if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already
+ if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;
+ } else {
+ float *qw = nullptr, *kw = nullptr;
+ if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))
+ return false;
+ std::vector<float> a((size_t) g.hc * g.dim), b(a.size());
+ CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));
+ for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product
+ CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));
+ l.eg_w = qw;
+ }
+ }
+ if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;
+ if (g.n_mtp > 0 && !l.mtp) {
+ if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;
+ if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))
+ return false;
+ }
+ if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns
+ const int hn = head_nm();
+ if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))
+ return false;
+ if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))
+ return false;
+ if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;
+ if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head
+ const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");
+ const int R = g.d_rank;
+ head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)
+ // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same
+ // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)
+ if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {
+ std::vector<uint16_t> rows((size_t) d.mk_vn * R);
+ if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {
+ err = "the markov head: read failed";
+ return false;
+ }
+ if (th->dtype == "BF16")
+ for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));
+ if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||
+ !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))
+ return false;
+ CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));
+ }
+ }
+ }
+ if (di == head_dev) {
+ head = d.head;
+ norm_w = d.norm_w;
+ d_norm = d.d_norm;
+ if (g.n_mtp > 0) {
+ const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";
+ if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))
+ return false;
+ const Tensor* te = pk.get(Z + "markov_head.embed.weight");
+ const Tensor* th = pk.get(Z + "markov_head.head.weight");
+ if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {
+ err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";
+ return false;
+ }
+ if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
+ !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))
+ return false;
+ }
+ }
+ if (g.n_mtp > 0 && di == d_dev) {
+ if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||
+ !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))
+ return false;
+ }
+ return true;
+ }
+
+#if defined(STRATA_METAL)
+ // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays
+ // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and
+ // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most
+ // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.
+ double metal_expert_cap() const {
+ uint64_t phys = 0;
+ size_t len = sizeof phys;
+ sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);
+ unsigned long long pages = 0;
+ size_t pl = sizeof pages;
+ double floor_b = 0;
+ if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;
+ size_t fr = 0, tot = 0;
+ cudaMemGetInfo(&fr, &tot);
+ const double gb = 1e9, P = (double) phys;
+ double keep = std::max(7.0 * gb, 0.15 * P);
+ if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;
+ double share = 0.40;
+ if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);
+ double leave = keep;
+ if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});
+ const double cap = std::min((double) tot, P - leave);
+ task_vm_info_data_t ti;
+ mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;
+ double foot = 0;
+ if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;
+ return std::max(4.0 * gb, cap - foot - 1.0 * gb);
+ }
+#endif
+#if defined(STRATA_METAL)
+ // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it
+ // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).
+ bool zc_try(Layer& l, int e) {
+ static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)
+ if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;
+ if (l.vm[(size_t) e][0].raw != nullptr) return true;
+ uint64_t a[3];
+ for (int r = 0; r < 3; ++r) {
+ a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));
+ if (a[r] == 0) return false;
+ }
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m = x3::Mat{};
+ m.K = gg::expert_type(l.lay, r);
+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
+ m.raw = (const void*) a[r];
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ }
+ return true;
+ }
+#endif
+ void set_vm(Layer& l, int e, int slot) {
+ uint8_t* b = l.vram_base + (size_t) slot * l.blob;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m = x3::Mat{};
+ m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)
+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
+ m.raw = b + l.lay.trellis[r];
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ }
+ return;
+ }
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);
+ m.suh = (const uint16_t*) (b + l.lay.suh[r]);
+ m.svh = (const uint16_t*) (b + l.lay.svh[r]);
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ m.K = l.lay.K[r];
+ m.cb = 2;
+ m.ld = 0;
+ }
+ }
+ // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)
+ void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = blob + l.lay.trellis[r];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+ void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ uint8_t* tr = blob + l.lay.trellis[r];
+ for (int X = 0; X < NH; ++X)
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);
+ std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],
+ (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back
+ void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,
+ blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);
+ std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);
+ std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);
+ }
+ void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,
+ src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);
+ if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);
+ std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);
+ if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);
+ }
+
+ bool place_experts(std::string& err) {
+ // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)
+ double room = 1e300, kept = 0;
+ for (const Dev& d : devs)
+ if (commit_charged(d)) room = commit_room();
+ const double room0 = room;
+ // per group: what its tightest member has left (each member holds its part of every VRAM expert of the
+ // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round
+ // the members layer by layer, so each member's total comes to about the mean's)
+ auto ucount = [](const Layer& l) {
+ double s = 0;
+ for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));
+ return s / l.nm;
+ };
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ double budget = 1e300;
+ bool charged = false;
+ for (int m : groups[gi].mem) {
+ cudaSetDevice(devs[(size_t) m].id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *
+ o.vram_experts);
+#if defined(STRATA_METAL)
+ if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {
+ const double cap = metal_expert_cap();
+ if (cap < budget) {
+ std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",
+ cap / 1e9, cap / 1e9);
+ budget = cap;
+ }
+ }
+#endif
+ charged = charged || commit_charged(devs[(size_t) m]);
+ }
+ if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)
+ double all = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;
+ const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);
+ kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;
+ budget = capped;
+ room = std::max(0.0, room - std::min(budget, all) * nm);
+ }
+ double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)
+ for (Layer& l : L)
+ if (l.grp == (int) gi && l.mtp) {
+ l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));
+ budget_t -= (double) l.n_vram * ucount(l);
+ }
+ double want = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;
+ const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
+ for (Layer& l : L) {
+ if (l.grp != (int) gi || l.mtp) continue;
+ if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)
+ l.n_vram = 0;
+ continue;
+ }
+ l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
+ // never more than fits: the floor of each layer's share can still round up the total
+ while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
+ }
+ double used = 0;
+ for (Layer& l : L)
+ if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
+ // what the floors left: whole experts, layer by layer
+ for (Layer& l : L) {
+ if (l.grp != (int) gi || l.cpu_only) continue;
+ while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
+ ++l.n_vram;
+ used += (double) ucount(l);
+ }
+ }
+ }
+ if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it
+ const os::Commit c = os::commit();
+ const double gib = (double) (1ull << 30);
+ const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;
+ commit_kept = kept;
+ pagefile_hint = std::ceil((pf + kept) / gib);
+ std::fprintf(stderr,
+ "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "
+ "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "
+ "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "
+ "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",
+ kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);
+ } else if (room0 < 1e300 && o.verbose) {
+ std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "
+ "programs)\n", room0 / 1e9, commit_margin() / 1e9);
+ }
+ for (Layer& l : L) {
+ // the hottest experts of the profile first, then by index
+ std::vector<int> order((size_t) l.n_exp);
+ std::iota(order.begin(), order.end(), 0);
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });
+ l.slot.assign((size_t) l.n_exp, -1);
+ l.host.assign((size_t) l.n_exp, nullptr);
+ // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps
+ l.n_slots = std::min(l.n_vram, l.n_exp);
+ if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)
+ l.n_vram = l.n_slots - 1;
+ l.spare_vram = l.n_slots - 1;
+ } else {
+ l.n_vram = l.n_slots;
+ l.spare_vram = -1;
+ }
+ for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;
+ for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {
+ Layer& x = member(l, m);
+ Dev& d = devs[(size_t) x.dev];
+ cudaSetDevice(d.id);
+ void* v = nullptr;
+ if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {
+ err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +
+ " experts on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ d.experts += (size_t) l.n_slots * unit(x);
+ x.vram_base = (uint8_t*) v;
+ }
+ l.heat.assign((size_t) l.n_exp, 0.0f);
+ { // the profile as the starting heat: its counts scaled to ~512 hits a layer
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ const double sc = sum > 0 ? 512.0 / sum : 0.0;
+ for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);
+ }
+ const int n_host = l.n_exp - l.n_vram;
+ { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;
+ l.hl.k[r] = k;
+ l.hl.n[r] = n;
+ l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;
+ off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);
+ l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;
+ off += align256((size_t) k * 2);
+ l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;
+ off += align256((size_t) n * 2);
+ }
+ l.hl.bytes = off;
+ }
+ l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});
+ if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);
+ } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (pk.map(l.region_file[(size_t) e]) == nullptr) {
+ err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";
+ return false;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) {
+ l.hh[(size_t) e][0] = file_part(l, e);
+ l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ } else if (n_host > 0) {
+ const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)
+ const size_t bytes = (size_t) n_parts * l.hl.bytes;
+ for (int X = 0; X < NH; ++X) {
+ void* m = node_map(bytes, X, NH);
+ if (m == nullptr) {
+ err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +
+ " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";
+ return false;
+ }
+ host_maps.emplace_back(m, bytes);
+ int hi = 0;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;
+ if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ l.vm.assign((size_t) l.n_exp, {});
+ if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);
+ if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0)
+ tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));
+ }
+ }
+ return true;
+ }
+
+ // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the
+ // file: the mapping reads them on first use)
+ bool load_experts_mx(std::string& err) {
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ size_t maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12; ++t)
+ th.emplace_back([&] {
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);
+ for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,
+ unit(l), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!ok) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the
+ // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them
+ struct Warm {
+ int layer, e;
+ double share;
+ };
+ std::vector<Warm> warm;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0) {
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);
+ } else {
+ warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});
+ }
+ }
+ }
+ std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });
+ if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left
+ std::vector<std::pair<int, int>> hot;
+ for (const Warm& w : warm) hot.push_back({w.layer, w.e});
+ if (!copy_experts(hot, err)) return false;
+ if (copy_mode) {
+ warm.erase(std::remove_if(warm.begin(), warm.end(),
+ [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),
+ warm.end());
+ const uint64_t av = ram_avail();
+ if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;
+ }
+ }
+ if (ram_tier && warm_bytes > 0 && !warm.empty()) {
+ // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for
+ // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not
+ // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on
+ // Windows - the engine's own pinned buffers come after this
+ const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));
+ while (ram_slots.size() < want + 1) {
+ const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());
+ uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);
+ if (c == nullptr) break;
+ ram_chunks.push_back(c);
+ ram_chunk_bytes.push_back(n * maxb);
+ for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);
+ }
+ ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;
+ for (Layer& l : L)
+ if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);
+ std::atomic<size_t> ri{0};
+ std::atomic<bool> rfail{false};
+ std::vector<std::thread> rt;
+ for (int t = 0; t < 12 && ram_held > 0; ++t)
+ rt.emplace_back([&] {
+ for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {
+ const Layer& l = L[(size_t) warm[i].layer];
+ if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ }
+ });
+ for (auto& t : rt) t.join();
+ if (rfail.load()) {
+ err = "reading the RAM tier's experts from the model's files failed";
+ return false;
+ }
+ for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;
+ if (ram_held > 0) ram_free.push_back(ram_held);
+ if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "
+ "read from the model's files (%.1f s)\n",
+ ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ size_t nw = 0;
+ for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)
+ b += (double) L[(size_t) warm[nw].layer].blob;
+ std::atomic<size_t> wi{0};
+ std::vector<std::thread> wt;
+ for (int t = 0; t < 12 && nw > 0; ++t)
+ wt.emplace_back([&] {
+ std::vector<uint8_t> buf(maxb);
+ for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());
+ });
+ for (auto& t : wt) t.join();
+ if (o.verbose && nw > 0)
+ std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,
+ nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
+ return true;
+ }
+
+ // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU
+ // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the
+ // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:
+ // none - read in place: the page cache follows a conversation's experts where a static set (the profile's
+ // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the
+ // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file
+ // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what
+ // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place
+ bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {
+ if (o.mmap_experts > 0 || hot.empty()) return true;
+ const double t0 = now_ms();
+ auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };
+ double all = 0;
+ {
+ std::vector<char> seen(L.size(), 0);
+ for (const auto& h : hot) {
+ const Layer& l = L[(size_t) h.first];
+ all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));
+ }
+ }
+ const uint64_t avail = ram_avail();
+ // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),
+ commit_room());
+ // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
+ // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
+ // on 64 GB and on 32 GB of RAM)
+ double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;
+ if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;
+ if (o.mmap_experts == 0 && all > can)
+ std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "
+ "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "
+ "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
+ if (budget <= 0) {
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "
+ "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",
+ all / 1e9, can / 1e9, kRamMargin / 1e9);
+ return true;
+ }
+ // NUMA parts: whole rows of each matrix, so any row count the nodes divide
+ int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));
+ if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)
+ nh = std::atoi(v) >= 2 ? 2 : 1;
+ if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;
+ NH = nh;
+ cp_lay.assign(L.size(), {0, 0, 0, 0});
+ for (const Layer& l : L) {
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ cp_lay[(size_t) l.il][(size_t) r] = off;
+ off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));
+ }
+ cp_lay[(size_t) l.il][3] = off;
+ }
+ // the hottest that fit the budget (a layer's first one brings its spare)
+ std::vector<int> n_copy(L.size(), 0);
+ double used = 0;
+ size_t take = 0;
+ for (; take < hot.size(); ++take) {
+ const Layer& l = L[(size_t) hot[take].first];
+ const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));
+ if (used + add > budget) break;
+ used += add;
+ ++n_copy[(size_t) l.il];
+ }
+ // each layer's slots: one block per node of (copies + spare) parts
+ std::vector<int> first(L.size(), -1);
+ copy_spare.assign(L.size(), -1);
+ for (Layer& l : L) {
+ l.ram.assign((size_t) l.n_exp, -1);
+ const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);
+ if (ns == 0) continue;
+ const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
+ std::array<uint8_t*, 2> base = {nullptr, nullptr};
+ bool ok = true;
+ for (int X = 0; X < NH && ok; ++X) {
+ base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
+ ok = base[(size_t) X] != nullptr;
+ if (ok) {
+ ram_chunks.push_back(base[(size_t) X]);
+ ram_chunk_bytes.push_back(bytes);
+ }
+ }
+ if (!ok) {
+ std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,
+ ns, (double) bytes * NH / 1e9);
+ n_copy[(size_t) l.il] = 0;
+ continue;
+ }
+ first[(size_t) l.il] = (int) ram_part.size();
+ for (int i = 0; i < ns; ++i)
+ ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});
+ if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;
+ }
+ struct Job {
+ int layer, e, slot;
+ };
+ std::vector<Job> jobs;
+ {
+ std::vector<int> next = first;
+ for (size_t i = 0; i < take; ++i) {
+ const int il = hot[i].first;
+ if (n_copy[(size_t) il] == 0) continue;
+ jobs.push_back({il, hot[i].second, next[(size_t) il]++});
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> ji{0};
+ std::atomic<bool> bad{false};
+ std::atomic<int64_t> done_b{0};
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12 && !jobs.empty(); ++t)
+ th.emplace_back([&] {
+ for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {
+ const Job& j = jobs[i];
+ const Layer& l = L[(size_t) j.layer];
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,
+ ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))
+ bad.store(true);
+ }
+ const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && i % 2048 == 0)
+ std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ });
+ for (auto& t : th) t.join();
+ if (bad.load()) {
+ err = "copying the CPU lane's experts from the model's files failed";
+ return false;
+ }
+ for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;
+ copy_mode = true;
+ ram_held = (int) jobs.size();
+ double cb = 0;
+ for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;
+ std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",
+ jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,
+ jobs.size() < hot.size() ? "; the others read in place" : "");
+ return true;
+ }
+
+ bool load_experts(std::string& err) {
+ if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);
+ // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place
+ // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their
+ // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are
+ // dropped from it again
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ for (const Layer& l : L)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ uint64_t warm = 0;
+ if (file_experts) {
+ std::vector<std::pair<double, Job>> cand;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ for (const auto& c : cand) {
+ const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];
+ if (warm + b > warm_bytes) break;
+ warm += b;
+ jobs.push_back(c.second);
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])
+ return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];
+ return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ const int nth = 12;
+ size_t maxr = 0, maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);
+ }
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < nth; ++t)
+ th.emplace_back([&] {
+ std::vector<uint8_t> region(maxr);
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ int cur_dev = -1;
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],
+ region.data())) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ if (l.slot[(size_t) e] >= 0) {
+ repack(l, region.data(), stage);
+ bool up = true;
+ for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ const Dev& d = devs[(size_t) x.dev];
+ if (cur_dev != d.id) {
+ cudaSetDevice(d.id);
+ cur_dev = d.id;
+ }
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,
+ unit(x), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!up) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);
+ failed.store(true);
+ break;
+ }
+ if (file_experts)
+ os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);
+ } else if (!file_experts) {
+ for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +
+ (int64_t) l.region_bytes[(size_t) e];
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),
+ db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ if (file_experts)
+ std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",
+ (now_ms() - t0) / 1000.0, warm / 1e9);
+ return true;
+ }
+
+ std::string describe() const {
+ std::ostringstream s;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ const Dev& d = devs[di];
+ int a = -1, b = -1, ev = 0, et = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ if (a < 0) a = l.il;
+ b = l.il;
+ ev += l.n_vram;
+ et += l.n_exp;
+ }
+ size_t fr = 0, tot = 0;
+ cudaSetDevice(d.id);
+ cudaMemGetInfo(&fr, &tot);
+ const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();
+ char tp[64] = "";
+ if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);
+ char line[384];
+ std::snprintf(line, sizeof line,
+ "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "
+ "%.2f GB, %.0f MiB free\n",
+ d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,
+ d.caches / 1e9, fr / 1048576.0);
+ s << line;
+ }
+ int nh = 0;
+ size_t hb_ = 0;
+ for (const Layer& l : L) {
+ nh += l.n_exp - l.n_vram;
+ hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;
+ }
+ if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)
+ uint64_t h = 1469598103934665603ull;
+ for (const Layer& l : L)
+ for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }
+ char hl[96];
+ std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);
+ s << hl;
+ }
+ char line[200];
+ if (copy_mode)
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "
+ "parts), the rest in place in the model's files; %d threads\n",
+ nh, hb_ / 1e9, ram_held, NH, cpu_threads());
+ else if (file_experts)
+ std::snprintf(line, sizeof line,
+ "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",
+ nh, hb_ / 1e9, cpu_threads());
+ else
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,
+ hb_ / 1e9, NH, cpu_threads());
+ s << line;
+ return s.str();
+ }
+
+ double store_gb() const {
+ static double cached = -2.0; // (decided once: the layout of the blobs depends on it)
+ if (cached > -1.5) return cached;
+ double gb = o.ssd_cache_gb;
+ if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);
+ if (!pk.gguf) gb = 0.0;
+ else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most
+ const uint64_t avail = ram_avail();
+ const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;
+ gb = room >= 4.0 ? std::min(48.0, room) : 0.0;
+ }
+ return cached = gb;
+ }
+ void init_store() {
+ const double gb = store_gb();
+ if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;
+ std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ ExpertStore::LayerGeo& gl = geo[(size_t) l.il];
+ gl.n_exp = l.n_exp;
+ for (int r = 0; r < 3; ++r) {
+ gl.file[r] = l.mx_file[r];
+ gl.off[r] = l.mx_off[r];
+ gl.stride[r] = l.mx_stride[r];
+ gl.bytes[r] = mx_part_bytes(l, r);
+ gl.tr[r] = l.lay.trellis[r];
+ }
+ gl.blob = l.blob;
+ }
+ ExpertStore::Cfg cfg;
+ cfg.budget_bytes = (size_t) (gb * 1e9);
+ cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
+ cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
+ cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
+ cfg.ram_avail = [] { return ram_avail(); };
+ cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
+ cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;
+ // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a
+ // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane
+ // took which expert, the CPU lane's are the mmap path's byte for byte)
+ const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&
+ L[0].nm == 1;
+ if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)
+ const int dev0 = devs[0].id;
+ cfg.map_chunk = [dev0](void* h, size_t n) -> void* {
+ cudaSetDevice(dev0);
+ if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return nullptr;
+ }
+ void* d = nullptr;
+ if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {
+ (void) cudaGetLastError();
+ cudaHostUnregister(h);
+ return nullptr;
+ }
+ if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);
+ return d;
+ };
+ cfg.unmap_chunk = [dev0](void* h) {
+ cudaSetDevice(dev0);
+ cudaHostUnregister(h);
+ };
+ }
+ bool direct = true;
+ for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
+ auto st = std::make_unique<ExpertStore>();
+ if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {
+ std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
+ return;
+ }
+ store = std::move(st);
+ for (Layer& l : L)
+ if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);
+ std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
+ store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
+ (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
+ if (want_zc) {
+ if (!store->gpu_mapped()) {
+ std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;
+ cudaSetDevice(devs[0].id);
+ if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {
+ (void) cudaGetLastError();
+ zc_stage_buf = nullptr;
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ for (Layer& l : L)
+ if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);
+ else if (!l.zc.empty()) {
+ (void) cudaGetLastError();
+ l.zc.clear();
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",
+ zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
+ }
+ }
+ // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)
+ void preload_store() {
+ if (!store) return;
+ if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;
+ std::vector<std::pair<double, std::pair<int, int>>> cand;
+ double tot = 0;
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) tot += p[e];
+ }
+ if (tot <= 0) return; // (no profile yet: the first tokens fill it)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ std::vector<std::pair<int, int>> list;
+ list.reserve(cand.size());
+ for (const auto& c : cand) list.push_back(c.second);
+ const double t0 = now_ms();
+ const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;
+ const size_t n = store->preload(list, leave);
+ std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,
+ (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
+ }
+ // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
+ // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
+ void store_sync() {
+ if (!store || !store->gpu_mapped()) return;
+ std::vector<ExpertStore::Vis> ev;
+ store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });
+ if (ev.empty()) return;
+ std::vector<char> dirty(L.size(), 0);
+ for (const auto& v : ev) {
+ Layer& l = L[(size_t) v.layer];
+ if (l.zc.empty()) continue;
+ l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;
+ dirty[(size_t) v.layer] = 1;
+ }
+ for (Layer& l : L) {
+ if (!dirty[(size_t) l.il]) continue;
+ if (l.zc_tab == nullptr) continue;
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.zc[(size_t) e]) {
+ const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;
+ tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;
+ if (s < 0) l.zc[(size_t) e] = 0;
+ }
+ cudaSetDevice(devs[0].id);
+ cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);
+ }
+ }
+
+ int cpu_threads() const {
+ int n = 0;
+ for (const auto& pl : pools) n += pl->size() - 1;
+ return n + 1;
+ }
+ // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over
+ // every node's cores (the pages lie where the cache put them)
+ void make_pools() {
+ std::vector<std::vector<int>> cs = cores;
+ if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)
+ std::vector<int> all = cs[0];
+ cs.assign((size_t) NH, {});
+ for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);
+ }
+ for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {
+ std::vector<int> c = cs[(size_t) X];
+ if (file_experts && NH == 1)
+ for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());
+ if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)
+ int nw = (int) c.size();
+ if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));
+ pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)
+ }
+ (void) gg::mx_tier(); // (measured here, not on the first token)
+ if (o.verbose) {
+ int nthr = 0;
+ for (auto& p : pools) nthr += p->size();
+ std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),
+ gg::mx_tier_report());
+ }
+ }
+
+ void reset() {
+ pos = 0;
+ toks.clear();
+ ctoks.clear();
+ ck[0].pos = ck[1].pos = -1;
+ for (Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ Layer& l = member(l0, m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);
+ if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
+ // state is indexed by position, and what lies past a restored position is written again)
+ void checkpoint(int slot) {
+ Check& c = ck[slot & 1];
+ c.pos = pos;
+ c.win.resize(L.size());
+ c.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ c.win[i].resize((size_t) g.win * win_rb());
+ cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);
+ if (l.c_kv) {
+ c.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyDeviceToHost, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
+ int best_check(int64_t limit) const {
+ int b = -1;
+ for (int i = 0; i < 2; ++i)
+ if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
+ return b;
+ }
+ int64_t restorable(int64_t limit) const {
+ const int b = best_check(limit);
+ return b < 0 ? -1 : ck[b].pos;
+ }
+
+ bool restore(int64_t limit) {
+ const int b = best_check(limit);
+ if (b < 0) return false;
+ const Check& c = ck[b];
+ for (size_t i = 0; i < L.size(); ++i)
+ for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)
+ Layer& l = member(L[i], m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);
+ if (l.c_kv) {
+ cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyHostToDevice, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ pos = c.pos;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
+ if (o2.pos > pos) o2.pos = -1;
+ return true;
+ }
+
+ size_t bounce_bytes() const {
+ return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+ size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }
+ size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }
+ size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }
+ static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need
+
+ // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the
+ // host; a prompt's pictures as their spans' rows (set_images)
+ VisionTower vt;
+ VisionGeo vg;
+ int img_tok = -1;
+ std::string vision_err; // why the tower did not load (pictures off), or empty
+ std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]
+ std::vector<Image> imgs; // the pictures of the prompt being fed, by start
+ // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst
+ bool read_bf16(const Tensor& t, size_t n, void* dst) {
+ if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);
+ if (t.dtype != "F32" || t.bytes != n * 4) return false;
+ std::vector<float> f(n);
+ if (!pk.read(t, f.data())) return false;
+ uint16_t* o = (uint16_t*) dst;
+ for (size_t i = 0; i < n; ++i) {
+ uint32_t u;
+ std::memcpy(&u, &f[i], 4);
+ o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);
+ }
+ return true;
+ }
+ bool load_vision(std::string& err) {
+ vg.layers = (int) pk.num("vision.num_hidden_layers", 0);
+ const int tok = (int) pk.num("image_token_id", -1);
+ if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none
+ vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);
+ vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);
+ vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);
+ vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;
+ const char* mark[3] = {"image_start", "image_newline", "image_end"};
+ for (int i = 0; i < 3; ++i) {
+ const Tensor* t = pk.get(mark[i]);
+ img_mark[i].resize((size_t) g.dim);
+ if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {
+ err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";
+ return false;
+ }
+ }
+ const double t0 = now_ms();
+ Dev& d = devs[0];
+ // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which
+ // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)
+ auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {
+ const Tensor* t = pk.get(n);
+ if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {
+ const std::string pre = n.substr(0, n.size() - 13);
+ const Tensor* tg = pk.get(pre + "mlp.gate.weight");
+ const Tensor* tu = pk.get(pre + "mlp.up.weight");
+ if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;
+ shape = {tg->shape[0] * 2, tg->shape[1]};
+ const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];
+ return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));
+ }
+ if (t == nullptr) return false;
+ shape = t->shape;
+ size_t n_el = 1;
+ for (int64_t x : t->shape) n_el *= (size_t) x;
+ return dst == nullptr || read_bf16(*t, n_el, dst);
+ };
+ if (!vt.load(vg, d.id, read, err)) return false;
+ img_tok = tok;
+ CK(cudaSetDevice(d.id));
+ CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",
+ vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {
+ if (img_tok < 0) {
+ err = "this model has no vision tower (or it is off)";
+ return false;
+ }
+ std::vector<uint16_t> rows;
+ int nh = 0, nw = 0;
+ if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;
+ const size_t D = (size_t) g.dim;
+ span.resize(((size_t) nh * (nw + 1) + 2) * D);
+ uint16_t* o = span.data();
+ auto put = [&](const uint16_t* r) {
+ std::memcpy(o, r, D * 2);
+ o += D;
+ };
+ put(img_mark[0].data());
+ for (int i = 0; i < nh; ++i) {
+ for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);
+ put(img_mark[1].data());
+ }
+ put(img_mark[2].data());
+ return true;
+ }
+ // the row of a picture's span at position p (null: p is in none)
+ const uint16_t* image_row(int64_t p) const {
+ if (imgs.empty()) return nullptr;
+ auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });
+ if (it == imgs.begin()) return nullptr;
+ const Image& im = *(it - 1);
+ const int64_t r = p - im.start;
+ if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;
+ return im.rows->data() + (size_t) r * g.dim;
+ }
+ // token tk's embedding row at position p: inside a picture's span, the picture's
+ const uint16_t* emb_row(int64_t p, int tk) const {
+ const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;
+ return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;
+ }
+ bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }
+ // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)
+ int32_t ctok(int64_t p, int tk) const {
+ if (tk == img_tok && image_row(p) != nullptr) return -1;
+ return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;
+ }
+ // a chunk's image rows (route()'s bits); false: none
+ bool vl_bits(int64_t p0, int T, VlBits& b) const {
+ if (imgs.empty()) return false;
+ std::memset(&b, 0, sizeof b);
+ bool any = false;
+ for (int t = 0; t < T && t < kVlRows; ++t)
+ if (in_image(p0 + t)) {
+ b.w[t >> 5] |= 1u << (t & 31);
+ any = true;
+ }
+ return any;
+ }
+
+ // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]
+ // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a
+ // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,
+ // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.
+ void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;
+ const size_t per = (size_t) cols * (hd + hd / 32);
+ b.clear();
+ b.reads.reserve((size_t) T * cols);
+ const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
+ const Tensor* tw = pk.get(P + "weight");
+ const Tensor* ts = pk.get(P + "scale");
+ const int fmt = eg_fmt[(size_t) li];
+ const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)
+ const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)
+ std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
+ int64_t hits = 0;
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (int t = 0; t < T; ++t) {
+ const int64_t p = p0 + t;
+ uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;
+ uint8_t* scales = rows + (size_t) cols * hd;
+ if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)
+ std::memset(rows, 0, per);
+ continue;
+ }
+ int64_t prod[8];
+ bool blocked = false;
+ for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)
+ blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;
+ const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];
+ prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];
+ }
+ int64_t rolling = prod[0];
+ for (int n = 1; n < pk.eg_ngram; ++n) {
+ rolling ^= prod[n];
+ for (int h = 0; h < pk.eg_heads; ++h) {
+ const int c = (n - 1) * pk.eg_heads + h;
+ const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];
+ uint8_t* rd = rows + (size_t) c * hd;
+ uint8_t* sd = scales + (size_t) c * (hd / 32);
+ const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;
+ if (const uint8_t* hit = eg_cache.find(key)) {
+ std::memcpy(rd, hit, (size_t) hd);
+ std::memcpy(sd, hit + hd, (size_t) hd / 32);
+ ++hits;
+ continue;
+ }
+ const auto fi = first.find(key);
+ if (fi != first.end()) {
+ b.dups.push_back({fi->second, {rd, sd}});
+ continue;
+ }
+ first.emplace(key, (int) b.reads.size());
+ EgRead r;
+ r.file = tw->file;
+ r.dst = rd;
+ r.scales = sd;
+ r.key = key;
+ if (mx8) {
+ r.mx8 = true;
+ r.n = (uint32_t) (hd / 32 * 33);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)
+ r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else {
+ r.n = (uint32_t) hd;
+ r.off = tw->off + (uint64_t) id * hd;
+ r.file2 = ts->file;
+ r.n2 = (uint32_t) (hd / 32);
+ r.off2 = ts->off + (uint64_t) id * (hd / 32);
+ }
+ b.reads.push_back(r);
+ }
+ }
+ }
+ }
+ eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();
+ eg_hits_n += hits + (int64_t) b.dups.size();
+ eg_reads_n += (int64_t) b.reads.size();
+ eg_io.submit(b);
+ }
+ // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)
+ void eg_finish(EgBatch& b) {
+ if (b.post.load(std::memory_order_acquire) == 2) return;
+ const int64_t t0 = now_ns();
+ eg_io.wait(b);
+ int want = 0;
+ if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {
+ const int hd = pk.eg_hd;
+ for (const auto& d : b.dups) {
+ const EgRead& r = b.reads[(size_t) d.first];
+ std::memcpy(d.second.first, r.dst, (size_t) hd);
+ std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
+ }
+ if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite
+ for (size_t i = 0; i < b.reads.size(); ++i) {
+ const EgRead& r = b.reads[i];
+ for (int k = 0; k < hd / 32; ++k) {
+ uint16_t hb;
+ std::memcpy(&hb, r.dst + (size_t) k * 18, 2);
+ if ((hb & 0x7c00) == 0x7c00)
+ std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),
+ r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);
+ }
+ }
+ }
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (size_t i = 0; i < b.reads.size(); ++i) {
+ const EgRead& r = b.reads[i];
+ if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)
+ std::memset(r.dst, 0, (size_t) hd);
+ std::memset(r.scales, 0, (size_t) hd / 32);
+ continue;
+ }
+ if (uint8_t* c = eg_cache.put(r.key)) {
+ std::memcpy(c, r.dst, (size_t) hd);
+ std::memcpy(c + hd, r.scales, (size_t) hd / 32);
+ }
+ }
+ }
+ eg_io_ns += b.t1.load() - b.t0;
+ b.live = false;
+ b.post.store(2, std::memory_order_release);
+ } else {
+ while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();
+ }
+ eg_wait_ns += now_ns() - t0;
+ }
+ void eg_stats() {
+ st.eg_rows = eg_rows_n.load();
+ st.eg_hits = eg_hits_n.load();
+ st.eg_reads = eg_reads_n.load();
+ st.eg_io_ms = (double) eg_io_ns.load() / 1e6;
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ st.eg_cached = (int64_t) eg_cache.size();
+ st.eg_cache_cap = (int64_t) eg_cache.capacity();
+ }
+ // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)
+ void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {
+ EgBatch b;
+ eg_issue(p0, T, only, dst, b);
+ eg_finish(b);
+ }
+
+ // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its
+ // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,
+ // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's
+ // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down
+ // pieces; the down bands). yd[slot] gets each row's expert output.
+ struct Ent {
+ int e = 0, n = 0, slot0 = 0;
+ int tok[kc::kExl3CpuMaxN] = {};
+ };
+ void run_nodes(int n, const std::function<void(int)>* f) {
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);
+ pools[0]->help();
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();
+ }
+
+ // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each
+ // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),
+ // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits
+ // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds
+ // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever
+ // number of entries take it, and each job its band of h straight into the down's int8 input.
+ // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)
+ void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);
+ constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);
+ const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);
+ const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)
+ const float lim = g.lim;
+ static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");
+ { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at
+ // once, so the SSD reads them with large requests side by side while the first entries compute
+ // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost
+ // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ os::prefetch_cold(want);
+ }
+ // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token
+ // index past the window (none on the decode or chunk paths) gets a row of its own after them
+ uint8_t* xq = (uint8_t*) c_xh.data();
+ uint8_t* hq = xq + (size_t) kWin * xa;
+ uint8_t* xq_extra = hq + (size_t) kSlots * ha;
+ auto x_act = [&](const Ent& en, int i) -> const void* {
+ const int t = en.tok[i];
+ return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));
+ }
+ }
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];
+ float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ gp[i] = yg[i];
+ up[i] = yu[i];
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's)
+ gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);
+ gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);
+ for (int i = 0; i < en.n; ++i) {
+ float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;
+ for (int r = r0; r < r1; ++r) {
+ float gv = yg[i][r - r0], uv = yu[i][r - r0];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);
+ }
+ };
+ run_nodes(ne * GB, f);
+ if (!band_q)
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float* yp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = hq + (size_t) (en.slot0 + i) * ha;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;
+ }
+ gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);
+ };
+ run_nodes(ne * DB, f);
+ }
+ // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input
+ // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same
+#if defined(STRATA_METAL)
+ // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD
+ // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN
+ // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the
+ // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per
+ // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).
+ static bool la_enabled() { return la_depth() > 0; }
+ static int la_depth() {
+ static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)
+ return std::min(d, 2);
+ }
+ struct LaPool {
+ std::mutex m;
+ std::condition_variable cv;
+ std::deque<std::pair<const void*, size_t>> q;
+ LaPool() {
+ const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;
+ for (int i = 0; i < nth; ++i)
+ std::thread([this] {
+ for (;;) {
+ std::pair<const void*, size_t> r;
+ {
+ std::unique_lock<std::mutex> lk(m);
+ cv.wait(lk, [&] { return !q.empty(); });
+ r = q.front();
+ q.pop_front();
+ while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)
+ }
+ const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;
+ const size_t len = r.second + ((uintptr_t) r.first - a);
+ std::vector<unsigned char> vec((len + 16383) / 16384);
+ bool cold = true;
+ if (mincore((void*) a, len, (char*) vec.data()) == 0) {
+ cold = false;
+ for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }
+ }
+ if (cold) os::prefetch({r});
+ }
+ }).detach();
+ }
+ void push(const void* p, size_t n) {
+ { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }
+ cv.notify_one();
+ }
+ };
+ std::mutex la_mu;
+ std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed
+ long la_hit = 0, la_all = 0, la_issued = 0;
+ void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {
+ if (l.nm != 1) return;
+ static LaPool* pool = nullptr;
+ static bool reg = false;
+ if (pool == nullptr) {
+ la_guess.assign(L.size(), {});
+ pool = new LaPool();
+ }
+ if (std::getenv("DS4_CPU_LOG") && !reg) {
+ reg = true;
+ static Impl* self = this;
+ std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });
+ }
+ for (const Ent& en : ents) {
+ ++la_all;
+ const auto& gs = la_guess[(size_t) l.il];
+ if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;
+ }
+ if (la_n <= 0) return;
+ cudaEventSynchronize(ev2);
+ static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;
+ for (int a = 1; a <= la_n; ++a) {
+ Layer& nx = L[(size_t) (l.il + a)];
+ std::vector<int> cpu;
+ for (int r = 0; r < nx.k_exp; ++r) {
+ const int e = h_idx_a[(a - 1) * kPicks + r];
+ if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);
+ }
+ la_guess[nx.il] = cpu;
+ for (int e : cpu)
+ for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }
+ }
+ }
+#endif
+ void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ constexpr int RB = 64; // rows a job
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ gg::IqCpu fm;
+ std::string ferr;
+ if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||
+ fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {
+ static std::atomic<bool> said{false};
+ if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());
+ for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);
+ return;
+ }
+ static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;
+ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }
+ ++n_calls;
+ double tp0 = now_ms();
+ const float lim = g.lim;
+ { // (os::prefetch, as above)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ if (std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)
+ static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;
+ if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }
+ for (const auto& w : want) {
+ const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;
+ unsigned char vec[64];
+ const size_t len = std::min<size_t>(w.second, 64 * 16384);
+ bool cold = false;
+ if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;
+ const double t0 = now_ms();
+ os::prefetch({w});
+ (cold ? cold_ms : warm_ms) += now_ms() - t0;
+ (cold ? cold_n : warm_n)++;
+ }
+ } else if (want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {
+ // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of
+ // all the call's experts at once keep the SSD busy)
+ std::function<void(int)> pf[4];
+ for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };
+ run_nodes((int) want.size(), pf);
+ } else
+ os::prefetch(want);
+ }
+ double tp1 = now_ms(); t_pre += tp1 - tp0;
+ // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of
+ // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or
+ // chunk paths), then (x no longer needed) the slot's h
+ uint8_t* aq = (uint8_t*) c_xh.data();
+ auto x_act = [&](const Ent& en, int i) -> uint8_t* {
+ const int t = en.tok[i];
+ return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
+ }
+ }
+ double tq1 = now_ms(); t_q1 += tq1 - tp1;
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float* hp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any
+ // block size of the type)
+ gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,
+ r0 - X * FFh, r1 - X * FFh, lim);
+ };
+ run_nodes(ne * GB, f);
+ double tr1 = now_ms(); t_r1 += tr1 - tq1;
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
+ aq + (size_t) (kWin + en.slot0 + i) * E * 4);
+ double tq2 = now_ms(); t_q2 += tq2 - tr1;
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* hp[gg::kMxCpuMaxN];
+ float* yp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)
+ }
+ gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
+ };
+ run_nodes(ne * DB, f);
+ t_r2 += now_ms() - tq2;
+ }
+
+ // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
+ struct StoreHold {
+ Impl* m = nullptr;
+ int layer = 0;
+ std::vector<int> es;
+ std::vector<char> ok;
+ StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
+ if (!m_->store) return;
+ m = m_;
+ layer = l.il;
+ for (const Ent& en : ents)
+ if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
+ m->store->ensure(layer, es.data(), (int) es.size(), ok);
+ }
+ ~StoreHold() {
+ if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
+ }
+ };
+ void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
+ const int* ord = nullptr) {
+ if (ldx == 0) ldx = (size_t) g.dim;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ StoreHold hold(this, l, ents);
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
+ const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
+ const int ne = (int) ents.size();
+ const float lim = g.lim;
+ auto part = [&](int e, int X, int r) {
+ const uint8_t* b = l.hh[(size_t) e][(size_t) X];
+ kc::Exl3Part p;
+ p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);
+ p.suh = (const uint16_t*) (b + l.hl.suh[r]);
+ p.svh = (const uint16_t*) (b + l.hl.svh[r]);
+ p.k = l.hl.k[r];
+ p.n = l.hl.n[r];
+ p.K = l.lay.K[r];
+ return p;
+ };
+ auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };
+ auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };
+ auto acc = [&](int X, int sl, int r, int q) {
+ return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;
+ };
+ auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };
+ auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };
+ auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / (2 * GQ))];
+ const int r = (j / GQ) % 2, q = j % GQ;
+ const kc::Exl3Part p = part(en.e, X, r);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ const float* xt = x + (size_t) en.tok[i] * ldx;
+ for (int b = 2 * q; b < 2 * q + 2; ++b)
+ xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);
+ xp[i] = xh(X, sl, r);
+ ap[i] = acc(X, sl, r, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);
+ };
+ run_nodes(ne * 2 * GQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / GB)];
+ const int b = j % GB;
+ const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float y[2][128], a[128], hb[128];
+ for (int r = 0; r < 2; ++r) {
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < GQ; ++q) {
+ const float* aq = acc(X, sl, r, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];
+ kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);
+ }
+ for (int c = 0; c < 128; ++c) {
+ float gv = y[0][c], uv = y[1][c];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ hb[c] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ const int gb = X * GB + b;
+ xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);
+ }
+ };
+ run_nodes(ne * GB, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DQ)];
+ const int q = j % DQ;
+ const kc::Exl3Part p = part(en.e, X, 2);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ xp[i] = xhd(en.slot0 + i);
+ ap[i] = accd(X, en.slot0 + i, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);
+ };
+ run_nodes(ne * DQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DB)];
+ const int b = j % DB;
+ const kc::Exl3Part pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float a[128];
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < DQ; ++q) {
+ const float* aq = accd(X, sl, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];
+ kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);
+ }
+ };
+ run_nodes(ne * DB, f);
+ }
+
+ // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it
+ // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs
+ // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's
+ // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:
+ // the CPU lane reads it from the file again.
+ void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot
+ for (;;) {
+ RamSwap rs;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });
+ if (quit.load()) return;
+ rs = ram_queue.front();
+ ram_queue.pop_front();
+ }
+ const Layer& l = L[(size_t) rs.layer];
+ if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {
+ for (int r = 0; r < 3; ++r) // (one copy of it in RAM)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ } else {
+ rs.in = -1; // (failed: the slot goes back)
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_ready.push_back(rs);
+ }
+ }
+
+ void cache_loop(SwapBufs& B) {
+ for (;;) {
+ Swap sw;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });
+ if (quit.load()) return;
+ sw = sw_queue.front();
+ sw_queue.pop_front();
+ }
+ Layer& l = L[(size_t) sw.layer];
+ if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)
+ swap_in_mx(l, sw, B);
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ continue;
+ }
+ parts_to_blob(l, l.hh[(size_t) sw.in], B.a);
+ if (l.nm == 1) {
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t cs = B.st[(size_t) l.dev];
+ cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);
+ if (!file_experts)
+ cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ } else { // a group: each member's part in, each member's part out
+ for (int m = 0; m < l.nm; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = B.st[(size_t) x.dev];
+ blob_to_half(l, B.a, m, B.h);
+ cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);
+ cudaStreamSynchronize(cs);
+ if (file_experts) continue;
+ cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ half_to_blob(l, B.h, m, B.b);
+ }
+ }
+ if (!file_experts)
+ for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);
+ if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte
+ std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);
+ bool ok = true;
+ for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)
+ const int e = pass == 0 ? sw.in : sw.out;
+ reg.resize(l.region_bytes[(size_t) e]);
+ pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());
+ if (pass == 0) {
+ repack(l, reg.data(), blob.data());
+ std::vector<uint8_t> dev(l.blob);
+ if (l.nm == 1)
+ cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);
+ else
+ for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)
+ std::vector<uint8_t> h(member(l, m).hblob);
+ cudaSetDevice(devs[(size_t) member(l, m).dev].id);
+ cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,
+ cudaMemcpyDeviceToHost);
+ half_to_blob(l, h.data(), m, dev.data());
+ }
+ for (int r = 0; r < 3; ++r)
+ ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&
+ std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&
+ std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;
+ } else {
+ for (int X = 0; X < NH; ++X) {
+ repack_part(l, reg.data(), X, part.data());
+ for (int r = 0; r < 3; ++r) {
+ const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];
+ ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&
+ std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&
+ std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;
+ }
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,
+ ok ? "both match the model file" : "MISMATCH");
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ }
+ }
+
+ // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);
+ // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)
+ // from the layer's spare RAM slot, which it is copied into from VRAM here
+ void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {
+ uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)
+ const std::vector<cudaStream_t>& copy_st = B.st;
+ bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till
+ // this swap's switch)
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);
+ ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ }
+ if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot
+ bool got = ok;
+ for (int m = 0; m < l.nm && got; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),
+ unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);
+ }
+ if (got) blob_to_ram(l, sw_b, sw.ram);
+ else sw.ram = -1; // (back to the file's pages then)
+ }
+ if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte
+ std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),
+ cudaMemcpyDeviceToHost);
+ if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());
+ }
+ ok = ok && read_mx_file(l, sw.in, file.data());
+ for (int r = 0; r < 3 && ok; ++r)
+ ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ bool ok_out = true;
+ if (sw.ram >= 0) {
+ ok_out = read_mx_file(l, sw.out, file.data());
+ ram_to_blob(l, sw.ram, dev.data());
+ for (int r = 0; r < 3 && ok_out; ++r)
+ ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,
+ sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");
+ } else if (!ok) {
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);
+ }
+ }
+
+ void cache_tick(int T) {
+ cache_tick_swaps(T);
+ store_sync();
+ store_watch();
+ }
+ // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time
+ // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free
+ void store_watch() {
+ if (!store) return;
+ static int tick = 0;
+ if (++tick % 16 != 0) return;
+ const uint64_t avail = ram_avail();
+ if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;
+ const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));
+ const size_t n = store->shrink_chunks(chunks);
+ if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());
+ }
+ void cache_tick_swaps(int T) {
+ if (!o.expert_cache || cache_ths.empty()) return;
+ std::vector<Swap> ready;
+ std::vector<RamSwap> rready;
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ready.swap(sw_ready);
+ rready.swap(ram_ready);
+ }
+ for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)
+ --ram_inflight;
+ Layer& l = L[(size_t) rs.layer];
+ if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {
+ ram_free.push_back(rs.slot);
+ continue;
+ }
+ l.ram[(size_t) rs.in] = rs.slot;
+ if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {
+ ram_free.push_back(l.ram[(size_t) rs.out]);
+ l.ram[(size_t) rs.out] = -1;
+ }
+ ++st.ram_moves;
+ }
+ for (const Swap& sw : ready) {
+ Layer& l = L[(size_t) sw.layer];
+ const int old_slot = l.slot[(size_t) sw.out];
+ const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];
+ if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)
+ const int s_in = l.ram[(size_t) sw.in];
+ l.ram[(size_t) sw.in] = -1;
+ if (sw.ram >= 0) {
+ l.ram[(size_t) sw.out] = sw.ram;
+ copy_spare[(size_t) l.il] = s_in;
+ } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)
+ --ram_held;
+ if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;
+ }
+ } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)
+ ram_free.push_back(l.ram[(size_t) sw.in]);
+ l.ram[(size_t) sw.in] = -1;
+ }
+ l.slot[(size_t) sw.in] = l.spare_vram;
+ if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)
+ l.zc[(size_t) sw.in] = 0;
+ store->unmark(l.il, sw.in);
+ }
+ if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
+ l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.in] = nullptr;
+ l.slot[(size_t) sw.out] = -1;
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)
+ l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);
+ } else if (file_experts) { // (back to its pages in the file)
+ l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];
+ } else {
+ l.hh[(size_t) sw.out] = l.spare_host;
+ l.host[(size_t) sw.out] = l.spare_host[0];
+ l.spare_host = old_parts;
+ }
+ l.spare_vram = old_slot;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const Dev& dm = devs[(size_t) x.dev];
+ l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));
+ l.tab_pin[2 * m + 1] = 0ull;
+ cudaSetDevice(dm.id); // (stream order: before the next window)
+ cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);
+ cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);
+ }
+ l.swap_pending = false;
+ --sw_inflight;
+ ++st.swaps;
+ }
+ decay_tokens += T;
+ if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens
+ decay_tokens = 0;
+ for (Layer& l : L)
+ for (float& h : l.heat) h *= c_decay;
+ }
+ bool queued = false, rqueued = false;
+ for (Layer& l : L) {
+ if (l.cpu_hits.empty()) continue;
+ std::vector<int> hits;
+ hits.swap(l.cpu_hits);
+ int best = -1, vin = -1;
+ for (int e : hits)
+ if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;
+ if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in
+ int cold = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;
+ if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {
+ l.swap_pending = true;
+ ++sw_inflight;
+ vin = best;
+ Swap sw{l.il, best, cold};
+ if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_queue.push_back(sw);
+ queued = true;
+ }
+ }
+ if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;
+ int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)
+ for (int e : hits)
+ if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&
+ (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))
+ rb = e;
+ if (rb < 0) continue;
+ int rc = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;
+ const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)
+ if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;
+ RamSwap rs;
+ rs.layer = l.il;
+ rs.in = rb;
+ rs.out = room ? -1 : rc;
+ rs.slot = ram_free.back();
+ ram_free.pop_back();
+ ++ram_inflight;
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_queue.push_back(rs);
+ rqueued = true;
+ }
+ if (queued) sw_cv.notify_all();
+ if (rqueued) ram_cv.notify_one();
+ }
+
+ // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes
+ void prefetch_loop() {
+ std::vector<std::pair<const void*, size_t>> b;
+ for (;;) {
+ {
+ std::unique_lock<std::mutex> lk(pf_mu);
+ pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });
+ if (quit.load(std::memory_order_relaxed)) return;
+ b.swap(pf_q);
+ }
+ os::prefetch_cold(b); // (the resident ones would only cost the page walk)
+ b.clear();
+ }
+ }
+ // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)
+ static bool file_read(const Layer& l, int e) {
+ return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);
+ }
+ // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4
+ // experts read in place, some of them from the files now (none: nothing to read early - null)
+ const Layer* ahead_layer(const Layer& l0, int a) const {
+ const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;
+ if (j >= end || j >= (int) L.size()) return nullptr;
+ const Layer& n = L[(size_t) j];
+ if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||
+ (int) n.slot.size() < n.n_exp)
+ return nullptr;
+ for (int e = 0; e < n.n_exp; ++e)
+ if (file_read(n, e)) return &n;
+ return nullptr;
+ }
+ // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the
+ // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts
+ std::vector<std::pair<const void*, size_t>> pf_want;
+ void ahead_ask(int li, const Req& r, unsigned seq, int T) {
+ pf_want.clear();
+ for (int a = 1; a <= 2; ++a) {
+ const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;
+ if (m <= 0 || j >= (int) L.size()) continue;
+ const Layer& n = L[(size_t) j];
+ uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;
+ uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;
+ if (ahead_for[(size_t) j] != seq + (unsigned) a) {
+ std::memset(fl, 0, 1024);
+ std::memset(rk, 0, 1024);
+ ahead_for[(size_t) j] = seq + (unsigned) a;
+ }
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < m; ++i) {
+ const int e = r.ahead[t][(a - 1) * kAhead1 + i];
+ if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;
+ fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);
+ if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
+ if ((fl[e] & 8) || !file_read(n, e)) continue;
+ fl[e] |= 8;
+ if (store) store->prefetch(j, e);
+ else
+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
+ ++st.ahead_pf;
+ }
+ }
+ if (pf_want.empty()) return;
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());
+ }
+ pf_cv.notify_one();
+ }
+
+ // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's
+ // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.
+ void serve_lane() {
+ unsigned next = 1;
+ auto last = std::chrono::steady_clock::now();
+ std::vector<Ent> ents;
+ while (!quit.load(std::memory_order_relaxed)) {
+ Req& r = reqs[next % kRing];
+ if (*(volatile unsigned*) &r.seq != next) {
+ cpu_relax();
+ if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))
+ std::this_thread::sleep_for(std::chrono::microseconds(20));
+ continue;
+ }
+ std::atomic_thread_fence(std::memory_order_acquire);
+ const int li = layer_of[next % kRing].load(std::memory_order_acquire);
+ Resp& rp = resps[next % kRing];
+ const int T = std::max(1, std::min(kWin, r.T));
+ for (int t = 0; t < T; ++t) rp.cpu[t] = 0;
+ if (li >= 0 && li < (int) L.size()) {
+ Layer& l = L[(size_t) li];
+ uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
+ ents.clear();
+ if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)
+ // lookahead: the guesses made for this layer (when they are for this request), counted on decode
+ const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
+ ? ahead_fl.data() + (size_t) li * 1024 : nullptr;
+ // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row
+ float wv[kWin * kPicks];
+ int tok_of[kWin * kPicks];
+ int nslots = 0;
+ std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
+ if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids
+ // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the
+ // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's
+ // predecessors (the request carries the guesses for the layers after it)
+ static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;
+ uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};
+ std::fwrite(hd, 1, 2, trace_f);
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int16_t v = (int16_t) r.ids[t][i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ if (with_ahead) {
+ const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};
+ std::fwrite(na, 1, 2, trace_f);
+ for (int t = 0; t < T; ++t) {
+ for (int i = 0; i < na[0]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ for (int i = 0; i < na[1]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
+ }
+ }
+ const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
+ for (int t = 0; t < T; ++t) {
+ float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)
+ if (dsk)
+ for (int i = 0; i < l.k_exp; ++i)
+ if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = r.ids[t][i];
+ if (e < 0 || e >= l.n_exp) continue;
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (fl != nullptr) {
+ ++st.ahead_picks;
+ st.ahead_hit_k += fl[e] & 1;
+ st.ahead_hit += (fl[e] & 6) != 0;
+ st.ahead_hit2 += (fl[e] >> 2) & 1;
+ }
+ if (l.slot[(size_t) e] < 0) {
+ l.cpu_hits.push_back(e);
+ if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot
+ store->touch(li, e);
+ ++st.store_zc;
+ continue;
+ }
+ if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
+ ++st.skipped;
+ continue;
+ }
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;
+ else if (fl != nullptr) {
+ ++st.ahead_cold;
+ st.ahead_cold_hit += (fl[e] >> 3) & 1;
+ }
+ picks.push_back({e, t * kPicks + i});
+ }
+ }
+ }
+ if (fl != nullptr) { // the guesses by rank: how many were picked
+ const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;
+ for (int e = 0; e < l.n_exp && e < 1024; ++e) {
+ if (rk[e] == 0 || rk[e] > 16) continue;
+ bool hit = false;
+ for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;
+ ++st.ahead_rank_n[rk[e] - 1];
+ st.ahead_rank_hit[rk[e] - 1] += hit;
+ }
+ }
+ if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);
+ std::sort(picks.begin(), picks.end());
+ for (size_t a = 0; a < picks.size();) {
+ Ent en;
+ en.e = picks[a].first;
+ en.slot0 = nslots;
+ en.n = 0;
+ while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {
+ const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;
+ en.tok[en.n++] = t;
+ tok_of[nslots] = t;
+ wv[nslots] = r.w[t][i];
+ ++nslots;
+ ++a;
+ }
+ ents.push_back(en);
+ }
+ // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for
+ // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)
+ int ord[kWin * kPicks];
+ const int* po = nullptr;
+ if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&
+ ents.size() <= (size_t) (kWin * kPicks)) {
+ const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;
+ int n = 0;
+ for (int pass = 0; pass < 2; ++pass)
+ for (size_t q = 0; q < ents.size(); ++q) {
+ const int e = ents[q].e;
+ const bool early = !file_read(l, e) || (fa[e] & 8) != 0;
+ if (early == (pass == 0)) ord[n++] = (int) q;
+ }
+ po = ord;
+ }
+ if (!ents.empty()) {
+ const double tc = now_ms();
+ cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) {
+ bool any = false;
+ for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;
+ if (!any) continue;
+ float* o = rp.part[t];
+ for (int e = 0; e < D; ++e) o[e] = 0.0f;
+ for (int sl = 0; sl < nslots; ++sl) {
+ if (tok_of[sl] != t) continue;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];
+ }
+ rp.cpu[t] = 1;
+ }
+ st.cpu_ms += now_ms() - tc;
+ st.cpu_experts += nslots;
+ }
+ }
+ std::atomic_thread_fence(std::memory_order_release);
+ *(volatile unsigned*) &rp.seq = next;
+ ++next;
+ last = std::chrono::steady_clock::now();
+ }
+ }
+
+ // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's
+ static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {
+ bool g = false;
+ for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;
+ return g ? gg::mv(j, n, s) : x3::mv(j, n, s);
+ }
+ bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {
+ x3::MvJob j[x3::kMaxMvJobs];
+ int n = 0;
+ for (const x3::MvJob& x : jobs) j[n++] = x;
+ if (!mv_any(j, n, d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {
+ for (int a = 0; a < n; a += x3::kMaxMvJobs)
+ if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs
+ static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,
+ float alpha = 1.0f) {
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = x;
+ j.y = y;
+ j.alpha = alpha;
+ j.ncol = T;
+ j.xf_cs = x_cs;
+ j.y_cs = y_cs;
+ return j;
+ }
+
+ // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:
+ // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm
+ // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x
+ // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),
+ // the trunk's head, the markov bias token by token (greedy) and the confidence head.
+ bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {
+ if (g.n_mtp == 0 || n <= 0) return true;
+ if (n > kWin * kPicks) {
+ err = "seeding the drafter with " + std::to_string(n) + " rows";
+ return false;
+ }
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;
+ if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps
+ for (Dev& e : devs) {
+ cudaSetDevice(e.id);
+ CK(cudaStreamSynchronize(e.st));
+ }
+ cudaSetDevice(d.id);
+ const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);
+ std::fflush(dump_d);
+ }
+ for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)
+ const int na = std::min(kWin, n - a);
+ CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,
+ cudaMemcpyHostToDevice, d.st));
+ if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;
+ }
+ CK(cudaEventRecord(ev_seed, d.st));
+ Finish fm;
+ fm.w = d_main_norm;
+ fm.eps = g.eps;
+ finish(d.ed, n, D, D, fm, d.st);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (l.dev != d_dev) {
+ err = "the DSpark blocks are split across devices";
+ return false;
+ }
+ for (int a = 0; a < n; a += kWin) {
+ const int na = std::min(kWin, n - a);
+ if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) (p0 + a);
+ fk.freqs = d.freqs_win;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, na, HD, HD, fk, d.st);
+ ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);
+ }
+ }
+ return true;
+ }
+
+ bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,
+ int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ const double t0 = now_ms();
+ drafts_q = temperature > 0.0f && sample_drafts;
+ dq_temp = temperature;
+ dq_top_p = top_p;
+ dq_top_k = top_k;
+ dq_min_p = min_p;
+ dq_seed = seed;
+ dq_counter = counter;
+ const bool ok = draft_(y, out, conf, err);
+ if (!ok) drafts_q = false;
+ if (max_drafts > 0 && (int) out.size() > max_drafts) {
+ out.resize((size_t) max_drafts);
+ conf.resize((size_t) max_drafts);
+ }
+ st.draft_ms += now_ms() - t0;
+ ++st.drafts_rounds;
+ return ok;
+ }
+ bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {
+ out.clear();
+ conf.clear();
+ const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;
+ const int64_t p0 = pos;
+ if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;
+ int tk[kWin];
+ tk[0] = y;
+ for (int t = 1; t < T; ++t) tk[t] = g.d_noise;
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float hh[4] = {0, 0, 0, 0};
+ cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);
+ }
+ init_pre(d.pre, T, d.st);
+ WinPlan dp; // (window attention alone: the mailbox seqs)
+ dp.async = !sync_decode && T <= async_max_t;
+ for (int i = 0; i < g.n_mtp && dp.async; ++i) {
+ const int il = g.nl + i;
+ dp.seq[il] = ++seq_issued;
+ layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;
+ if (dump_d != nullptr && h_dump == nullptr &&
+ (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))
+ return false;
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)
+ ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);
+ CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));
+ }
+ if (!layer(l, 0, p0, T, dp, err)) return false;
+ if (dump_d != nullptr) {
+ cudaSetDevice(d.id);
+ CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));
+ }
+ }
+ cudaSetDevice(d.id);
+ hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)
+ Dev& hd = devs[(size_t) head_dev];
+ // (through the host to the head's device, and to the head group's other members - the drafter's device can be
+ // the head group's lead itself, a group of every GPU)
+ if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {
+ CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(d.ev_out, d.st));
+ }
+ if (d_dev != head_dev) {
+ cudaSetDevice(hd.id);
+ CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));
+ }
+ // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)
+ const std::vector<int>& hg = groups[(size_t) hd.grp].mem;
+ const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)
+ for (int m = (int) hg.size() - 1; m >= 0; --m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ if (m > 0) {
+ CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));
+ } else
+ CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));
+ Finish fn;
+ fn.w = e.d_norm;
+ fn.eps = g.eps;
+ finish(e.ao, T, D, D, fn, e.st);
+ if (!head_share(e, m, e.ao, T, err, !mk)) return false;
+ }
+ h_idx[0] = y;
+ if (mk) {
+ // the markov steps over the group: each member its rows of the bias and its share's argmax, every member
+ // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,
+ // each step ~1/nm of their 66 MB and no gather of the shares
+ const int nm = (int) hg.size();
+ const unsigned seq = ++mk_seq;
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));
+ CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));
+ }
+ for (int i = 0; i < T; ++i)
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;
+ markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);
+ argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);
+ markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,
+ e.st);
+ }
+ cudaSetDevice(hd.id);
+ } else {
+ CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)
+ sp.greedy = false;
+ sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);
+ sp.top_p = dq_top_p;
+ sp.top_k = dq_top_k;
+ sp.min_p = dq_min_p;
+ sp.seed = dq_seed;
+ }
+ for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
+ markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
+ if (drafts_q) {
+ sp.counter = dq_counter + (uint64_t) i;
+ strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);
+ } else
+ strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);
+ }
+ }
+ confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
+ int tok[kWin + 1];
+ float cf[kWin];
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];
+ for (int i = 0; i < T; ++i) cf[i] = h_w[i];
+ if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden
+ std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream
+ CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ const int64_t hdr[4] = {'D', p0, y, T};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(tok + 1, 4, (size_t) T, dump_d);
+ std::fwrite(cf, 4, (size_t) T, dump_d);
+ std::fwrite(xs.data(), 4, xs.size(), dump_d);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);
+ std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);
+ }
+ std::fflush(dump_d);
+ cudaSetDevice(hd.id);
+ }
+ for (int i = 0; i < T; ++i) {
+ out.push_back(tok[i + 1]);
+ conf.push_back(cf[i]);
+ }
+ return true;
+ }
+
+ int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,
+ const float* bias) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::dsv41k::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ sp.logit_bias = bias;
+ strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
+ int t = -1;
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.
+ // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own
+ // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution
+ // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the
+ // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.
+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {
+ const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);
+ const int T = n + 1, V = g.vocab;
+ const int64_t p0 = pos;
+ int tk[kWin];
+ tk[0] = y;
+ for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];
+ spec_win = true;
+ const double tv = now_ms();
+ const bool ok = forward(tk, T, T, err, 0);
+ st.verify_ms += now_ms() - tv;
+ spec_win = false;
+ if (!ok) return false;
+ Dev& hd = devs[(size_t) head_dev];
+ cudaSetDevice(hd.id);
+ int m = 0;
+ if (temperature <= 0.0f) {
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = h_idx[m];
+ } else {
+ // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)
+ // (never d itself): each row a token distributed as the model's own draw at that position, p through the
+ // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,
+ // never kept); past the first rejection the rows do not count
+ const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&
+ dq_min_p == min_p && dq_seed == seed && dq_counter == counter;
+ if (!own) { // the drafter's argmax: q a point at each draft
+ for (int i = 0; i < n; ++i) {
+ h_specq[(size_t) i].n = 1;
+ h_specq[(size_t) i].ids[0] = drafts[(size_t) i];
+ h_specq[(size_t) i].p[0] = 1.0;
+ }
+ CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),
+ cudaMemcpyHostToDevice, hd.st));
+ }
+ for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];
+ CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = false;
+ sp.temperature = temperature;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);
+ }
+ drafts_q = false;
+ rollback(p0, T, m + 1);
+ if (!seed_drafter(p0, m + 1, 0, err)) return false;
+ accepted = m;
+ st.drafts_offered += n;
+ st.drafts_accepted += m;
+ for (int i = 0; i < n && i <= m && i < 8; ++i) {
+ ++st.draft_reached[i];
+ if (i < m) ++st.draft_kept[i];
+ }
+ return true;
+ }
+
+ // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone
+ // (the compressed KV and the index keys past them are written again when those positions come)
+ void rollback(int64_t p0, int T, int keep) {
+ if (keep >= T) return;
+ // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor
+ // parallel, a millisecond of the host's before the next draft)
+ std::vector<RingBatch> rb(devs.size());
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ RingBatch& b = rb[(size_t) l.dev];
+ if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)
+ cudaSetDevice(devs[(size_t) l.dev].id);
+ ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);
+ continue;
+ }
+ b.ring[b.n] = l.win;
+ b.bak[b.n++] = l.ring_bak;
+ }
+ for (size_t di = 0; di < devs.size(); ++di)
+ if (rb[di].n > 0) {
+ cudaSetDevice(devs[di].id);
+ ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);
+ }
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ Dev& d = devs[(size_t) l.dev];
+ if (l.c_snap != nullptr) {
+ cudaSetDevice(d.id);
+ const size_t ns = (size_t) l.ratio * g.hd * 4;
+ cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);
+ compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,
+ g.hd, l.c_norm, g.eps, d.latent, d.st);
+ }
+ }
+ pos = p0 + keep;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ }
+
+ // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
+ bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }
+
+
+ // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
+ // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
+ // place, the CPU lane's streamed in batches)
+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ Chunk& c = *chunks[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ prof_mark(d, 0);
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ab = groups[(size_t) L0.grp].arb;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ // Engram, in slabs of 256 tokens
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)
+ uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;
+ CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.eg_ev[slot], s));
+ for (int s0 = 0; s0 < T; s0 += 256) {
+ const int ns = std::min(256, T - s0);
+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
+ x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
+ if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
+ engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
+ }
+ }
+ // attention
+ if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
+ dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
+ hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
+ hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
+ dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
+ dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);
+ dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);
+ dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);
+ dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
+ return false;
+ if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
+ if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
+ dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);
+ dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
+ // the window rows: the ring's (positions p0 - win ..) then the chunk's
+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = c.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(c.kvr, T, HD, HD, fk, s);
+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
+ const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
+ if (snap) {
+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
+ }
+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ const uint8_t* ckv = nullptr;
+ const long long ss = (long long) o.max_context + 64;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ const int64_t g_first = p0 / R, g_end = (p0 + T) / R;
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ const float* latent = c.latent;
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(c.ckr, T, HD, HD, fc, s);
+ latent = c.ckr;
+ } else {
+ if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)
+ const int64_t ps = p0 + wp.snap_t;
+ for (int sl = 0; sl < R; ++sl) {
+ const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;
+ const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;
+ const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;
+ CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ }
+ }
+ compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);
+ }
+ }
+ const int clen_max = (int) ((p0 + T) / R);
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);
+ if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ }
+ if (clen_max > 0) {
+ if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
+ return false;
+ for (int a0 = 0; a0 < T; a0 += c.AI) {
+ const int na = std::min(c.AI, T - a0);
+ const int64_t pa = p0 + a0;
+ if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,
+ g.ix_heads * g.ix_hd, 0.0f, err))
+ return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) pa;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ Rows lim;
+ lim.gen = 1;
+ lim.p0 = pa;
+ lim.R = R;
+ lim.kcap = g.ix_topk;
+ lim.ccap = g.cand_topk;
+ lim.blk = std::max(1, g.cand_blk);
+ const int nlast = (int) ((pa + na) / R); // the slab's widest query
+ int* tk = c.topk + (size_t) a0 * topk_stride();
+ int* cd = c.cand + (size_t) a0 * cand_stride();
+ const float* iw = c.iw + (size_t) a0 * 64;
+ if (nlast <= 0) continue;
+ if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {
+ Rows cl = lim;
+ cl.gen = 2;
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,
+ cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);
+ Rows tr = lim;
+ tr.gen = 3;
+ topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);
+ } else {
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,
+ lim, c.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr = lim;
+ cr.kcap = g.cand_topk;
+ cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,
+ (int) cand_stride(), s);
+ }
+ topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);
+ }
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(const_cast<float*>(latent), ng, HD, HD, fc, s);
+ if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once
+ const int slot = wp.k % 2; // they took the chunk before last's)
+ if (wp.k >= 2)
+ for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.stage(slot);
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);
+ }
+ }
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
+ if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
+ for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
+ const int na = std::min(c.A, T - a0);
+ if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) (p0 + a0);
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(c.q, na * NHL, HD, HD, fr, s);
+ if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);
+ AttnRows ar;
+ ar.gen = 1;
+ ar.p0 = p0;
+ ar.t0 = a0;
+ ar.win = g.win;
+ ar.R = l.ratio;
+ ar.kcap = g.ix_topk;
+ sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
+ na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
+ if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) (p0 + a0);
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(c.o, na * NHL, HD, HD, fo, s);
+ if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);
+ x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,
+ OG * g.o_lora, 0.0f, err))
+ return false;
+ x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
+ if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
+ if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);
+ }
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);
+ hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)
+ prof_mark(d, 1);
+ // the MoE
+ if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
+ hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);
+ hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)
+ for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)
+ const int nr = std::min(kVlRows, T - r0);
+ VlBits vb;
+ const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);
+ route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,
+ c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);
+ }
+ const int n_vs = L0.n_slots + (int) L0.host_list.size();
+ CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)
+ moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);
+ CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));
+ // the shared expert into acc while the host waits for the bounds
+ if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))
+ return false;
+ swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);
+ x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);
+ if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;
+ prof_mark(d, 2);
+ CK(cudaStreamSynchronize(s));
+ std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);
+ if (dump_r != nullptr && mi == 0) {
+ const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;
+ std::vector<int> ri(np), rt(nk);
+ std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);
+ CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
+ cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};
+ std::fwrite(hdr, 8, 8, dump_r);
+ std::fwrite(ri.data(), 4, np, dump_r);
+ std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);
+ std::fwrite(b.data(), 4, b.size(), dump_r);
+ std::fwrite(rt.data(), 4, nk, dump_r);
+ std::fwrite(gl.data(), 4, gl.size(), dump_r);
+ std::fwrite(gb.data(), 4, gb.size(), dump_r);
+ std::fflush(dump_r);
+ }
+ if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids
+ std::vector<int> ri((size_t) T * kPicks);
+ CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};
+ std::fwrite(hd, 1, 3, trace_f);
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
+ if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const int v = L0.vs_pin[e];
+ pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ }
+ if (c.flow) {
+ if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+ // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones
+ // in shares between them (the device works while the copies run)
+ const int di = l.dev, P = Chunk::kParts;
+ int nb = 0;
+ for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;
+ int v_done = 0;
+ for (int bi = 0; bi < nb; ++bi) {
+ const Batch& bt = S.b[S.kp];
+ const int half = (int) (S.kp % P);
+ CK(cudaStreamWaitEvent(s, c.filled[half], 0));
+ const int v0 = L0.n_slots + bt.j0;
+ if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,
+ b.data(), v0, v0 + bt.n, T, err))
+ return false;
+ CK(cudaEventRecord(c.freed[half], s));
+ ++S.kp;
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+
+ // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)
+ void plan_streams() {
+ for (Layer& l : L) {
+ if (l.mtp || l.vs_pin == nullptr) continue;
+ l.host_list.clear();
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);
+ for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)
+ Layer& x = member(l, m);
+ int j = 0;
+ for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;
+ }
+ }
+ }
+
+ // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows
+ Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {c.h, c.pre, c.topk, c.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the
+ // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows
+ // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across
+ // layers and chunks
+ struct Run {
+ std::vector<int64_t> p0;
+ std::vector<int> T;
+ std::vector<unsigned> seq;
+ int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
+ };
+ // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
+ // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
+ // layers and back before the next - so a layer's experts are read from the files once for the whole run (the
+ // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk
+ bool run_dev_layers(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S; // (none: the flow streams)
+ const size_t hb = chunk_hand_bytes();
+ // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while
+ // the one before runs (slot u % 2, once unit u - 2's copy of it is done)
+ std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
+ for (int il : c.fl_layers)
+ if (L[(size_t) il].eg >= 0)
+ for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
+ if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
+ const int li = egu[u].first, k = egu[u].second;
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);
+ };
+ size_t eg_u = 0;
+ std::future<void> eg_next;
+ if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);
+ for (size_t li = 0; li < c.fl_layers.size(); ++li) {
+ const int il = c.fl_layers[li];
+ Layer& L0 = L[(size_t) il];
+ const bool last_l = li + 1 == c.fl_layers.size();
+ for (int k = 0; k < nk; ++k) {
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way
+ const double te = now_ms();
+ eg_next.get();
+ st.engram_ms += now_ms() - te;
+ wp.eg_slot = (int) (eg_u % 2);
+ if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);
+ }
+ if (li == 0) { // the chunk's input: its embedding rows
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = c.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ init_pre(c.pre, T, s);
+ } else { // or its state after the layer before
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));
+ }
+ if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;
+ if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
+ if (!last_l) { // the chunk's state kept for the next layer
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));
+ continue;
+ }
+ // through every layer: the chunk's end, and the logits
+ if ((size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ lm_layers.store((int) li + 1, std::memory_order_release);
+ }
+ return true;
+ }
+ bool run_dev_chunks(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S;
+ const int B = std::max(1, c.ring_n / Chunk::kParts);
+ if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)
+ CK(cudaStreamSynchronize(s));
+ CK(cudaStreamSynchronize(c.cs));
+ c.items.clear();
+ c.base = c.n_read = c.n_dma = c.n_comp = 0;
+ c.n_file = 0;
+ std::fill(c.stage_item.begin(), c.stage_item.end(), -1);
+ c.fl_layers.clear();
+ for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)
+ for (int m = 0; m < L[(size_t) il].nm; ++m)
+ if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
+ c.fl_T = run.T;
+ c.spec_k = c.spec_il = -1;
+ c.wait_ms = 0;
+ c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
+ c.lc_il = -1;
+ c.lc_used = 0;
+ c.lc_first = c.lc_chk = 0;
+ c.io_d0 = c.io->direct_bytes();
+ c.io_b0 = c.io->buffered_bytes();
+ if (!c.fl_layers.empty()) {
+ flow_spec(c, 0, c.fl_layers[0], run.T[0]);
+ if (!flow_pump(c, err)) return false;
+ }
+ if (c.lm) return run_dev_layers(di, run, err);
+ }
+ for (int k = 0; k < nk && !c.flow; ++k)
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& L0 = L[(size_t) il];
+ bool here = false;
+ for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
+ if (!here) continue;
+ for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)
+ S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});
+ }
+ if (!S.b.empty() && c.ring == nullptr) {
+ err = "no slots to stream experts into on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ std::atomic<bool> ra_stop{false};
+ std::thread ra;
+ if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });
+ struct Join { // (every way out of the run: the read-ahead ends before S does)
+ std::atomic<bool>& stop;
+ std::thread& th;
+ ~Join() {
+ stop.store(true);
+ if (th.joinable()) th.join();
+ }
+ } ra_join{ra_stop, ra};
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once
+ // chunk k - 2's copy of it is done)
+ std::vector<int> egs;
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egs, eg_per](int k) {
+ if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);
+ for (int li : egs)
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);
+ };
+ std::future<void> eg_next;
+ if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
+ for (int k = 0; k < nk; ++k) {
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way
+ const double te = now_ms();
+ eg_next.get();
+ if (d.mi == 0) st.engram_ms += now_ms() - te;
+ if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);
+ }
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = c.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ init_pre(c.pre, T, s);
+ } else {
+ Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);
+ set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);
+ }
+ }
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ Xfer x;
+ if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot);
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (ng > 0 && sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);
+ set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)
+ }
+ if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
+ const int nnext = L[(size_t) il + 1].nm;
+ if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);
+ xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);
+ }
+ }
+ if (di == head_dev && (size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ if (di != head_dev) continue;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ // ---- a prompt's path: the flow or windows of 16
+ // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a
+ // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in
+ // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the
+ // flow, as before)
+ void calibrate_paths() {
+ if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;
+ if (const char* v = std::getenv("DS4_PATH_PICK"))
+ if (std::atoi(v) == 0) return;
+ int di = -1;
+ for (size_t i = 0; i < chunks.size() && di < 0; ++i)
+ if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;
+ if (di < 0) return;
+ Chunk& c = *chunks[(size_t) di];
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const double t0 = now_ms();
+ // the DMA: staging -> the ring (once to warm up, once timed)
+ const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});
+ double ms = 0;
+ for (int rep = 0; rep < 2; ++rep) {
+ const double a = now_ms();
+ if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||
+ cudaStreamSynchronize(c.cs) != cudaSuccess) {
+ cudaGetLastError();
+ return;
+ }
+ ms = now_ms() - a;
+ }
+ pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);
+ // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are
+ auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {
+ const double a = now_ms();
+ left.store((int) jobs.size());
+ c.io->push(jobs.data(), (int) jobs.size());
+ while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ return now_ms() - a;
+ };
+ {
+ const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;
+ std::vector<uint8_t> src(part * 8, 1);
+ std::vector<IoQueue::Job> jobs(8);
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < 8; ++i) {
+ jobs[(size_t) i].mem = src.data() + (size_t) i * part;
+ jobs[(size_t) i].dst = c.stage + (size_t) i * part;
+ jobs[(size_t) i].len = part;
+ jobs[(size_t) i].left = &left;
+ jobs[(size_t) i].bad = &bad;
+ }
+ run_jobs(jobs, left);
+ const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);
+ pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);
+ }
+ // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:
+ // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane
+ // three times (from the disk, then in RAM: the faster of two)
+ int best = -1, most = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int n = 0;
+ for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;
+ if (n > most) most = n, best = il;
+ }
+ if (best < 0) return;
+ const Layer& l = L[(size_t) best];
+ std::vector<int> cold;
+ const int n_read = std::min(8, c.S), n_cpu = 8;
+ for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {
+ if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;
+ bool res = false;
+ for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);
+ if (!res) cold.push_back(e);
+ }
+ if ((int) cold.size() >= n_read + n_cpu) {
+ std::vector<IoQueue::Job> jobs;
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < n_read; ++i) {
+ uint8_t* sb = c.stage + (size_t) i * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ IoQueue::Job j;
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];
+ j.len = mx_part_bytes(l, r);
+ j.dst = sb;
+ j.left = &left;
+ j.bad = &bad;
+ j.map = pk.at(l.mx_file[r], j.off);
+ jobs.push_back(j);
+ sb += flow_region(r);
+ }
+ }
+ const double rd = run_jobs(jobs, left);
+ if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);
+ }
+ { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM
+ const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;
+ std::vector<Ent> ents;
+ for (int i = 0; i < n_cpu; ++i) {
+ Ent en;
+ en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;
+ if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)
+ for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)
+ if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;
+ if (en.e < 0) break;
+ en.n = 1;
+ en.slot0 = i;
+ en.tok[0] = 0;
+ ents.push_back(en);
+ }
+ if (ents.empty()) return;
+ std::vector<float> x((size_t) g.dim);
+ for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);
+ const double bytes = (double) ents.size() * (double) l.blob;
+ double a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ const double t_first = now_ms() - a;
+ double t_warm = 1e30;
+ for (int rep = 0; rep < 2; ++rep) {
+ a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ t_warm = std::min(t_warm, now_ms() - a);
+ }
+ pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);
+ if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);
+ }
+ pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;
+ if (o.verbose) {
+ char dk[32] = "not measured", cc[32] = "not measured";
+ if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);
+ if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);
+ std::fprintf(stderr,
+ "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "
+ "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",
+ pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);
+ }
+ }
+ // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's
+ // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)
+ double pm_rate(const Layer& l, int& n_cold) const {
+ const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0, cold = 0;
+ n_cold = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ sum += pr[e];
+ if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;
+ }
+ if (n_cold == 0) return 0.0;
+ const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;
+ return pm.m * (double) l.k_exp * share / n_cold;
+ }
+ // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))
+ double pm_picks(int n) const {
+ double d = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int nc = 0;
+ const double a = pm_rate(L[(size_t) il], nc);
+ d += nc * (1.0 - std::exp(-a * n));
+ }
+ return d;
+ }
+ // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation
+ // that (bisection over its log; a running mean of the log)
+ void pm_learn(int n, double picked) {
+ if (!pm.on || n < 2 || picked < 1) return;
+ const double m0 = pm.m;
+ double lo = std::log(1e-3), hi = std::log(1e3);
+ for (int it = 0; it < 40; ++it) {
+ pm.m = std::exp(0.5 * (lo + hi));
+ (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);
+ }
+ const double obs = 0.5 * (lo + hi);
+ pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);
+ ++pm.n_m;
+ }
+ // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's
+ // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -
+ // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:
+ // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at
+ // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
+ // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
+ // faster (by half, with less than kPickRes in RAM); said with --verbose
+ bool pick_path(int n) {
+ pm.last = -1;
+ if (!pm.on || n >= kPickMax) return true;
+ double f_res = 0, f_cold = 0, w = 0;
+ const int nw = n / kWin, rem = n % kWin;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ int nc = 0;
+ const double a = pm_rate(l, nc);
+ if (nc == 0) continue;
+ int sampled = 0, res = 0, seen = 0;
+ const int step = std::max(1, nc / 32);
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;
+ bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ if (!r) {
+ r = true;
+ for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);
+ }
+ ++sampled;
+ res += r;
+ }
+ const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;
+ auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };
+ f_res += share * u(n) * b;
+ f_cold += (1.0 - share) * u(n) * b;
+ w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;
+ }
+ const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;
+ const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);
+ const double t_flow = f_res / r_fr + f_cold / r_fc;
+ const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;
+ const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;
+ // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the
+ // fixed costs of every layer on every GPU again, which a pass of the flow pays once)
+ const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;
+ // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane
+ // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would
+ // cost the most; windows win where they are all in RAM)
+ const double res_share = f_res / std::max(1.0, f_res + f_cold);
+ const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));
+ pm.last = flow ? 1 : 0;
+ pm.last_cls = cls;
+ pm.last_nw = n_win;
+ pm.last_pred = flow ? t_flow : t_win;
+ if (o.verbose)
+ std::fprintf(stderr,
+ "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "
+ "experts' %.1f GB in RAM)\n",
+ n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,
+ 100.0 * res_share, (f_res + f_cold) / 1e9);
+ return flow;
+ }
+ // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more
+ static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }
+ // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs
+ // is mostly the fixed costs a window pays again and again)
+ double pm_k(int c) const {
+ for (int dc = 0; dc < 4; ++dc)
+ for (int cc : {c - dc, c + dc})
+ if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];
+ return 1.0;
+ }
+ // n tokens through windows of 16, the last one's logits_rows rows of logits
+ bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {
+ logits_all = false;
+ for (int a = 0; a < n; a += kWin) {
+ const int T = std::min(kWin, n - a);
+ if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;
+ }
+ return true;
+ }
+ // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken
+ // / predicted), windows' time a window - running means
+ void path_done(double ms) {
+ if (pm.last < 0 || pm.last_pred <= 0) return;
+ if (pm.last == 0) {
+ const double w = ms / 1000.0 / std::max(1, pm.last_nw);
+ pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;
+ ++pm.win_n;
+ } else {
+ const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));
+ double& k = pm.k[0][pm.last_cls];
+ int64_t& n = pm.kn[0][pm.last_cls];
+ k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;
+ ++n;
+ }
+ pm.last = -1;
+ }
+ // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the
+ // first would read from the files again more than the chunks' states moved to host memory and back at every layer
+ // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,
+ // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB
+ // sets it) - at least one expert, or not layer-major
+ bool lm_begin(const Run& run) {
+ static const char* env = std::getenv("DS4_LAYER_MAJOR");
+ if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;
+ Chunk& c = *chunks[0];
+ if (!c.flow) return false;
+ size_t need = 0, n_exp = 0, blob_max = 0;
+ double file_b = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ size_t ne = 0;
+ for (int e : l.host_list)
+ if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;
+ const size_t stride = (l.blob + 63) & ~(size_t) 63;
+ need = std::max(need, ne * stride);
+ n_exp = std::max(n_exp, (size_t) l.n_exp);
+ blob_max = std::max(blob_max, stride);
+ file_b += (double) ne * (double) l.blob;
+ }
+ int64_t rows = 0;
+ for (int T : run.T) rows += T;
+ const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;
+ const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;
+ if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;
+ size_t lc_bytes = need;
+ if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);
+ else {
+ const uint64_t avail = ram_avail();
+ if (avail != ~uint64_t{0})
+ lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);
+ }
+ if (lc_bytes < blob_max) return false;
+ void* st_p = nullptr;
+ c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;
+ if (!c.lm_pinned) {
+ cudaGetLastError();
+ st_p = os::anon_alloc(state_b, -1, 1);
+ if (st_p == nullptr) return false;
+ }
+ c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);
+ if (c.lc == nullptr) {
+ if (c.lm_pinned) cudaFreeHost(st_p);
+ else os::anon_free(st_p, state_b);
+ return false;
+ }
+ c.lm_state = (uint8_t*) st_p;
+ c.lm_state_bytes = state_b;
+ c.lc_bytes = lc_bytes;
+ c.lc_of.assign(n_exp, -1);
+ c.lc_writer.assign(n_exp, -1);
+ c.lm = true;
+ lm_layers.store(0);
+ return true;
+ }
+ void lm_end() {
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ Chunk& c = *chunks[di];
+ if (!c.lm) continue;
+ cudaSetDevice(devs[di].id);
+ cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's
+ cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)
+ for (size_t i = c.base; i < c.n_read; ++i)
+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ if (c.lm_pinned) cudaFreeHost(c.lm_state);
+ else os::anon_free(c.lm_state, c.lm_state_bytes);
+ os::anon_free(c.lc, c.lc_bytes);
+ c.lm_state = c.lc = nullptr;
+ c.lm_state_bytes = c.lc_bytes = 0;
+ c.lm = c.lm_pinned = false;
+ }
+ }
+
+ // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
+ // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
+ bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
+ if (n < 1) return true;
+ if (pos + n > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < n; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);
+ Run run;
+ for (int a = 0; a < n; a += per) {
+ run.p0.push_back(pos + a);
+ run.T.push_back(std::min(per, n - a));
+ run.seq.push_back(++win_seq);
+ }
+ logits_all = logits_rows > kWin;
+ if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
+ for (int t = 0; t < n; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ plan_streams();
+ if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way
+ run.snap_pos = pos + n - 1;
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ Dev& d = devs[(size_t) l.dev];
+ if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;
+ if (l.c_kv != nullptr && l.snap_kv == nullptr &&
+ (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ }
+ const int T_last = run.T.back();
+ n_logits = logits_all ? 0 : std::min(logits_rows, T_last);
+ const double tl = now_ms();
+ { // each chunk's end on the head device, for the progress reports
+ cudaSetDevice(devs[(size_t) head_dev].id);
+ while (chunk_done.size() < run.T.size()) {
+ cudaEvent_t e = nullptr;
+ CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ chunk_done.push_back(e);
+ }
+ chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);
+ for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);
+ }
+ prof_chunk = prof_on;
+ const bool lm = lm_begin(run);
+ struct LmEnd { // (every way out: its host memory freed once the device is done with it)
+ Impl* p;
+ bool on;
+ ~LmEnd() {
+ if (on) p->lm_end();
+ }
+ } lm_end_{this, lm};
+ std::vector<std::string> errs(devs.size());
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, &run, &errs] {
+ std::string e;
+ if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ size_t reported = 0;
+ for (;;) {
+ bool all = true;
+ for (auto& w : workers) all = all && w->idle();
+ if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
+ const int nd = lm_layers.load(std::memory_order_acquire);
+ if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
+ } else if (progress) { // chunks through every layer (the last layer's device past their event)
+ while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&
+ cudaEventQuery(chunk_done[reported]) == cudaSuccess)
+ ++reported;
+ if (reported > 0 && reported < run.T.size())
+ progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);
+ }
+ if (all) break;
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ for (const std::string& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ st.layer_ms += now_ms() - tl;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ cudaSetDevice(d.id);
+ prof_collect();
+ std::fprintf(stderr,
+ "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "
+ "(with the waits for streamed ones), between layers %.0f ms\n",
+ n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);
+ for (double& v : cprof_ms) v = 0;
+ prof_chunk = false;
+ }
+ if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {
+ const double ms = now_ms() - tl;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ const Chunk& c = *chunks[di];
+ if (!c.flow) continue;
+ const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;
+ const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;
+ const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)
+ char lcs[160] = "";
+ if (c.lm)
+ std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "
+ "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);
+ std::fprintf(stderr,
+ "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "
+ "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "
+ "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "
+ "%.0f ms for reads\n",
+ devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),
+ (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,
+ (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,
+ (unsigned long long) c.it_spec_unused, c.wait_ms);
+ }
+ }
+ if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)
+ bool tp = false;
+ for (const Layer& l : L) tp = tp || l.nm > 1;
+ double picked = 0;
+ for (const auto& cp : chunks)
+ if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;
+ if (!tp) pm_learn(n, picked);
+ }
+ if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)
+ pos += n;
+ st.tokens += n;
+ cache_tick(n);
+ const int nr = std::min(T_last, tap_cap);
+ if (!seed_drafter(pos - nr, nr, 0, err)) return false;
+ if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -
+ // the next feed seeds them again from its own rows, past this position)
+ Check& ckp = ck[snap_slot & 1];
+ ckp.pos = run.snap_pos;
+ ckp.win.resize(L.size());
+ ckp.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& dv = devs[(size_t) l.dev];
+ cudaSetDevice(dv.id);
+ const bool kept = !l.mtp && l.snap_win != nullptr;
+ ckp.win[i].resize((size_t) g.win * win_rb());
+ CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),
+ cudaMemcpyDeviceToHost, dv.st));
+ if (l.c_kv) {
+ ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,
+ ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,
+ ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ }
+ }
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ }
+ return true;
+ }
+
+ // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)
+ void plan_window(int64_t p0, int T, WinPlan& wp) {
+ wp.par = par;
+ wp.async = par || (!sync_decode && T <= async_max_t);
+ wp.vl = 0;
+ for (int t = 0; t < T && !imgs.empty(); ++t)
+ if (in_image(p0 + t)) wp.vl |= 1u << t;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ if (l.ratio > 0) {
+ const int R = l.ratio;
+ int n[kWin], k[kWin], clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ n[t] = (int) ((p0 + t + 1) / R);
+ k[t] = std::min(g.ix_topk, n[t]);
+ clen_max = std::max(clen_max, n[t]);
+ }
+ if (l.ix_src) {
+ for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];
+ if (clen_max > 0 && l.cand_src)
+ for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);
+ for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];
+ }
+ for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];
+ }
+ if (wp.async) {
+ wp.seq[il] = ++seq_issued;
+ layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ }
+ }
+
+ int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }
+ // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's
+ // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows
+ void head_cols(int nm, int mi, int& v0, int& vn) const {
+ const Tensor* tw = pk.get("head.weight");
+ if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {
+ vn = g.vocab / nm;
+ v0 = mi * vn;
+ } else
+ split128(g.vocab, nm, mi, v0, vn);
+ }
+ // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's
+ // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two
+ // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather
+ // false: each member keeps its share (the drafter's markov steps over the group, mk_split)
+ bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {
+ const int V = g.vocab, D = g.dim, nm = head_nm();
+ int v0 = 0, vn = 0, Vmax = 0, z = 0;
+ head_cols(nm, mi, v0, vn);
+ head_cols(nm, 0, z, Vmax);
+ Group& G = groups[(size_t) d.grp];
+ float* out = mi == 0 ? d.logits : d.logits_part;
+ const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;
+ cudaSetDevice(d.id);
+ // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4
+ // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)
+ const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head)); // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)
+ for (int c = 0; c < rows; c += per)
+ if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))
+ return false;
+ if (nm == 1 || !gather) return true;
+ const unsigned seq = ++d.head_seq;
+ const int slot = (int) (seq % 2);
+ auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };
+ auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };
+ if (mi > 0) {
+ if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = d.logits_part;
+ x.dst[0] = stage(mi);
+ x.bytes[0] = (size_t) rows * vn * 4;
+ xfer_put(x, flag(mi), seq, d.xcnt, d.st);
+ return true;
+ }
+ for (int m = 1; m < nm; ++m) {
+ int m0 = 0, mn = 0;
+ head_cols(nm, m, m0, mn);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = stage(m);
+ x.dst[0] = d.logits_part;
+ x.bytes[0] = (size_t) rows * mn * 4;
+ xfer_get(x, flag(m), seq, d.st);
+ CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,
+ (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));
+ }
+ set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);
+ return true;
+ }
+
+ // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)
+ bool head_rows_dev(int di, int T, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const int D = g.dim, H = g.hc;
+ if (n_logits <= 0) return true;
+ const int t0 = T - n_logits;
+ hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);
+ return head_share(d, d.mi, d.x, n_logits, err);
+ }
+
+ // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read
+ // h_taps)
+ bool tap(Dev& d, int il, int T, std::string& err) {
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (g.n_mtp == 0 || it == g.d_targets.end()) return true;
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int D = g.dim;
+ cudaSetDevice(d.id);
+ hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);
+ CK(cudaStreamWaitEvent(d.st, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,
+ cudaMemcpyDeviceToHost, d.st));
+ return true;
+ }
+
+ bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {
+#if defined(STRATA_METAL)
+ struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)
+ bool on;
+ BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }
+ ~BusyAll() { if (on) ds4_busy_end(); }
+ } busy_all;
+#endif
+ if (T < 1 || T > kWin) {
+ err = "a window of " + std::to_string(T) + " tokens";
+ return false;
+ }
+ if (pos + T > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < T; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ const int64_t p0 = pos;
+ for (int t = 0; t < T; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is
+ // left of their reads by then is the window's Engram time
+ const double te = now_ms();
+ const int64_t eg_w0 = eg_wait_ns.load();
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ EgBatch& b = *eg_win[(size_t) li];
+ if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)
+ eg_issue(p0, T, li, nullptr, b);
+ }
+ st.engram_ms += now_ms() - te;
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)
+ std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);
+ plan_window(p0, T, wplan);
+ prof_window = prof_on && T == 1;
+ n_logits = std::min(logits_rows, T);
+ const double tl = now_ms();
+ if (par) {
+ if (!forward_par(p0, T, err)) return false;
+ } else if (!forward_serial(p0, T, err)) return false;
+ st.layer_ms += now_ms() - tl;
+ for (auto& b : eg_win)
+ if (b->live) eg_finish(*b);
+ st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ prof_collect();
+ pos += T;
+ st.tokens += T;
+ cache_tick(T);
+ if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;
+ return true;
+ }
+
+ // one thread: the layers device after device, the hand-offs through pinned memory and events
+ void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer
+ ds4_metal_flush();
+ const float* hp = (const float*) strata::gpu::host_view(dev);
+ size_t nf = 0, first = 0; float mx = 0.0f;
+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }
+ std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);
+ }
+ void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains
+ ds4_metal_flush();
+ const float* hp = (const float*) strata::gpu::host_view(d.h);
+ float mx = 0.0f; size_t nf = 0;
+ const size_t n = (size_t) T * g.hc * g.dim;
+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }
+ std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);
+ }
+ bool forward_serial(int64_t p0, int T, std::string& err) {
+ const int D = g.dim, H = g.hc;
+ int cur = L[0].dev;
+ if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();
+ {
+ Dev& d = devs[(size_t) cur];
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ init_pre(d.pre, T, d.st);
+ if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);
+ }
+ std::vector<float> dump;
+ if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ if (l.dev != cur) {
+ Dev& a = devs[(size_t) cur];
+ Dev& b = devs[(size_t) l.dev];
+ cudaSetDevice(a.id);
+ const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ uint8_t* bb = a.bounce;
+ CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaEventRecord(a.ev_out, a.st));
+ cudaSetDevice(b.id);
+ CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));
+ CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));
+ cur = l.dev;
+ }
+ if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
+ if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);
+ if (!layer(l, 0, p0, T, wplan, err)) return false;
+ if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
+ Dev& dd = devs[(size_t) l.dev];
+ std::vector<float> hv((size_t) T * H * D);
+ cudaSetDevice(dd.id);
+ CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));
+ CK(cudaStreamSynchronize(dd.st));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,
+ (size_t) H * D * 4);
+ }
+ }
+ if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);
+ if (cur != head_dev) {
+ err = "the head is not on the last layer's GPU";
+ return false;
+ }
+ return head_rows_dev(head_dev, T, err);
+ }
+
+ // threads: each device's thread enqueues its layers (a group's members each their share) at once, the
+ // devices waiting on each other through host-mapped flags (the window's seq)
+ bool forward_par(int64_t p0, int T, std::string& err) {
+ wplan.win = ++win_seq;
+ std::vector<std::string> errs(devs.size());
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, p0, T, &errs] {
+ std::string e;
+ if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ for (auto& w : workers) w->wait();
+ for (const std::string& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ return true;
+ }
+
+ // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer
+ Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {d.h, d.pre, d.topk, d.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
+ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, s);
+ if (std::getenv("DS4_DBG_EMB")) {
+ ds4_metal_flush();
+ float *hp = d.h;
+ float hh[4] = {0,0,0,0};
+ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
+ }
+ init_pre(d.pre, T, s);
+ } else {
+ const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);
+ }
+ }
+ // the compressed KV / index keys another group's source layer wrote this window
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ if (ng == 0) continue;
+ Xfer x;
+ if (sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage;
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);
+ }
+ if (mi == 0 && !tap(d, il, T, err)) return false;
+ if (!layer(L0, mi, p0, T, wp, err)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);
+ return true;
+ }
+
+ // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,
+ // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it
+ bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,
+ size_t bytes, int from, std::string& err) {
+ bool any = false;
+ for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);
+ if (!any) return true;
+ Dev& d = devs[(size_t) from];
+ CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(ev, d.st));
+ for (size_t od = 0; od < devs.size(); ++od) {
+ if (copies[od] == nullptr || (int) od == from) continue;
+ Dev& o2 = devs[od];
+ cudaSetDevice(o2.id);
+ CK(cudaStreamWaitEvent(o2.st, ev, 0));
+ CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));
+ }
+ cudaSetDevice(d.id);
+ return true;
+ }
+
+ // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;
+ // the partial sums all-reduced across the group)
+ bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ag = groups[(size_t) L0.grp].ar;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's
+ ds4_metal_flush();
+ int shown = 0;
+ for (int e = 0; e < l.n_exp && shown < 3; ++e) {
+ if (l.slot[(size_t) e] < 0) continue;
+ std::vector<uint8_t> want(l.blob, 0);
+ read_mx_file(l, e, want.data());
+ const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));
+ size_t bad = 0, first = 0;
+ for (size_t i = 0; i < l.blob; ++i)
+ if (want[i] != have[i]) { if (!bad) first = i; ++bad; }
+ std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);
+ ++shown;
+ }
+ }
+ if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);
+ if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
+ std::vector<float> hv((size_t) T * H * D);
+ cudaSetDevice(d.id);
+ ds4_metal_flush();
+ {
+ const float* hvp = (const float*) strata::gpu::host_view(d.h);
+ if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);
+ }
+ int nf = 0, nzv = 0;
+ float mx = 0.0f;
+ size_t firstnan = 0, lastnan = 0;
+ for (size_t i = 0; i < hv.size(); ++i) {
+ const float v = hv[i];
+ if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }
+ if (v != 0.0f) ++nzv;
+ if (std::fabs(v) > mx) mx = std::fabs(v);
+ }
+ std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",
+ l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],
+ hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);
+ }
+ prof_mark(d, 0);
+ // Engram
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)
+ uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;
+ for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]
+ CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,
+ cudaMemcpyHostToDevice, s));
+ uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;
+ for (int t = 0; t < T; ++t)
+ CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
+ (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
+ dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);
+ dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);
+ if (std::getenv("DS4_DBG_EG")) {
+ dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
+ const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
+ for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)
+ if (!std::isfinite(xv[i])) {
+ const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;
+ const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;
+ const uint8_t* hv = src + t * per + c * ehd;
+ std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);
+ for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);
+ std::fprintf(stderr, "\n");
+ break;
+ }
+ }
+ if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
+ if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
+ dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);
+ dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);
+ engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
+ dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
+ }
+ dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));
+ // attention
+ hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
+ hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
+ dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);
+ dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);
+ dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);
+ dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);
+ hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
+ dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);
+ dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
+ {
+ x3::MvJob j[4];
+ int nj = 0;
+ j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);
+ j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);
+ if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);
+ if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);
+ if (!mv_n(d, j, nj, err)) return false;
+ }
+ dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
+ dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(d.qr, T, g.q_lora, g.q_lora, fq, s);
+ dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
+ if (NHL > 0) {
+ if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) p0;
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(d.q, T * NHL, HD, HD, fr, s);
+ }
+ if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);
+ // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark
+ // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of
+ // that ring and to the whole block, and leaves the ring as it was
+ ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, T, HD, HD, fk, s);
+ if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
+ dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);
+ dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)
+ AttnRows ar;
+ for (int t = 0; t < T; ++t) {
+ if (l.mtp) {
+ ar.win0[t] = 0;
+ ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);
+ ar.nsel[t] = T;
+ continue;
+ }
+ const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)
+ ar.win0[t] = first;
+ ar.nwin[t] = g.win + t - first + 1;
+ ar.nsel[t] = 0;
+ }
+ const void* ckv = nullptr;
+ int ckv_fmt = kKvFp4;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ // the groups this window completes: g_first .. g_first + ng - 1
+ const int64_t g_first = p0 / R; // the first group whose last position is >= p0
+ const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(d.ckr, T, HD, HD, fc, s);
+ CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));
+ } else {
+ if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected
+ const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;
+ CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
+ }
+ compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
+ dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);
+ dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);
+ }
+ }
+ Rows lim;
+ int clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ lim.n[t] = (int) ((p0 + t + 1) / R);
+ lim.k[t] = std::min(g.ix_topk, lim.n[t]);
+ clen_max = std::max(clen_max, lim.n[t]);
+ }
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),
+ (size_t) ng * ik_rb(), l.dev, err))
+ return false;
+ }
+ if (clen_max > 0) {
+ if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) p0;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ // (iw rows are 64 apart)
+ gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);
+ const long long ss = (long long) o.max_context + 64;
+ bool restrict_ = false;
+ for (int t = 0; t < T; ++t)
+ restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;
+ if (l.uses_cand && restrict_) {
+ Rows cl; // the candidate positions of each query: its blocks x blk
+ int nmax = 0;
+ for (int t = 0; t < T; ++t) {
+ cl.n[t] = lim.n[t];
+ cl.k[t] = wp.cand[l.il][t];
+ nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);
+ }
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,
+ (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);
+ Rows tr;
+ for (int t = 0; t < T; ++t) {
+ tr.n[t] = wp.cand[l.il][t] * g.cand_blk;
+ tr.k[t] = lim.k[t];
+ }
+ topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);
+ } else {
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,
+ lim, d.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr;
+ for (int t = 0; t < T; ++t) {
+ cr.n[t] = lim.n[t];
+ cr.k[t] = g.cand_topk;
+ }
+ cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,
+ (int) cand_stride(), s);
+ }
+ topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(d.latent, ng, HD, HD, fc, s);
+ if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),
+ (size_t) ng * ckv_rb(), l.dev, err))
+ return false;
+ if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.x_stage;
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.x_stage + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
+ }
+ }
+ if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);
+ dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);
+ for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection
+ ckv = d.ext + (size_t) g.win * HD;
+ ckv_fmt = kKvBf16;
+ iota_rows(d.topk, T, T, (int) topk_stride(), s);
+ }
+ if (NHL > 0) {
+ sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,
+ 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);
+ dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) p0;
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(d.o, T * NHL, HD, HD, fo, s);
+ dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);
+ x3::MvJob j[16];
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);
+ if (!mv_n(d, j, OG, err)) return false;
+ if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;
+ } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)
+ dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);
+ if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
+ dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ prof_mark(d, 1);
+ // the MoE
+ hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
+ hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);
+ hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
+ dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);
+ dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);
+ dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);
+ dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
+ if (wp.async) {
+ // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members
+ // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the
+ // router made), and waits for the answer only before the experts' sum
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ const unsigned sq = wp.seq[l.il];
+ const int slot = (int) (sq % kRing);
+ const bool lead = mi == 0;
+ int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below
+ for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {
+ const Layer* nx = ahead_layer(L0, a);
+ if (nx == nullptr) break;
+ const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);
+ float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;
+ gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN
+ // weights instead: the same hit rate)
+ route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);
+ na[a - 1] = m;
+ }
+ route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,
+ d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],
+ na[1]);
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;
+ swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;
+ prof_mark(d, 2);
+ const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;
+ if (T == 1 && moe_token_path) {
+ x3::MoeArgs ma;
+ ma.plan_ptr = d.plan_ptr;
+ ma.plan_w = d.rw;
+ ma.cpu_part = lead ? resps[slot].part[0] : nullptr;
+ ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;
+ ma.lay = lay;
+ ma.k = l.k_exp;
+ ma.n_embd = D;
+ ma.n_ff = FF;
+ ma.limit = g.lim;
+ const bool mx = lay.fmt == gg::kFmtMXFP4;
+ if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)
+ zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);
+ if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
+ else x3::moe_gate_up(ma, d.x, d.xg, s);
+ prof_mark(d, 3);
+ if (lead) wait_seq(&resps[slot].seq, sq, s);
+ prof_mark(d, 4);
+ if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
+ else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
+ } else { // each distinct expert read once for the window's tokens
+ gf::MoeWin mw;
+ mw.plan_ptr = d.plan_ptr;
+ mw.T = T;
+ mw.k = l.k_exp;
+ mw.grp_ptr = d.grp_ptr;
+ mw.grp_start = d.grp_start;
+ mw.grp_ent = d.grp_ent;
+ mw.n_groups = d.n_groups;
+ gf::moe_win_group(mw, s);
+ x3::WinArgs wa;
+ wa.grp_ptr = d.grp_ptr;
+ wa.grp_start = d.grp_start;
+ wa.grp_ent = d.grp_ent;
+ wa.n_groups = d.n_groups;
+ wa.plan_ptr = d.plan_ptr;
+ wa.plan_w = d.rw;
+ wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;
+ wa.cpu_stride = kMaxDim;
+ wa.cpu_flag = lead ? resps[slot].cpu : nullptr;
+ wa.wait_ptr = lead ? &resps[slot].seq : nullptr;
+ wa.wait_val = sq;
+ wa.x = d.x;
+ wa.x_stride = (size_t) D;
+ wa.h = d.xg;
+ wa.part = d.ed;
+ wa.sh_out = d.sh_out;
+ wa.out = d.moe_out;
+ wa.out_stride = (size_t) D;
+ wa.lay = lay;
+ wa.T = T;
+ wa.k = l.k_exp;
+ wa.n_embd = D;
+ wa.n_ff = FF;
+ wa.limit = g.lim;
+ if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {
+ err = "exl3 win_experts failed";
+ return false;
+ }
+ }
+ if (lead) st.experts += (int64_t) T * l.k_exp;
+ if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 5);
+ return true;
+ }
+ if (nm > 1) {
+ err = "a tensor-parallel layer on the host-routed path";
+ return false;
+ }
+ // (glog rows are 1024 apart: route reads them so)
+ static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes
+ static double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;
+ const double ph0 = phases ? now_ms() : 0.0;
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ VlBits vb = {};
+ vb.w[0] = l.mtp ? 0u : wp.vl;
+ route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,
+ vb.w[0] != 0 ? &vb : nullptr);
+ CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaEventRecord(d.ev, s));
+ // the shared expert while the host looks at the picks
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;
+#if defined(STRATA_METAL)
+ int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer
+ if (T == 1 && la_enabled()) {
+ for (int a = 1; a <= la_depth(); ++a) {
+ const int il = l.il + a;
+ if (il >= g.nl) break;
+ const Layer& nx = L[(size_t) il];
+ if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;
+ gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);
+ route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,
+ d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);
+ CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));
+ la_n = a;
+ }
+ if (la_n > 0) CK(cudaEventRecord(d.ev2, s));
+ }
+#endif
+ if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);
+ std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }
+ swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);
+ if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;
+ const double tw = now_ms();
+ CK(cudaEventSynchronize(d.ev));
+ st.gpu_wait_ms += now_ms() - tw;
+ const double ph2 = phases ? now_ms() : 0.0;
+ dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);
+ dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);
+ dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);
+ dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);
+ // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the
+ // CPU lane's entries
+ Combine cm;
+ Gather ga;
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];
+ int nvex = 0, nrows = 0;
+ std::vector<Ent> ents;
+ std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)
+ std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)
+ for (int t = 0; t < T; ++t) {
+ cm.n[t] = 0;
+ cm.cpu[t] = 0;
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = h_idx[t * kPicks + i];
+ if (e < 0 || e >= l.n_exp) {
+ err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";
+ return false;
+ }
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
+ per[(size_t) e].push_back({t, i});
+ }
+ }
+ int nslots = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const auto& pe = per[(size_t) e];
+ if (pe.empty()) continue;
+#if defined(STRATA_METAL)
+ const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));
+#else
+ const bool on_gpu = l.slot[(size_t) e] >= 0;
+#endif
+ if (on_gpu) {
+ vex[nvex] = e;
+ vrow0[nvex] = nrows;
+ vn[nvex] = (int) pe.size();
+ for (const auto& tp : pe) {
+ ga.src[nrows] = tp.first;
+ const int t = tp.first;
+ cm.row[t][cm.n[t]] = nrows;
+ cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];
+ ++cm.n[t];
+ ++nrows;
+ }
+ ++nvex;
+ } else {
+ for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {
+ Ent en;
+ en.e = e;
+ en.slot0 = nslots;
+ en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);
+ for (int i = 0; i < en.n; ++i) {
+ en.tok[i] = pe[a + (size_t) i].first;
+ cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});
+ cm.cpu[en.tok[i]] = 1;
+ }
+ nslots += en.n;
+ ents.push_back(en);
+ }
+ }
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ st.cpu_experts += nslots;
+#if defined(STRATA_METAL)
+ if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);
+#endif
+ if (nrows > 0) {
+ gather_rows(d.x, ga, nrows, D, d.xe, s);
+ dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);
+ { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }
+ dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }
+ x3::MvJob j[2 * kWin * kPicks];
+ int nj = 0;
+ for (int i = 0; i < nvex; ++i) {
+ const auto& vm = l.vm[(size_t) vex[i]];
+ j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ }
+ if (!mv_n(d, j, nj, err)) return false;
+ dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);
+ dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);
+ swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);
+ dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);
+ nj = 0;
+ for (int i = 0; i < nvex; ++i)
+ j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,
+ vn[i], g.ff, D);
+ if (!mv_n(d, j, nj, err)) return false;
+ dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);
+ }
+ const double ph3 = phases ? now_ms() : 0.0;
+ if (!ents.empty()) {
+ const double tc = now_ms();
+#if defined(STRATA_METAL)
+ ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
+ ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)
+#endif
+ static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
+ if (!skip_cpu) cpu_experts(l, h_x, ents);
+ else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)
+#if defined(STRATA_METAL)
+ ds4_busy_end();
+#endif
+ std::fill(h_part, h_part + (size_t) T * D, 0.0f);
+ for (int sl = 0; sl < nslots; ++sl) {
+ float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
+ const float wv = cpu_rows[(size_t) sl].second;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv * y[e];
+ }
+ { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;
+ ms += now_ms() - tc; ++calls; exps += (long) ents.size();
+ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }
+ (void) 0; (void) ms; }
+ st.cpu_ms += now_ms() - tc;
+ dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);
+ CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));
+ }
+ combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);
+ dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);
+ if (std::getenv("DS4_DBG_MOE")) {
+ float hp = 0.0f;
+ for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));
+ std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);
+ dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);
+ dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);
+ }
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ if (phases && T == 1) {
+ const double ph4 = now_ms();
+ ph[0] += ph2 - ph0; ph[1] += ph3 - ph2; ph[2] += ph4 - ph3; if (ph_prev > 0 && ph0 - ph_prev < 5.0) ph[3] += ph0 - ph_prev; ph_prev = ph4; ++ph_n;
+ static bool reg = false;
+ if (!reg) { reg = true; std::atexit([] { std::fprintf(stderr, "phases per layer call (%ld): route enqueue + shared expert + wait %.3f ms, routing analysis + GPU experts enqueue %.3f ms, CPU lane + combine + hc_post %.3f ms, attention enqueue %.3f ms%s", ph_n, ph[0] / ph_n, ph[1] / ph_n, ph[2] / ph_n, ph[3] / ph_n, "\n"); }); }
+ }
+ // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
+ return true;
+ }
+
+ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::dsv41k::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
+ sp, d.tok, d.st);
+ int t = -1;
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ bool logits(std::vector<float>& out) {
+ if (logits_all) {
+ out = chunk_logits;
+ return true;
+ }
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ out.resize((size_t) std::max(1, n_logits) * g.vocab);
+#if defined(STRATA_METAL)
+ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
+#endif
+ return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
+ }
+
+ void save_profile() const {
+ if (o.profile.empty()) return;
+ const std::string tmp = o.profile + ".tmp";
+ std::FILE* f = std::fopen(tmp.c_str(), "wb");
+ if (f == nullptr) return;
+ const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();
+ std::fclose(f);
+ // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile
+ // never moved on and only .tmp files were left)
+ if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());
+ }
+
+ std::vector<std::string> info_lines() const {
+ std::vector<std::string> v;
+ int nh = 0;
+ for (const Layer& l : L) nh += l.n_exp - l.n_vram;
+ char b[512];
+ // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)
+ std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "
+ "cpu_experts=%d ram_experts=%d engine=ds4-0.1",
+ (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);
+ v.push_back(b);
+ if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)
+ std::snprintf(b, sizeof b,
+ "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "
+ "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "
+ "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);
+ v.push_back(b);
+ std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),
+ pagefile_hint);
+ v.push_back(b);
+ }
+ if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor
+ std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,
+ pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);
+ v.push_back(b);
+ }
+ if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)
+ std::snprintf(b, sizeof b,
+ "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "
+ "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",
+ vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,
+ (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),
+ (long long) (vt.device_bytes() >> 20));
+ v.push_back(b);
+ } else if (!vision_err.empty()) { // (the server offers no pictures and says why)
+ std::string why = vision_err;
+ for (char& ch : why)
+ if (ch == '\n' || ch == '\r') ch = ' ';
+ v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");
+ v.push_back("INFO vision=off");
+ }
+ return v;
+ }
+};
+
+Model::Model() : p_(new Impl) {}
+Model::~Model() = default;
+bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }
+bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }
+bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);
+ if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
+ const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
+ const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
+ : p.feed_windows(tokens, n, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ return p.feed_windows(tokens, n, logits_rows, err);
+}
+bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ const double t0 = now_ms();
+ if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
+ const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;
+ p.checkpoint(slot);
+ const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)
+ return ok;
+}
+int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
+void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
+int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
+ return p_->sample(t, tp, tk, mp, seed, counter);
+}
+bool Model::logits(std::vector<float>& out) { return p_->logits(out); }
+int Model::image_token() const { return p_->img_tok; }
+int Model::dim() const { return p_->g.dim; }
+bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {
+ return p_->encode_image(rgb, width, height, span, err);
+}
+void Model::set_images(std::vector<Image> images) {
+ std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });
+ p_->imgs = std::move(images);
+}
+int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }
+bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,
+ float mp, uint64_t seed, uint64_t counter) {
+ return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);
+}
+bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,
+ int& accepted, int& next, std::string& err) {
+ return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);
+}
+int64_t Model::pos() const { return p_->pos; }
+int64_t Model::max_context() const { return p_->o.max_context; }
+int Model::vocab() const { return p_->g.vocab; }
+const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
+void Model::reset() { p_->reset(); }
+void Model::checkpoint(int slot) { p_->checkpoint(slot); }
+bool Model::restore(int64_t limit) { return p_->restore(limit); }
+int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
+void Model::save_profile() const { p_->save_profile(); }
+const Stats& Model::stats() const {
+ Stats& st = p_->st;
+ if (p_->store) { // (the store keeps its own counters: copied in when asked)
+ const ExpertStore& x = *p_->store;
+ st.store_hits = x.stats().hits.load();
+ st.store_misses = x.stats().misses.load();
+ st.store_prefetched = x.stats().prefetched.load();
+ st.store_pf_used = x.stats().prefetch_used.load();
+ st.store_pf_wasted = x.stats().prefetch_wasted.load();
+ st.store_evictions = x.stats().evictions.load();
+ st.store_bytes = x.stats().bytes_read.load();
+ st.store_fails = x.stats().fails.load();
+ st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
+ st.store_busy_ms = x.busy_ms();
+ st.store_zc_visible = (int64_t) x.visible_count();
+ st.store_slots = (int64_t) x.slots_allocated();
+ st.store_slots_max = (int64_t) x.slots_max();
+ }
+ return st;
+}
+std::string Model::describe() const { return p_->describe(); }
+std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
+
+} // namespace strata::ds4