Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: import Project Maya's DeepSeek-V4.1 engine unmodified (v1.1.17, 8d9a9b5d)

mw00bbd0a2c67300
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hpp
new file mode 100644
index 000000000..12f862aaf
--- /dev/null
+++ b/include/strata/dsv41/exl3.hpp
@@ -0,0 +1,185 @@
+// include/strata/kernels/exl3.hpp - EXL3 (exllamav3) quantized weights on the device: the decode GEMV, the routed
+// experts, and the full reconstruction for the prompt path.
+//
+// The format (exllamav3, MIT, turboderp). A weight with k inputs and n outputs (y = x W) is stored as
+// trellis int16 [k/16][n/16][16 K]: each 16x16 tile is a tail-biting trellis of 256 K-bit steps read MSB-first
+// from its 8 K uint32 words (little-endian pairs of the int16s). Step p's state is the 16-bit window that
+// ends at bit (p + 1) K, modulo the tile's 256 K bits; the codebook turns the state into a value:
+// mul1 (cb 2): fp16(1024 + bytesum(state * 0x83DCD12D)) * fp16(0x1eee) + fp16(0xc931), one FP16 FMA
+// mcg (cb 1): x = state * 0xCBAC1FED, 3inst (cb 0): x = state * 89226354 + 64248484; then
+// x = (x & 0x8fff8fff) ^ 0x3b603b60 and the value is the FP16 sum of x's two halves.
+// Step p sits at tile row (p/8 % 4) * 2 + (p & 1) + 8 (p/2 & 1), column p/32 + 8 (p/4 & 1) (the tensor-
+// core fragment order).
+// suh, svh FP16 [k], [n]: the input and output channel scales (signs and magnitudes).
+// With H the 128-point Sylvester Hadamard over sqrt(128), applied to each 128 consecutive values:
+// y = H(H(x * suh) W') * svh, W' the decoded k x n matrix.
+//
+// Activations enter as F32 or as the engine's q8_1 blocks; the kernels rotate them on the fly (each block rotates
+// the slice it reads), so callers keep their existing buffers.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::kernels::exl3 {
+
+/// The weight type the engine's MvJob / WSlot carry for EXL3: their weight pointer is a (host) const Mat*.
+constexpr int kTypeEXL3 = 200;
+
+/// One EXL3 matrix, or a column range of one (a view: trellis/svh advanced to its first column, ld the row stride).
+struct Mat {
+ const uint32_t* trellis = nullptr; // device; 4-byte aligned
+ const uint16_t* suh = nullptr; // device FP16 bits [k]
+ const uint16_t* svh = nullptr; // device FP16 bits [n]
+ int k = 0, n = 0;
+ int K = 0; // bits per weight, 1..8
+ int cb = 2; // codebook: 0 3inst, 1 mcg, 2 mul1
+ int ld = 0; // tiles per 16-row slice in memory (the full matrix's n / 16); 0: n / 16
+ // a GGUF's plain matrix instead (DeepSeek-V4.1's maya-ds4: fmt 1 BF16): raw = [n][ld] (row r = output r, ld
+ // values a row; 0: k), the EXL3 fields unused - the engine sends it to its own kernels, never to these
+ int fmt = 0;
+ const void* raw = nullptr;
+};
+
+/// Trellis bytes of a k x n matrix at K bits.
+inline size_t trellis_bytes(int64_t k, int64_t n, int K) { return (size_t) (k / 16) * (size_t) (n / 16) * 32u * K; }
+/// The columns [c0, c0 + nc) of m (c0 and nc multiples of 128: the output rotation's blocks stay whole).
+Mat view(const Mat& m, int c0, int nc);
+/// Whether the kernels take this matrix (k, n multiples of 128, K 1..8, cb 0..2).
+bool supported(const Mat& m);
+
+/// y[r] = alpha * (x W)[r] + (bias ? bias[r] : 0). x: xf (F32) when given, else xq (q8_1 blocks of k values).
+/// ncol > 1: as many activations (a verify window's tokens), activation c at xf + c xf_cs / xq + c xq_cs bytes, its
+/// y at y + c y_cs - the weights read and decoded once for up to 8 of them, and each y bit for bit its own job's.
+struct MvJob {
+ const Mat* m = nullptr;
+ const float* xf = nullptr;
+ const void* xq = nullptr;
+ float* y = nullptr;
+ const float* bias = nullptr;
+ float alpha = 1.0f;
+ int ncol = 1;
+ size_t xf_cs = 0, xq_cs = 0, y_cs = 0;
+};
+constexpr int kMaxMvJobs = 8;
+constexpr int kMaxMvCols = 16; // a job's activations at most
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+/// The calling thread's launches of mv(), moe_gate_up() and moe_down() read `flag` (a device int) first and do no work
+/// while it is non-zero (glmf::set_skip's, which sets it: a pipelined window's tokens past a wrong draft); nullptr (the
+/// default): no check. The flag changes only between launches (skip_gate, on the same stream). (v1.1.8's kernels
+/// before sm_80 do not read it.)
+void set_skip(const int* flag);
+/// How many pieces mv cuts m's inputs into on the current device (0: not covered) - one piece, and every column's
+/// sum is the same whatever columns the job holds (a view's, or the whole matrix's).
+int mv_splits(const Mat& m);
+/// How many activations a lone job of m can take with its weights read once on the current device (the input split's
+/// partials of them in the scratch; 0: not covered) - a caller with more cuts them in runs of this many.
+int mv_cols(const Mat& m);
+
+/// Allocates the per-stream scratch of mv / moe / win_experts on the current device (call once per stream before a
+/// graph capture; the first use does it otherwise).
+bool prepare(cudaStream_t s);
+
+/// One routed expert's blob: three parts (gate, up, down) at byte offsets from the blob base, each holding the FP16
+/// suh, the FP16 svh and the trellis of that matrix.
+struct ExpertLayout {
+ size_t suh[3] = {0, 0, 0}, svh[3] = {0, 0, 0}, trellis[3] = {0, 0, 0};
+ int K[3] = {0, 0, 0};
+ int cb = 2;
+ int fmt = 0; // 2: a GGUF's MXFP4 expert (maya-ds4's own kernels): trellis[r] = matrix r's offset, [out][in / 32]
+};
+/// The routed experts of one token (the engine's device plan: plan_ptr[i] = the blob in VRAM, 0 = not on the device;
+/// plan_w[i] its weight). gate/up over x (n_embd F32) -> clamped swiglu -> h (F32 [k][n_ff], scratch the caller
+/// owns), then down over h, the weighted sum in plan order, + cpu_part when *cpu_flag, + sh_out when given -> out.
+struct MoeArgs {
+ const unsigned long long* plan_ptr = nullptr;
+ const float* plan_w = nullptr;
+ const float* cpu_part = nullptr;
+ const int* cpu_flag = nullptr;
+ ExpertLayout lay;
+ int k = 8, n_embd = 4096, n_ff = 2048;
+ float limit = 0.0f; // 0: no clamp
+};
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
+
+/// A verify window's routed experts, each distinct expert read once for all the tokens it serves (after exllamav3's
+/// fused MoE, which runs the slots that picked one expert as the rows of one tile): T tokens' plans [T][8] grouped by
+/// expert (glmf::moe_win_group: group g's blob at grp_ptr[g], its entries e = t * 8 + i at grp_ent[grp_start[g] ..
+/// grp_start[g + 1])), token t's F32 input at x + t x_stride, its output at out + t out_stride (plus the CPU lane's
+/// part when cpu_flag[t], plus sh_out's row t) - every token's output is moe_gate_up + moe_down's for it alone, bit for
+/// bit. h [T * 8][n_ff] and part [T * 8][n_embd] are the caller's scratch.
+struct WinArgs {
+ const unsigned long long* grp_ptr = nullptr;
+ const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+ const unsigned long long* plan_ptr = nullptr; // [T][8]
+ const float* plan_w = nullptr; // [T][8]
+ const float* cpu_part = nullptr; // [T][n_embd] (rows cpu_stride apart when given)
+ size_t cpu_stride = 0;
+ const int* cpu_flag = nullptr; // [T]
+ const float* x = nullptr;
+ size_t x_stride = 0;
+ float* h = nullptr;
+ float* part = nullptr;
+ const float* sh_out = nullptr; // [T][n_embd] or null
+ float* out = nullptr;
+ size_t out_stride = 0;
+ ExpertLayout lay;
+ int T = 0, k = 8, n_embd = 4096, n_ff = 2048;
+ float limit = 0.0f;
+ // (given) the combine first waits on the device until *wait_ptr == wait_val (the host's answer to the CPU lane's
+ // part arrives while the device computes the experts); cpu_flag may be null (no CPU part)
+ const unsigned* wait_ptr = nullptr;
+ unsigned wait_val = 0;
+};
+bool win_experts(const WinArgs& a, cudaStream_t s);
+/// Whether the current device runs the several-activation kernels (win_experts; mv's ncol > 1): every device does
+/// (they take ~10 KB of shared memory a block).
+bool several_ok();
+
+/// The prompt path's tensor-core GEMMs (sm_80 and later; mma_supported() says whether this device runs them).
+bool mma_supported();
+/// A set of n_exp routed experts in VRAM, expert e's blob at wbase + e * stride (the layout in lay), owning the sorted
+/// rows [bounds[e], bounds[e + 1]) (device, n_exp + 1 entries); a sorted row's LOCAL row is bounds[e] - row0 + r.
+struct SetArgs {
+ const uint8_t* wbase = nullptr;
+ size_t stride = 0;
+ ExpertLayout lay;
+ int n_exp = 0;
+ const int* bounds = nullptr;
+ int row0 = 0;
+ int max_rows = 0; // the most rows one expert of the set has
+ int n_embd = 4096, n_ff = 2048;
+ float limit = 0.0f;
+};
+/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).
+void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);
+/// y[local row] = down(h[local row]) (F32 rows, ldy floats apart).
+void set_down(const SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);
+/// Y[r] = beta Y[r] + (X[r] W) for r < rows: one matrix over FP16 rows (ldx), F32 output rows (ldy).
+void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s);
+
+/// The full weight in the engine's row layout: dst[r][c] = W[c][r] (n rows of k values), BF16 or F16 bits.
+void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);
+/// The decoded W' alone in that layout: dst[c][r] = W'[r][c], F16 bits (exactly the decoded values) - for a product
+/// that rotates its activations and outputs instead of the weight (x W = H(H(x * suh) W') * svh): rows_rot_f16 before
+/// the GEMM, rows_rot_out after it. No Hadamard over the k x n weight: a plain decode, at the memory's speed.
+void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s);
+/// out[r][c] = FP16(H(x_r * suh)[c]) for r < nrows, c < ncols (ncols % 128 == 0), x_r = x[idx ? idx[r] : r] (F32, ldx
+/// floats a row): an EXL3 matrix's rotated input rows, as the decode kernels rotate theirs.
+void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,
+ cudaStream_t s);
+/// y[r][c] = H(y_r)[c] * svh[c] in place for r < nrows, c < ncols (each 128 values one block; ldy floats a row).
+void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s);
+/// The decoded W' alone, row-major [k][n] F16 bits (tests).
+void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s);
+
+/// out[r][c] = FP16(x[idx ? idx[r] : r][c]) for r < nrows, c < ncols (x: ldx floats a row): the prompt path's rows of
+/// one expert, gathered for its GEMMs.
+void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s);
+
+/// Launch failures of this family since the process started (each is also printed).
+int launch_errors();
+
+} // namespace strata::kernels::exl3
diff --git a/include/strata/dsv41/exl3_cpu.hpp b/include/strata/dsv41/exl3_cpu.hpp
new file mode 100644
index 000000000..f72b674c4
--- /dev/null
+++ b/include/strata/dsv41/exl3_cpu.hpp
@@ -0,0 +1,67 @@
+// include/strata/kernels/cpu/exl3_cpu.hpp - EXL3 routed experts on the CPU (the fast path's CPU lane).
+//
+// The mul1 codebook is affine in the byte sum of state * 0x83DCD12D: w = fp16(1024 + s) * k_inv + k_bias (one FP16
+// FMA on the device). The CPU keeps the affine form without the FP16 rounding, so a 128-output band is
+// y' = k_inv * sum_k xh_k s_kc + (1024 k_inv + k_bias) * sum_k xh_k, xh = H(x * suh)
+// and y = H(y') * svh. The byte sums come from AVX-512 (permutes and funnel shifts pull 16 columns' states at once,
+// vpmaddubsw / vpmaddwd add the bytes up, an F32 FMA takes the activation), or from AVX2 + FMA the same way 8 columns
+// at a time - the same FMAs in the same order, so the two give the same bits; a CPU with neither runs a scalar loop.
+// The format: include/strata/kernels/exl3.hpp.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::kernels::cpu {
+
+/// One EXL3 matrix (mul1 codebook): k inputs, n outputs, K bits (1..8), tiles in exllamav3's natural layout.
+struct Exl3Part {
+ const uint32_t* trellis = nullptr;
+ const uint16_t* suh = nullptr; // FP16 bits [k]
+ const uint16_t* svh = nullptr; // FP16 bits [n]
+ int k = 0, n = 0, K = 0;
+};
+/// Whether a vector kernel runs here (AVX-512F/BW or AVX2 + FMA); the scalar one otherwise.
+bool exl3_cpu_vector();
+/// The kernel the functions below run here: 2 AVX-512 (F, BW), 1 AVX2 + FMA (+ F16C), 0 scalar. STRATA_FORCE_AVX2=1
+/// keeps an AVX-512 CPU on AVX2 (to compare the two; they give the same bits).
+int exl3_cpu_tier();
+/// xh = H(x * suh) (k values, k % 128 == 0); returns the sum of xh.
+float exl3_cpu_prepare(const Exl3Part& m, const float* x, float* xh);
+/// One 128-value block of that: xh[i] = H(x * suh)[i] for i < 128 (suh: the block's 128 scales); returns its sum.
+float exl3_cpu_prepare_block(const uint16_t* suh, const float* x, float* xh);
+/// The 128 outputs from column c0 (a multiple of 128) of y = H(x W') * svh, from exl3_cpu_prepare's xh and sum.
+void exl3_cpu_band(const Exl3Part& m, const float* xh, float xsum, int c0, float* y);
+/// The same in parts: the byte-sum products of the 16-row slices [ks0, ks1) (xh from slice 0) ...
+void exl3_cpu_band_acc(const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc);
+/// exl3_cpu_band_acc for n activations of the same matrix at once (xh[i] -> acc[i], each 128 floats): each tile row is
+/// decoded once for up to kExl3CpuMaxN of them, and acc[i] is exl3_cpu_band_acc's for xh[i] alone, bit for bit.
+constexpr int kExl3CpuMaxN = 8;
+void exl3_cpu_band_acc_n(const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+ float* const* acc);
+/// The byte-sum products of the slices [ks0, ks1) for EVERY column (acc: m.n floats, overwritten) - a split-K piece,
+/// whose tile rows lie in memory order where a band's are 1 KB runs a whole tile row apart (from DRAM, ~1.5x the
+/// bands' speed). Each column's sums are exl3_cpu_band_acc's over the same slices, bit for bit.
+void exl3_cpu_rows_acc(const Exl3Part& m, const float* xh, int ks0, int ks1, float* acc);
+/// ... for n activations at once (xh[i] -> acc[i]), each tile row decoded once; acc[i] is the call for xh[i] alone.
+void exl3_cpu_rows_acc_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, float* const* acc);
+/// ... over the columns [c0, c1) alone (multiples of 128; acc[i] indexed by column, only those overwritten): a piece cut
+/// by columns too, for more jobs than pieces - each column's sums are the whole piece's, bit for bit.
+void exl3_cpu_rows_acc_cols_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int c0, int c1,
+ float* const* acc);
+/// The byte-sum products of nb bands from c0 (128 nb columns; acc[i]: 128 nb floats) over every input for any n
+/// activations, as a GEMM: each 256-input chunk of a band decoded once into F32 in `wc` (exl3_cpu_gemm_scratch()
+/// floats, 64-byte aligned, the caller's), then multiplied for all n, the chunk of the activations kept in L2 across
+/// the nb bands - for many tokens an expert (the prompt path), where the band kernels decode a tile row again every
+/// kExl3CpuMaxN. The same sums as exl3_cpu_band_acc, in its order.
+size_t exl3_cpu_gemm_scratch();
+void exl3_cpu_band_gemm(const Exl3Part& m, const float* const* xh, int n, int c0, int nb, float* const* acc,
+ float* wc);
+/// exl3_cpu_band_acc_n on a given tier (0, 1, 2 as exl3_cpu_tier) - tests and benchmarks; false (nothing computed)
+/// where this CPU cannot run it.
+bool exl3_cpu_band_acc_on(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+ float* const* acc);
+/// ... and, with the parts' acc summed over every slice, the outputs (the codebook's affine form, H, svh).
+void exl3_cpu_band_out(const Exl3Part& m, const float* acc, float xsum, int c0, float* y);
+
+} // namespace strata::kernels::cpu
diff --git a/include/strata/dsv41/sampler.hpp b/include/strata/dsv41/sampler.hpp
new file mode 100644
index 000000000..a7779c92d
--- /dev/null
+++ b/include/strata/dsv41/sampler.hpp
@@ -0,0 +1,92 @@
+// include/strata/kernels/sampler.hpp - the sampler chain, host-callable (P2.S2).
+//
+// penalties -> top_k -> top_p -> min_p -> temperature -> pick
+//
+// llama.cpp's default chain (issue #53): ONE penalties stage, first, and temperature AFTER the truncation filters;
+// top_p cuts before min_p. `src/kernels/cuda/sampler.cu` has the details and `sampler_parity` pins them.
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace strata::kernels {
+
+struct SamplerParams {
+ int top_k = 20; // sampled path: 1..64 as given; 0 (off) or more than 64 keep the widest list, 64
+ float top_p = 0.95f; // 1.0 disables the filter
+ float min_p = 0.0f; // 0 disables; keeps tokens with p >= min_p * p_max (a prefix of the top_k list)
+ float temperature = 1.0f; // <= 0 means greedy
+ int min_keep = 1; // top_p keeps at least this many
+ int penalty_last_n = 0; // 0 disables; the window over `history` to count occurrences in
+ float penalty_repeat = 1.0f;
+ float penalty_freq = 0.0f;
+ float penalty_present = 0.0f;
+ uint64_t seed = 0; // drives Philox, which is counter-based on (seed, token index)
+ uint64_t counter = 0; // absolute draw index of row 0; advance across decode calls
+ bool greedy = false;
+ // a request's logit_bias: a DEVICE array of n_vocab floats on the sampling device, added to every row's logits
+ // before the penalties and the filters (-inf bans a token); nullptr = none
+ const float* logit_bias = nullptr;
+ // the GLM engine's penalty history, on the HOST (never read by a kernel): every token the request's state has been
+ // fed so far, oldest first - its prompt, then each token it wrote. Glm5Model::pick_tokens hands each pick the last
+ // penalty_last_n of them (a verify window's row t: these, then the window's drafts 1..t). nullptr = none
+ const std::vector<int32_t>* host_history = nullptr;
+};
+
+// logits (n_tokens, n_vocab) -> one sampled token id per row in `out`.
+//
+// **`logits` AND `out` ARE DEVICE POINTERS.** This is a CUDA kernel launch, not a host function, and nothing
+// in the parameter names or the types says so - `const float* logits` reads exactly like a host buffer.
+// Measured: passing host memory for `logits` faults inside the kernel with an ILLEGAL MEMORY ACCESS that is
+// reported by whatever synchronising call happens NEXT, which will be somewhere else entirely and will name a
+// buffer that has nothing to do with it. A sticky async fault does not know where it came from.
+//
+// `history` is (n_tokens, history_len) int32 - ONE ROW PER TOKEN ROW, at a stride of `history_len` - the most
+// recent tokens that row's pick follows, with any unused slots set to -1; only the last `p.penalty_last_n` of
+// each row are counted, and ids outside [0, n_vocab) are ignored. Pass nullptr and 0 when no penalties apply.
+// A verify window's rows need DIFFERENT histories: row t follows the window's drafts 1..t (`penalty_rows`).
+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, int* out, void* stream);
+
+// SPECULATIVE SAMPLING (Leviathan et al. 2023, Chen et al. 2023) for the pipelined NextN decode - opt-in
+// (STRATA_GLM_SPEC_SAMPLE=1). The draft is DRAWN from the draft block's own distribution q (this chain on its logits,
+// a Philox stream of its own) and its kept list is stored; the model's token is then the draft with probability
+// min(1, p(d) / q(d)), else a draw from the residual max(0, p - q) renormalised - so it is distributed exactly as the
+// plain draw from p, and a draft stands whenever the token is the draft (sum over x of min(p(x), q(x)) of the time
+// instead of p(argmax q)). Same (seed, counter): the same tokens - but not the plain draw's tokens.
+struct SpecList {
+ int n = 0; // the kept list's length (1..64)
+ int ids[64]; // descending, as the selection produced them
+ double p[64]; // q over the kept list (sums to 1)
+};
+
+// draft rows: logits (n_tokens, n_vocab) -> out[t] drawn through this chain, q[t] its kept list. DEVICE pointers,
+// like sample_tokens; a sampled request without penalties only (greedy keeps the argmax draft).
+void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,
+ void* stream);
+// verify rows: the model's logits (n_tokens, n_vocab), the draft draft[t] drawn from q[t] (same p.seed / p.counter as
+// its sample_draft) -> out[t], distributed as sample_tokens' draw. A draft q[t] does not hold gets the plain draw.
+void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,
+ const int* draft, int* out, void* stream);
+
+// The penalty-history rows of a verify window, on the host: row t of `out` (T rows of `h` slots) is the last `h`
+// tokens of `tail[0..n_tail)` followed by `window[0..t]`, most recent LAST, -1 in the unused front slots.
+// `tail` is what the state consumed before the window, `window[0]` the fed-back token and `window[1..]` the
+// drafts: row t is exactly the history plain decode counts when it picks the token after `window[t]`, so drafting
+// cannot change which tokens are penalised. Row 0 alone is the single row the engine staged before 0.1.19.
+inline void penalty_rows(const int32_t* tail, int64_t n_tail, const int32_t* window, int T, int h, int32_t* out) {
+ for (int t = 0; t < T; ++t) {
+ int32_t* row = out + (size_t) t * (size_t) h;
+ const int64_t avail = n_tail + t + 1; // tail + window[0..t]
+ const int take = (int) std::min<int64_t>(h, avail);
+ std::fill(row, row + (h - take), -1);
+ for (int j = 0; j < take; ++j) {
+ const int64_t i = avail - take + j; // index into (tail, window)
+ row[h - take + j] = i < n_tail ? tail[i] : window[i - n_tail];
+ }
+ }
+}
+
+} // namespace strata::kernels
diff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hpp
new file mode 100644
index 000000000..1e44c41fb
--- /dev/null
+++ b/src/dsv41/ds4_engram_io.hpp
@@ -0,0 +1,233 @@
+// src/ds4/ds4_engram_io.hpp - DeepSeek's Engram (n-gram memory) rows read off the decode's critical path.
+//
+// A token's Engram rows depend only on the token ids, so they are asked for as soon as the window's tokens are known
+// and collected only right before the layer that adds them (layer 1's hide behind the embedding and layer 0, layer
+// 14's behind layers 0-13). The reads run on the reader's own threads, each with its own handles to the model's files
+// (on Windows the ReadFile calls on one synchronous handle run one at a time), FIFO: the first Engram layer's rows
+// first. A bounded row cache (clock eviction) keeps rows already fetched: a row that recurs (the same n-gram again,
+// in the prompt or later) is a copy, not a read. The rows are the file's bytes either way, so the output is the same.
+#pragma once
+
+#include "ds4_os.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstring>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace strata::ds4 {
+
+inline int64_t now_ns() {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())
+ .count();
+}
+
+/// Rows already fetched: key -> `bytes` of payload, at most `cap` rows (grown on demand in blocks), clock eviction.
+/// Not thread-safe (the caller locks).
+class EgRowCache {
+public:
+ void init(size_t cap, size_t bytes) {
+ cap_ = cap;
+ bytes_ = bytes;
+ map_.reserve(std::min<size_t>(cap, 1u << 20));
+ }
+ size_t capacity() const { return cap_; }
+ size_t size() const { return keys_.size(); }
+ /// The row's payload, or nullptr (a hit marks it recently used).
+ const uint8_t* find(uint64_t key) {
+ const auto it = map_.find(key);
+ if (it == map_.end()) return nullptr;
+ ref_[it->second] = 1;
+ return at(it->second);
+ }
+ /// Room for key's payload (the caller fills it): a new slot while below capacity, else the clock's victim.
+ uint8_t* put(uint64_t key) {
+ if (cap_ == 0 || bytes_ == 0) return nullptr;
+ const auto it = map_.find(key);
+ if (it != map_.end()) return at(it->second);
+ uint32_t s;
+ if (keys_.size() < cap_) {
+ s = (uint32_t) keys_.size();
+ if (s % kBlock == 0) blocks_.emplace_back(new uint8_t[(size_t) kBlock * bytes_]);
+ keys_.push_back(key);
+ ref_.push_back(0);
+ } else {
+ while (ref_[hand_]) { // (a second chance for rows used since the hand last passed)
+ ref_[hand_] = 0;
+ hand_ = (hand_ + 1) % (uint32_t) cap_;
+ }
+ s = hand_;
+ hand_ = (hand_ + 1) % (uint32_t) cap_;
+ map_.erase(keys_[s]);
+ keys_[s] = key;
+ ref_[s] = 0;
+ }
+ map_.emplace(key, s);
+ return at(s);
+ }
+
+private:
+ static constexpr uint32_t kBlock = 4096;
+ uint8_t* at(uint32_t s) { return blocks_[s / kBlock].get() + (size_t) (s % kBlock) * bytes_; }
+ size_t cap_ = 0, bytes_ = 0;
+ uint32_t hand_ = 0;
+ std::unordered_map<uint64_t, uint32_t> map_;
+ std::vector<uint64_t> keys_;
+ std::vector<uint8_t> ref_;
+ std::vector<std::unique_ptr<uint8_t[]>> blocks_;
+};
+
+/// One row to fetch: its codes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks
+/// (a GGUF's MXFP8 row); else n code bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a
+/// GGUF's Q6_K row, its scales inside its n bytes).
+struct EgRead {
+ int file = -1;
+ uint64_t off = 0;
+ uint32_t n = 0;
+ uint8_t* dst = nullptr;
+ uint8_t* scales = nullptr;
+ bool mx8 = false;
+ int file2 = -1;
+ uint64_t off2 = 0;
+ uint32_t n2 = 0;
+ uint64_t key = 0;
+};
+
+/// A batch of rows (one Engram layer of a window): reads, the other places a row read once goes, and its state.
+struct EgBatch {
+ std::vector<EgRead> reads;
+ std::vector<std::pair<int, std::pair<uint8_t*, uint8_t*>>> dups; // read i -> another (codes, scales)
+ std::vector<uint8_t> ok; // per read: 1 read whole
+ std::atomic<int> left{0}; // reads not yet done
+ std::atomic<int> post{0}; // 0 pending, 1 finishing, 2 finished
+ int64_t t0 = 0; // submitted (now_ns())
+ std::atomic<int64_t> t1{0}; // the last read done
+ bool live = false; // submitted and not yet finished
+ void clear() {
+ reads.clear();
+ dups.clear();
+ ok.clear();
+ left.store(0);
+ post.store(0);
+ live = false;
+ }
+};
+
+/// The reader's threads: each read on whichever is free, in submission order, with that thread's own file handles.
+class EgReader {
+public:
+ ~EgReader() { stop(); }
+ void start(const std::vector<std::string>& paths, int threads) {
+ paths_ = paths;
+ for (int i = 0; i < threads; ++i) th_.emplace_back([this] { loop(); });
+ }
+ void stop() {
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ quit_ = true;
+ }
+ cv_.notify_all();
+ for (auto& t : th_) t.join();
+ th_.clear();
+ }
+ int threads() const { return (int) th_.size(); }
+ void submit(EgBatch& b) {
+ b.ok.assign(b.reads.size(), 0);
+ b.post.store(0);
+ b.t0 = now_ns();
+ b.t1.store(b.t0);
+ b.live = true;
+ b.left.store((int) b.reads.size(), std::memory_order_release);
+ const int n = (int) b.reads.size();
+ if (n == 0) return;
+ // a token's rows one job each; a prompt chunk's in runs (fewer trips through the queue's lock)
+ const int step = std::max(1, std::min(32, n / (int) (th_.size() * 4)));
+ int jobs = 0;
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ for (int i = 0; i < n; i += step, ++jobs) q_.push_back({&b, i, std::min(n, i + step)});
+ }
+ if (jobs >= (int) th_.size()) cv_.notify_all();
+ else
+ for (int i = 0; i < jobs; ++i) cv_.notify_one();
+ }
+ /// Until every read of b is done.
+ void wait(EgBatch& b) {
+ if (b.left.load(std::memory_order_acquire) == 0) return;
+ std::unique_lock<std::mutex> lk(done_mu_);
+ done_cv_.wait(lk, [&] { return b.left.load(std::memory_order_acquire) == 0; });
+ }
+
+private:
+ void loop() {
+ std::vector<os::fd_t> fds(paths_.size(), os::kBadFd); // this thread's own handles, opened on first use
+ std::vector<uint8_t> buf;
+ for (;;) {
+ Job job;
+ {
+ std::unique_lock<std::mutex> lk(mu_);
+ cv_.wait(lk, [&] { return quit_ || !q_.empty(); });
+ if (quit_ && q_.empty()) break;
+ job = q_.front();
+ q_.pop_front();
+ }
+ EgBatch& b = *job.b;
+ for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, b.reads[(size_t) i], buf) ? 1 : 0;
+ const int64_t t = now_ns();
+ for (int64_t prev = b.t1.load(); t > prev && !b.t1.compare_exchange_weak(prev, t);) {
+ }
+ if (b.left.fetch_sub(job.i1 - job.i0, std::memory_order_acq_rel) == job.i1 - job.i0) { // (b is the waiter's again)
+ std::lock_guard<std::mutex> lk(done_mu_);
+ done_cv_.notify_all();
+ }
+ }
+ for (os::fd_t f : fds) os::close_fd(f);
+ }
+ bool read(std::vector<os::fd_t>& fds, const EgRead& r, std::vector<uint8_t>& buf) {
+ auto fd = [&](int file) {
+ if (file < 0 || file >= (int) fds.size()) return os::kBadFd;
+ if (fds[(size_t) file] == os::kBadFd) fds[(size_t) file] = os::open_ro(paths_[(size_t) file]);
+ return fds[(size_t) file];
+ };
+ const os::fd_t f = fd(r.file);
+ if (f == os::kBadFd) return false;
+ if (!r.mx8) {
+ if (!os::pread_full(f, r.dst, r.n, r.off)) return false;
+ if (r.n2 == 0) return true;
+ const os::fd_t f2 = fd(r.file2);
+ return f2 != os::kBadFd && os::pread_full(f2, r.scales, r.n2, r.off2);
+ }
+ buf.resize(r.n);
+ if (!os::pread_full(f, buf.data(), r.n, r.off)) return false;
+ for (uint32_t k = 0; k < r.n / 33; ++k) { // [scale, 32 codes] -> codes | scales
+ r.scales[k] = buf[(size_t) k * 33];
+ std::memcpy(r.dst + (size_t) k * 32, buf.data() + (size_t) k * 33 + 1, 32);
+ }
+ return true;
+ }
+
+ std::vector<std::string> paths_;
+ std::vector<std::thread> th_;
+ std::mutex mu_;
+ std::condition_variable cv_;
+ struct Job {
+ EgBatch* b = nullptr;
+ int i0 = 0, i1 = 0; // reads [i0, i1)
+ };
+ std::deque<Job> q_;
+ bool quit_ = false;
+ std::mutex done_mu_;
+ std::condition_variable done_cv_;
+};
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
new file mode 100644
index 000000000..379d0766d
--- /dev/null
+++ b/src/dsv41/ds4_gguf.cu
@@ -0,0 +1,1461 @@
+// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).
+#include "ds4_gguf.cuh"
+
+#include "ds4_iq.cuh"
+#include "ds4_numerics.cuh"
+
+#include <cuda_fp16.h>
+#if !defined(STRATA_USE_HIP)
+#include <mma.h>
+#endif
+
+#include <algorithm>
+#include <cstdlib>
+#include <atomic>
+#include <cstdio>
+#include <cstring>
+#include <mutex>
+#include <unordered_map>
+
+namespace strata::ds4::gguf {
+namespace {
+
+std::atomic<int> g_errors{0};
+
+void check(const char* what) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "ds4 gguf %s: %s\n", what, cudaGetErrorString(e));
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+ }
+}
+void fail(const char* what) {
+ std::fprintf(stderr, "ds4 gguf %s\n", what);
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+}
+
+constexpr int kWarps = 8, kThreads = kWarps * 32;
+constexpr int kMaxEnt = 16; // a group's entries in one pass (a window's tokens at most)
+
+__device__ __forceinline__ float warp_sum(float v) {
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+ return v;
+}
+
+// ---------------------------------------------------------------- BF16 matrices
+struct BfJob {
+ const void* w;
+ size_t ld; // values a row (BF16), bytes a row (MXFP4)
+ const float* x;
+ size_t xcs;
+ float* y;
+ size_t ycs;
+ const float* bias;
+ float alpha;
+ int k, n, ncol;
+ const int8_t* xq; // (Q8_0: the activations rounded to int8, q8_quant's)
+ const float* xd;
+};
+struct BfJobs {
+ BfJob j[x3::kMaxMvJobs];
+};
+
+// grid (max rows / 8, jobs): a warp a row, each of its activations' dot in F32 - a lane 8 BF16 weights (16 bytes) and
+// 8 activations a step (VEC), or one of each (rows or activations not 16-byte aligned)
+template <int NC, bool VEC>
+__global__ void __launch_bounds__(kThreads) bf16_mv_kernel(const BfJobs J) {
+ const BfJob& b = J.j[blockIdx.y];
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (row >= b.n) return;
+ const uint16_t* w = (const uint16_t*) b.w + (size_t) row * b.ld;
+ float acc[NC];
+#pragma unroll
+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
+ if (VEC) {
+ for (int i = lane * 8; i < b.k; i += 256) {
+ const uint4 q = *(const uint4*) (w + i);
+ const float wf[8] = {__uint_as_float(q.x << 16), __uint_as_float(q.x & 0xffff0000u),
+ __uint_as_float(q.y << 16), __uint_as_float(q.y & 0xffff0000u),
+ __uint_as_float(q.z << 16), __uint_as_float(q.z & 0xffff0000u),
+ __uint_as_float(q.w << 16), __uint_as_float(q.w & 0xffff0000u)};
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float4* xp = (const float4*) (b.x + c * b.xcs + i);
+ const float4 x0 = __ldg(xp), x1 = __ldg(xp + 1);
+ float a = acc[c];
+ a = fmaf(wf[0], x0.x, a);
+ a = fmaf(wf[1], x0.y, a);
+ a = fmaf(wf[2], x0.z, a);
+ a = fmaf(wf[3], x0.w, a);
+ a = fmaf(wf[4], x1.x, a);
+ a = fmaf(wf[5], x1.y, a);
+ a = fmaf(wf[6], x1.z, a);
+ a = fmaf(wf[7], x1.w, a);
+ acc[c] = a;
+ }
+ }
+ } else {
+ for (int i = lane; i < b.k; i += 32) {
+ const float wf = __uint_as_float((uint32_t) w[i] << 16);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) acc[c] = fmaf(wf, b.x[c * b.xcs + i], acc[c]);
+ }
+ }
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float v = warp_sum(acc[c]);
+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+
+__global__ void q8_rows_kernel(const uint8_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,
+ bool bf16) {
+ const size_t total = (size_t) n * (size_t) k;
+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
+ const size_t r = i / (size_t) k, c = i - r * (size_t) k;
+ const uint8_t* p = w + r * ld + (c / 32) * kQ8Bytes;
+ const float v = __half2float(__ushort_as_half(*(const uint16_t*) p)) * (float) (int8_t) p[2 + c % 32];
+ dst[i] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));
+ }
+}
+
+__global__ void bf16_rows_kernel(const uint16_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,
+ bool bf16) {
+ const size_t total = (size_t) n * (size_t) k;
+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {
+ const size_t r = i / (size_t) k, c = i - r * (size_t) k;
+ const uint16_t b = w[r * ld + c];
+ dst[i] = bf16 ? b : __half_as_ushort(__float2half_rn(__uint_as_float((uint32_t) b << 16)));
+ }
+}
+
+// ---------------------------------------------------------------- MXFP4 experts
+__device__ __forceinline__ float e8m0_half(uint32_t e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF
+ return __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);
+}
+__device__ __forceinline__ void lut_init(float* lut) {
+ if (threadIdx.x < 16) {
+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};
+ lut[threadIdx.x] = threadIdx.x < 8 ? v[threadIdx.x] : -v[threadIdx.x - 8];
+ }
+}
+// block p's 32 values, exactly (a code times a power of two)
+__device__ __forceinline__ void mx_block(const uint8_t* __restrict__ p, const float* lut, float* w) {
+ const float d = e8m0_half(p[0]);
+#pragma unroll
+ for (int j = 0; j < 16; ++j) {
+ const uint32_t q = p[1 + j];
+ w[j] = lut[q & 15u] * d;
+ w[j + 16] = lut[q >> 4] * d;
+ }
+}
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
+ if (limit > 0.0f) {
+ gv = fminf(gv, limit);
+ uv = fminf(fmaxf(uv, -limit), limit);
+ }
+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
+ float m = 0.0f;
+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
+ return __fadd_rn(m, sh);
+}
+
+// a Q8_0 block's 32 values, exactly (an FP16 scale times an int8; the block is only 2-byte aligned)
+__device__ __forceinline__ void q8_block(const uint8_t* __restrict__ p, float* w) {
+ const float d = __half2float(__ushort_as_half(*(const uint16_t*) p));
+ const uint16_t* q = (const uint16_t*) (p + 2);
+#pragma unroll
+ for (int j = 0; j < 16; ++j) {
+ const uint16_t v = q[j];
+ w[2 * j] = d * (float) (int8_t) (v & 0xff);
+ w[2 * j + 1] = d * (float) (int8_t) (v >> 8);
+ }
+}
+
+// ---- int8 activations (llama.cpp's MMVQ: its q8_1 blocks' rounding)
+// rows of k F32 values (row r at x + r ldx; k % 32 == 0) -> q[r][k] int8 and d[r][k / 32] = max|x| / 127 per 32
+__global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restrict__ x, size_t ldx, int k,
+ int8_t* __restrict__ q, float* __restrict__ d) {
+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (b * 32 >= k) return;
+ const float v = x[(size_t) r * ldx + (size_t) b * 32 + lane];
+ float am = fabsf(v);
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+ const float sc = am / 127.0f;
+ q[(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
+ if (lane == 0) d[(size_t) r * (k / 32) + b] = sc;
+}
+void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {
+ const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);
+ q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);
+}
+
+// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),
+// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4
+__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {
+ const uint32_t t0 = 0x03020100u, t1 = 0x0C080604u, t2 = 0xFDFEFF00u, t3 = 0xF4F8FAFCu;
+ const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
+ uint32_t tmp[2];
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ const uint32_t idx = (q4 >> (16 * i)) & 0x7777u;
+ const uint32_t lo = __byte_perm(t0, t1, idx), hi = __byte_perm(t2, t3, idx);
+ tmp[i] = __byte_perm(lo, hi, sel >> (16 * i));
+ }
+ return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));
+}
+// an MXFP4 block's codes (4 words) against 32 int8 activations (xa: values 0..15, xb: 16..31)
+__device__ __forceinline__ int mx_dot(const uint4& q, const int4& xa, const int4& xb) {
+ int sum = 0;
+ int2 v = e2m1_x8(q.x);
+ sum = __dp4a(v.x, xa.x, sum);
+ sum = __dp4a(v.y, xb.x, sum);
+ v = e2m1_x8(q.y);
+ sum = __dp4a(v.x, xa.y, sum);
+ sum = __dp4a(v.y, xb.y, sum);
+ v = e2m1_x8(q.z);
+ sum = __dp4a(v.x, xa.z, sum);
+ sum = __dp4a(v.y, xb.z, sum);
+ v = e2m1_x8(q.w);
+ sum = __dp4a(v.x, xa.w, sum);
+ sum = __dp4a(v.y, xb.w, sum);
+ return sum;
+}
+// the warp copies the blocks [b0, b0 + nbk) of a row (BB bytes each; the row 4-byte aligned, nbk BB % 4 == 0) into its
+// stage in 4-byte words; every lane calls it
+template <int BB>
+__device__ __forceinline__ void stage_chunk(const uint8_t* row, int b0, int nbk, uint32_t* st) {
+ const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);
+ __syncwarp(); // (the lanes are done with the chunk before)
+ for (int i = threadIdx.x & 31; i < nbk * BB / 4; i += 32) st[i] = src[i];
+ __syncwarp();
+}
+// a row's next chunk of 32 blocks (BB bytes each) in the lanes' registers, then into the warp's stage
+template <int BB>
+struct Fetch {
+ static constexpr int W = (32 * BB / 4 + 31) / 32; // words a lane
+ uint32_t r[W];
+ int n = 0;
+ __device__ __forceinline__ void load(const uint8_t* row, int b0, int nbk) {
+ const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);
+ n = nbk * BB / 4;
+#pragma unroll
+ for (int j = 0; j < W; ++j) {
+ const int i = (int) (threadIdx.x & 31) + 32 * j;
+ r[j] = i < n ? src[i] : 0u;
+ }
+ }
+ __device__ __forceinline__ void store(uint32_t* st) const {
+ __syncwarp(); // (the lanes are done with the chunk before)
+#pragma unroll
+ for (int j = 0; j < W; ++j) {
+ const int i = (int) (threadIdx.x & 31) + 32 * j;
+ if (i < n) st[i] = r[j];
+ }
+ __syncwarp();
+ }
+};
+
+// MXFP4 block i of a stage (17-byte blocks from a 4-byte aligned start): its scale byte, its 16 code bytes as 4 words
+__device__ __forceinline__ uint32_t mx_words_s(const uint32_t* st, int i, uint4& q) {
+ const int o = 17 * i, w = o >> 2, sh = (o & 3) * 8;
+ const uint32_t a0 = st[w], a1 = st[w + 1], a2 = st[w + 2], a3 = st[w + 3], a4 = st[w + 4];
+ const uint32_t s0 = __funnelshift_r(a0, a1, sh), s1 = __funnelshift_r(a1, a2, sh), s2 = __funnelshift_r(a2, a3, sh),
+ s3 = __funnelshift_r(a3, a4, sh), s4 = a4 >> sh;
+ q.x = __funnelshift_r(s0, s1, 8);
+ q.y = __funnelshift_r(s1, s2, 8);
+ q.z = __funnelshift_r(s2, s3, 8);
+ q.w = __funnelshift_r(s3, s4, 8);
+ return s0 & 0xffu;
+}
+// ... the same from global memory, byte by byte (rows not 4-byte aligned)
+__device__ __forceinline__ uint32_t mx_words_g(const uint8_t* __restrict__ p, uint4& q) {
+ uint32_t w[4];
+#pragma unroll
+ for (int j = 0; j < 4; ++j)
+ w[j] = (uint32_t) p[1 + 4 * j] | (uint32_t) p[2 + 4 * j] << 8 | (uint32_t) p[3 + 4 * j] << 16 |
+ (uint32_t) p[4 + 4 * j] << 24;
+ q = make_uint4(w[0], w[1], w[2], w[3]);
+ return p[0];
+}
+// Q8_0 block i of a stage (34-byte blocks): its scale, its 32 int8 as 8 words
+__device__ __forceinline__ float q8_words_s(const uint32_t* st, int i, int* v) {
+ const int o = 34 * i, w = o >> 2, sh = (o & 3) * 8;
+ uint32_t a[9];
+#pragma unroll
+ for (int j = 0; j < 9; ++j) a[j] = st[w + j];
+ uint32_t sw[9];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) sw[j] = __funnelshift_r(a[j], a[j + 1], sh);
+ sw[8] = a[8] >> sh;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) v[j] = (int) __funnelshift_r(sw[j], sw[j + 1], 16);
+ return __half2float(__ushort_as_half((uint16_t) (sw[0] & 0xffffu)));
+}
+__device__ __forceinline__ float q8_words_g(const uint8_t* __restrict__ p, int* v) {
+#pragma unroll
+ for (int j = 0; j < 8; ++j)
+ v[j] = (int) ((uint32_t) p[2 + 4 * j] | (uint32_t) p[3 + 4 * j] << 8 | (uint32_t) p[4 + 4 * j] << 16 |
+ (uint32_t) p[5 + 4 * j] << 24);
+ return __half2float(__ushort_as_half((uint16_t) (p[0] | p[1] << 8)));
+}
+
+// grid (max rows / 8, jobs): a Q8_0 matrix's rows against the jobs' int8 activations (q8_quant's: activation c of job
+// j at xq + c k, its scales at xd + c k / 32) - a warp a row, a lane a block a step, the blocks staged (ST) or read
+template <int NC, bool ST>
+__global__ void __launch_bounds__(kThreads) q8_mv_kernel(const BfJobs J) {
+ __shared__ __align__(16) uint32_t stage[kWarps][32 * kQ8Bytes / 4];
+ const BfJob& b = J.j[blockIdx.y];
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (row >= b.n) return;
+ const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;
+ const int nb = b.k / 32;
+ float acc[NC];
+#pragma unroll
+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
+ Fetch<kQ8Bytes> F;
+ if (ST) F.load(w, 0, min(32, nb));
+ for (int b0 = 0; b0 < nb; b0 += 32) {
+ const int nbk = min(32, nb - b0), kb = b0 + lane;
+ if (ST) {
+ F.store(stage[threadIdx.x >> 5]);
+ if (b0 + 32 < nb) F.load(w, b0 + 32, min(32, nb - b0 - 32));
+ }
+ if (lane >= nbk) continue;
+ int v[8];
+ const float dw = ST ? q8_words_s(stage[threadIdx.x >> 5], lane, v) : q8_words_g(w + (size_t) kb * kQ8Bytes, v);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);
+ const int4 xa = xp[0], xb = xp[1];
+ int sum = __dp4a(v[0], xa.x, 0);
+ sum = __dp4a(v[1], xa.y, sum);
+ sum = __dp4a(v[2], xa.z, sum);
+ sum = __dp4a(v[3], xa.w, sum);
+ sum = __dp4a(v[4], xb.x, sum);
+ sum = __dp4a(v[5], xb.y, sum);
+ sum = __dp4a(v[6], xb.z, sum);
+ sum = __dp4a(v[7], xb.w, sum);
+ acc[c] = fmaf(dw * b.xd[(size_t) c * nb + kb], (float) sum, acc[c]);
+ }
+ }
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float v = warp_sum(acc[c]);
+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// grid (max rows / 8, jobs): bf16_mv_kernel for MXFP4 (Q8 false) or Q8_0 (Q8 true) matrices - a lane a 32-value block
+// a step, dequantised once for the job's activations
+template <int NC, bool VEC, bool Q8>
+__global__ void __launch_bounds__(kThreads) mx_mv_kernel(const BfJobs J) {
+ __shared__ float lut[16];
+ lut_init(lut);
+ __syncthreads();
+ const BfJob& b = J.j[blockIdx.y];
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (row >= b.n) return;
+ const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;
+ float acc[NC];
+#pragma unroll
+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (int kb = lane; kb < b.k / kMxBlock; kb += 32) {
+ float wv[32];
+ if (Q8) q8_block(w + (size_t) kb * kQ8Bytes, wv);
+ else mx_block(w + (size_t) kb * kMxBytes, lut, wv);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float* x = b.x + c * b.xcs + kb * kMxBlock;
+ float a = acc[c];
+ if (VEC) {
+#pragma unroll
+ for (int q = 0; q < 8; ++q) {
+ const float4 xv = __ldg((const float4*) x + q);
+ a = fmaf(wv[4 * q], xv.x, a);
+ a = fmaf(wv[4 * q + 1], xv.y, a);
+ a = fmaf(wv[4 * q + 2], xv.z, a);
+ a = fmaf(wv[4 * q + 3], xv.w, a);
+ }
+ } else {
+#pragma unroll
+ for (int j = 0; j < 32; ++j) a = fmaf(wv[j], x[j], a);
+ }
+ acc[c] = a;
+ }
+ }
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float v = warp_sum(acc[c]);
+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// a token's experts (grp_ptr null: group i = expert i of the plan, plan_ptr[i] its blob or 0 off the device, its one
+// entry i, every entry's input x) or a window's groups (group g: one expert, blob grp_ptr[g], entries e = t * 8 + i at
+// grp_ent[grp_start[g] ..), entry e's input x + (e >> 3) x_stride)
+struct MxArgs {
+ const unsigned long long* plan_ptr;
+ const unsigned long long* grp_ptr;
+ const int *grp_start, *grp_ent, *n_groups;
+ const float* x;
+ size_t x_stride;
+ float* h; // [entries][n_ff]
+ float* part; // [entries][n_embd]
+ size_t tr[3];
+ int n_embd, n_ff;
+ float limit;
+ const int8_t* xq; // x rounded to int8 (a token's: row 0; a window's: row t), q8_quant's rows of n_embd
+ const float* xd;
+ const int8_t* hq; // h rounded to int8, row e = the entry, rows of n_ff
+ const float* hd;
+};
+
+__device__ __forceinline__ bool mx_group(const MxArgs& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
+ if (a.grp_ptr == nullptr) {
+ const unsigned long long p = a.plan_ptr[grp];
+ if (p == 0ull) return false;
+ blob = (const uint8_t*) p;
+ m0 = grp;
+ m1 = grp + 1;
+ return true;
+ }
+ if (grp >= *a.n_groups) return false;
+ blob = (const uint8_t*) a.grp_ptr[grp];
+ m0 = a.grp_start[grp];
+ m1 = a.grp_start[grp + 1];
+ return true;
+}
+
+// grid (n_ff / 8, groups): a warp a row of the gate and of the up, for NE of the group's entries a pass (each weight
+// block decoded once for them) against the entries' int8 x -> h = the clamped swiglu
+template <int NE, bool ST>
+__global__ void __launch_bounds__(kThreads) mx_gate_up_kernel(const MxArgs a) {
+ __shared__ __align__(16) uint32_t stage[kWarps][2][32 * kMxBytes / 4];
+ __shared__ int es[NE], xr[NE];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!mx_group(a, blockIdx.y, blob, m0, m1)) return; // (uniform)
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ const size_t rb = mx_row_bytes(a.n_embd);
+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
+ const int nb = a.n_embd / kMxBlock;
+ for (int mm = m0; mm < m1; mm += NE) {
+ const int ne = min(NE, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) {
+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ es[threadIdx.x] = e;
+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
+ }
+ __syncthreads();
+ if (row >= a.n_ff) continue;
+ float ag[NE], au[NE];
+#pragma unroll
+ for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;
+ Fetch<kMxBytes> FG, FU;
+ if (ST) {
+ FG.load(g, 0, min(32, nb));
+ FU.load(u, 0, min(32, nb));
+ }
+ for (int b0 = 0; b0 < nb; b0 += 32) {
+ const int nbk = min(32, nb - b0), b = b0 + lane;
+ if (ST) {
+ FG.store(stage[wp][0]);
+ FU.store(stage[wp][1]);
+ if (b0 + 32 < nb) {
+ FG.load(g, b0 + 32, min(32, nb - b0 - 32));
+ FU.load(u, b0 + 32, min(32, nb - b0 - 32));
+ }
+ }
+ if (lane >= nbk) continue;
+ uint4 qg, qu;
+ const uint32_t eg = ST ? mx_words_s(stage[wp][0], lane, qg) : mx_words_g(g + (size_t) b * kMxBytes, qg);
+ const uint32_t eu = ST ? mx_words_s(stage[wp][1], lane, qu) : mx_words_g(u + (size_t) b * kMxBytes, qu);
+ const float dg = e8m0_half(eg), du = e8m0_half(eu);
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) b * kMxBlock);
+ const int4 xa = xp[0], xb = xp[1];
+ const float dx = a.xd[(size_t) xr[i] * nb + b];
+ ag[i] = fmaf(dg * dx, (float) mx_dot(qg, xa, xb), ag[i]);
+ au[i] = fmaf(du * dx, (float) mx_dot(qu, xa, xb), au[i]);
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const float gv = warp_sum(ag[i]), uv = warp_sum(au[i]);
+ if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);
+ }
+ }
+}
+
+// grid (n_embd / 8, groups): a warp a row of the down for NE entries a pass, against their int8 h -> part
+template <int NE, bool ST>
+__global__ void __launch_bounds__(kThreads) mx_down_kernel(const MxArgs a) {
+ __shared__ __align__(16) uint32_t stage[kWarps][32 * kMxBytes / 4];
+ __shared__ int es[NE];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!mx_group(a, blockIdx.y, blob, m0, m1)) return;
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ const uint8_t* d = blob + a.tr[2] + (size_t) row * mx_row_bytes(a.n_ff);
+ const int nb = a.n_ff / kMxBlock;
+ for (int mm = m0; mm < m1; mm += NE) {
+ const int ne = min(NE, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ __syncthreads();
+ if (row >= a.n_embd) continue;
+ float ad[NE];
+#pragma unroll
+ for (int i = 0; i < NE; ++i) ad[i] = 0.0f;
+ Fetch<kMxBytes> FD;
+ if (ST) FD.load(d, 0, min(32, nb));
+ for (int b0 = 0; b0 < nb; b0 += 32) {
+ const int nbk = min(32, nb - b0), b = b0 + lane;
+ if (ST) {
+ FD.store(stage[wp]);
+ if (b0 + 32 < nb) FD.load(d, b0 + 32, min(32, nb - b0 - 32));
+ }
+ if (lane >= nbk) continue;
+ uint4 q;
+ const float dd = e8m0_half(ST ? mx_words_s(stage[wp], lane, q) : mx_words_g(d + (size_t) b * kMxBytes, q));
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) b * kMxBlock);
+ const int4 ha = hp[0], hb = hp[1];
+ ad[i] = fmaf(dd * a.hd[(size_t) es[i] * nb + b], (float) mx_dot(q, ha, hb), ad[i]);
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const float v = warp_sum(ad[i]);
+ if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;
+ }
+ }
+}
+
+struct CombArgs {
+ const unsigned long long* plan_ptr; // [T][8]
+ const float* plan_w; // [T][8]
+ const float* part; // [T * 8][n_embd]
+ const float* cpu_part;
+ size_t cpu_stride;
+ const int* cpu_flag;
+ const float* sh_out; // [T][n_embd]
+ float* out;
+ size_t out_stride;
+ int k, n_embd;
+};
+// grid (n_embd / 128, T): each token's outputs - the experts in plan order (an expert off the device adds 0), then
+// the CPU lane's part and the shared expert's (x3's combine)
+__global__ void __launch_bounds__(128) mx_combine_kernel(const CombArgs a) {
+ const int t = blockIdx.y, r = blockIdx.x * 128 + threadIdx.x;
+ if (r >= a.n_embd) return;
+ float p[8];
+ for (int i = 0; i < a.k; ++i) {
+ const int e = t * 8 + i;
+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
+ }
+ a.out[(size_t) t * a.out_stride + r] =
+ combine(a.plan_w + t * 8, p, a.k,
+ a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,
+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
+}
+
+__global__ void mx_wait_kernel(const unsigned* p, unsigned v) {
+ if (threadIdx.x != 0) return;
+ while (*(volatile const unsigned*) p != v) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+ __nanosleep(200);
+#endif
+ }
+ __threadfence_system();
+}
+
+// ---------------------------------------------------------------- the prompt path's expert sets
+// a set's expert i (blob wbase + i stride) owns the sorted rows [bounds[i], bounds[i + 1]); local row = sorted row -
+// row0. A block: BM of its rows x BN outputs, the weights dequantised a 32-value block at a time into shared memory,
+// F32 FMAs (each thread TM x TN outputs)
+constexpr int BM = 64, BN = 64, TM = 4, TN = 4;
+static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");
+
+struct SetK {
+ const uint8_t* wbase;
+ size_t stride;
+ size_t tr[3];
+ const int* bounds;
+ int row0;
+ int n_embd, n_ff;
+ float limit;
+ const float* xf; // gate / up: F32 input rows x[idx[local]] (ldx)
+ const int* idx;
+ const __half* xh; // down: F16 rows (ldx)
+ int ldx;
+ __half* h; // gate / up: F16 out
+ float* y; // down: F32 out
+ int ldo;
+ int skip_le; // the experts of at most this many rows left out (the light kernels took them; ds4_iq.cuh)
+};
+
+// one 32-value block of BN weight rows (rows c0 .., row bytes rb, block kb) -> ws[32][BN + 1]
+__device__ __forceinline__ void mx_tile(const uint8_t* w, size_t rb, int c0, int kb, const float* lut,
+ float (*ws)[BN + 1]) {
+ const int r = threadIdx.x >> 2, q = threadIdx.x & 3; // row r of the tile, values [8 q, 8 q + 8)
+ const uint8_t* p = w + (size_t) (c0 + r) * rb + (size_t) kb * kMxBytes;
+ const float d = e8m0_half(p[0]);
+ const int b0 = (q & 1) * 8, hi = q >> 1;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const uint32_t c = p[1 + b0 + j];
+ ws[q * 8 + j][r] = lut[hi ? (c >> 4) : (c & 15u)] * d;
+ }
+}
+
+template <bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) mx_set_kernel(const SetK a) {
+ __shared__ float lut[16];
+ __shared__ float wg[32][BN + 1];
+ __shared__ float wu[GATE_UP ? 32 : 1][BN + 1];
+ __shared__ float xs[BM][33];
+ __shared__ int src[BM];
+ lut_init(lut);
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform: before any barrier but lut_init's writes, which nobody
+ // reads then; skip_le: the light kernels' experts)
+ const int nr = min(BM, hi - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GATE_UP ? a.n_embd : a.n_ff;
+ const size_t rb = mx_row_bytes(K);
+ const int tid = (int) threadIdx.x;
+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
+ __syncthreads();
+ const int ty = threadIdx.x / (BN / TN), tx = threadIdx.x % (BN / TN);
+ float ag[TM][TN], au[TM][TN];
+#pragma unroll
+ for (int i = 0; i < TM; ++i)
+#pragma unroll
+ for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;
+ for (int kb = 0; kb < K / kMxBlock; ++kb) {
+ mx_tile(blob + a.tr[GATE_UP ? 0 : 2], rb, c0, kb, lut, wg);
+ if (GATE_UP) mx_tile(blob + a.tr[1], rb, c0, kb, lut, wu);
+ for (int i = threadIdx.x; i < BM * kMxBlock; i += kThreads) { // the rows' 32 inputs
+ const int r = i / kMxBlock, c = i % kMxBlock, sr = src[r];
+ float v = 0.0f;
+ if (sr >= 0) {
+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + c;
+ v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);
+ }
+ xs[r][c] = v;
+ }
+ __syncthreads();
+#pragma unroll 8
+ for (int kk = 0; kk < kMxBlock; ++kk) {
+ float xv[TM], gw[TN], uw[TN];
+#pragma unroll
+ for (int i = 0; i < TM; ++i) xv[i] = xs[ty * TM + i][kk];
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ gw[j] = wg[kk][tx * TN + j];
+ if (GATE_UP) uw[j] = wu[kk][tx * TN + j];
+ }
+#pragma unroll
+ for (int i = 0; i < TM; ++i)
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);
+ if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);
+ }
+ }
+ __syncthreads();
+ }
+#pragma unroll
+ for (int i = 0; i < TM; ++i) {
+ const int r = ty * TM + i;
+ if (r >= nr) continue;
+ const size_t lr = (size_t) (r0 + r);
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ const int c = c0 + tx * TN + j;
+ if (GATE_UP) a.h[lr * a.ldo + c] = __float2half_rn(swiglu(ag[i][j], au[i][j], a.limit));
+ else a.y[lr * a.ldo + c] = ag[i][j];
+ }
+ }
+}
+
+// the same on the tensor cores: a block BM rows x BN outputs, 8 warps each 16 rows x 32 outputs (2 fragments, gate and
+// up: 4), a 32-value MXFP4 block of K a step (2 WMMA k-steps), the tiles in shared memory as F16
+#if !defined(STRATA_USE_HIP)
+namespace wm = nvcuda::wmma;
+constexpr int kLdT = 40; // halves a tile row (32 + 8: WMMA wants a multiple of 8; the pad spreads the banks)
+template <bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) mx_set_tc_kernel(const SetK a) {
+ struct Tiles {
+ __half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];
+ };
+ struct Out {
+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
+ };
+ __shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
+ __shared__ float lut[16];
+ __shared__ int src[BM];
+ Tiles& T = *reinterpret_cast<Tiles*>(smem);
+ Out& O = *reinterpret_cast<Out*>(smem);
+ lut_init(lut);
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
+ const int nr = min(BM, hi - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GATE_UP ? a.n_embd : a.n_ff;
+ const size_t rb = mx_row_bytes(K);
+ const int tid = (int) threadIdx.x, warp = tid >> 5;
+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
+ __syncthreads();
+ const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the warp's 16 rows, its 2 x 16 outputs
+ wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];
+ for (int j = 0; j < 2; ++j) {
+ wm::fill_fragment(fg[j], 0.0f);
+ wm::fill_fragment(fu[j], 0.0f);
+ }
+ const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)
+ for (int kb = 0; kb < K / kMxBlock; ++kb) {
+ { // the weights: row tr of the outputs, values 8 tq ..
+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
+ const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;
+ const float d = e8m0_half(q[0]);
+ const int b0 = (tq & 1) * 8, hn = tq >> 1;
+ __half* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const uint32_t c = q[1 + b0 + j];
+ dst[j] = __float2half_rn(lut[hn ? (c >> 4) : (c & 15u)] * d);
+ }
+ }
+ // the inputs: row tr, values 8 tq ..
+ const int sr = src[tr];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ __half v = __float2half_rn(0.0f);
+ if (sr >= 0) {
+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;
+ v = GATE_UP ? __float2half_rn(a.xf[at]) : a.xh[at];
+ }
+ T.x[tr][tq * 8 + j] = v;
+ }
+ }
+ __syncthreads();
+#pragma unroll
+ for (int kk = 0; kk < 32; kk += 16) {
+ wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;
+ wm::load_matrix_sync(fa, &T.x[wm_ * 16][kk], kLdT);
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;
+ wm::load_matrix_sync(fb, &T.wg[(wn + j) * 16][kk], kLdT);
+ wm::mma_sync(fg[j], fa, fb, fg[j]);
+ if (GATE_UP) {
+ wm::load_matrix_sync(fb, &T.wu[(wn + j) * 16][kk], kLdT);
+ wm::mma_sync(fu[j], fa, fb, fu[j]);
+ }
+ }
+ }
+ __syncthreads();
+ }
+ for (int j = 0; j < 2; ++j) {
+ wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);
+ if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);
+ }
+ __syncthreads();
+ for (int i = tid; i < BM * BN; i += kThreads) {
+ const int r = i / BN, c = i % BN;
+ if (r >= nr) continue;
+ const size_t lr = (size_t) (r0 + r);
+ if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit));
+ else a.y[lr * a.ldo + c0 + c] = O.g[r][c];
+ }
+}
+#endif
+#if defined(STRATA_USE_HIP)
+// the same on an AMD wave32 GPU's WMMA (RDNA3 / RDNA3.5 gfx11, RDNA4 gfx12): 8 waves each 16 rows x 32 outputs, the
+// tiles in shared memory as F16, F32 accumulation - each lane gives its row / column's 16 values of a k-step (gfx12:
+// its half of them), and holds 8 of the 16 x 16 results (rows 2 i + hi on gfx11, 8 hi + i on gfx12; column lane % 16)
+using h16v = _Float16 __attribute__((ext_vector_type(16)));
+using f8v = float __attribute__((ext_vector_type(8)));
+__device__ __forceinline__ f8v set_wmma(h16v a, h16v b, f8v c) {
+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \
+ defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__)
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#elif defined(__gfx1200__) || defined(__gfx1201__)
+ using h8 = _Float16 __attribute__((ext_vector_type(8)));
+ const int k0 = (threadIdx.x & 31) / 16 * 8;
+ h8 aa, bb;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ aa[i] = a[k0 + i];
+ bb[i] = b[k0 + i];
+ }
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(aa, bb, c);
+#else
+ __builtin_trap();
+ return c;
+#endif
+}
+__device__ __forceinline__ int set_wmma_row(int i, int hi) {
+#if defined(__gfx1200__) || defined(__gfx1201__)
+ return 8 * hi + i;
+#else
+ return 2 * i + hi;
+#endif
+}
+constexpr int kLdH = 40; // halves a tile row (32 + 8: the pad spreads the banks)
+template <bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) mx_set_wmma_kernel(const SetK a) {
+ struct Tiles {
+ _Float16 x[BM][kLdH], wg[BN][kLdH], wu[GATE_UP ? BN : 1][kLdH];
+ };
+ struct Out {
+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
+ };
+ __shared__ __align__(16) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
+ __shared__ float lut[16];
+ __shared__ int src[BM];
+ Tiles& T = *reinterpret_cast<Tiles*>(smem);
+ Out& O = *reinterpret_cast<Out*>(smem);
+ lut_init(lut);
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
+ const int nr = min(BM, hi_r - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GATE_UP ? a.n_embd : a.n_ff;
+ const size_t rb = mx_row_bytes(K);
+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
+ __syncthreads();
+ const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the wave's 16 rows, its 2 x 16 outputs
+ f8v fg[2], fu[2];
+#pragma unroll
+ for (int j = 0; j < 2; ++j)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) fg[j][i] = fu[j][i] = 0.0f;
+ const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)
+ for (int kb = 0; kb < K / kMxBlock; ++kb) {
+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
+ const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;
+ const float d = e8m0_half(q[0]);
+ const int b0 = (tq & 1) * 8, hn = tq >> 1;
+ _Float16* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const uint32_t cc = q[1 + b0 + j];
+ dst[j] = (_Float16) (lut[hn ? (cc >> 4) : (cc & 15u)] * d);
+ }
+ }
+ const int sr = src[tr];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ float v = 0.0f;
+ if (sr >= 0) {
+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;
+ v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);
+ }
+ T.x[tr][tq * 8 + j] = (_Float16) v;
+ }
+ __syncthreads();
+#pragma unroll
+ for (int kk = 0; kk < 32; kk += 16) {
+ h16v av;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) av[i] = T.x[wm_ * 16 + l16][kk + i];
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ h16v bv;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) bv[i] = T.wg[(wn + j) * 16 + l16][kk + i];
+ fg[j] = set_wmma(av, bv, fg[j]);
+ if (GATE_UP) {
+#pragma unroll
+ for (int i = 0; i < 16; ++i) bv[i] = T.wu[(wn + j) * 16 + l16][kk + i];
+ fu[j] = set_wmma(av, bv, fu[j]);
+ }
+ }
+ }
+ __syncthreads();
+ }
+#pragma unroll
+ for (int j = 0; j < 2; ++j)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) {
+ const int r = wm_ * 16 + set_wmma_row(i, hi), cc = (wn + j) * 16 + l16;
+ O.g[r][cc] = fg[j][i];
+ if (GATE_UP) O.u[r][cc] = fu[j][i];
+ }
+ __syncthreads();
+ for (int i = tid; i < BM * BN; i += kThreads) {
+ const int r = i / BN, cc = i % BN;
+ if (r >= nr) continue;
+ const size_t lr = (size_t) (r0 + r);
+ if (GATE_UP) a.h[lr * a.ldo + c0 + cc] = __float2half_rn(swiglu(O.g[r][cc], O.u[r][cc], a.limit));
+ else a.y[lr * a.ldo + c0 + cc] = O.g[r][cc];
+ }
+}
+#endif
+bool tc_ok() { // sm_70 and later (WMMA F16) / AMD gfx11, gfx12 (wave32 WMMA); STRATA_DS4_NO_TC=1 keeps the F32 kernel
+#if defined(STRATA_USE_HIP)
+ static const bool ok = [] {
+ int dev = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceProp prop{};
+ if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+ const char* v = std::getenv("STRATA_DS4_NO_TC");
+ if (v != nullptr && v[0] == '1') return false;
+ const char* arch = prop.gcnArchName;
+ for (const char* p : {"gfx1100", "gfx1101", "gfx1102", "gfx1103", "gfx1150", "gfx1151", "gfx1152", "gfx1200", "gfx1201"})
+ if (std::strncmp(arch, p, 7) == 0) return true;
+ return false;
+ }();
+ return ok;
+#else
+ static const bool ok = [] {
+ int dev = 0, major = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ const char* v = std::getenv("STRATA_DS4_NO_TC");
+ return major >= 7 && !(v != nullptr && v[0] == '1');
+ }();
+ return ok;
+#endif
+}
+
+// ---------------------------------------------------------------- scratch
+constexpr int kMaxEmbd = 8192;
+constexpr int kMaxQK = 32768; // a Q8_0 GEMV's inputs at most
+struct Scratch {
+ float* part = nullptr; // [8][kMaxEmbd]: a token's experts' down outputs
+ int8_t *xq = nullptr, *hq = nullptr, *mq = nullptr; // int8 rows: x [kMaxEnt], h [kMaxEnt * 8], the GEMVs'
+ float *xd = nullptr, *hd = nullptr, *md = nullptr; // [kMaxMvJobs * kMaxMvCols] inputs; their scales
+};
+std::mutex g_mu;
+std::unordered_map<cudaStream_t, Scratch> g_scratch;
+
+Scratch* scratch(cudaStream_t s) {
+ std::lock_guard<std::mutex> lk(g_mu);
+ auto it = g_scratch.find(s);
+ if (it != g_scratch.end()) return &it->second;
+ Scratch sc;
+ const size_t nx = (size_t) kMaxEnt * kMaxEmbd, nh = (size_t) kMaxEnt * 8 * kMaxEmbd,
+ nm = (size_t) x3::kMaxMvJobs * x3::kMaxMvCols * kMaxQK;
+ if (cudaMalloc(&sc.part, (size_t) 8 * kMaxEmbd * sizeof(float)) != cudaSuccess ||
+ cudaMalloc(&sc.xq, nx) != cudaSuccess || cudaMalloc(&sc.xd, nx / 32 * 4) != cudaSuccess ||
+ cudaMalloc(&sc.hq, nh) != cudaSuccess || cudaMalloc(&sc.hd, nh / 32 * 4) != cudaSuccess ||
+ cudaMalloc(&sc.mq, nm) != cudaSuccess || cudaMalloc(&sc.md, nm / 32 * 4) != cudaSuccess) {
+ fail("scratch: cudaMalloc failed");
+ return nullptr;
+ }
+ return &g_scratch.emplace(s, sc).first->second;
+}
+
+bool mx_geometry(const char* who, int k, int n_embd, int n_ff) {
+ if (k < 1 || k > 8 || n_embd % BN != 0 || n_ff % BN != 0 || n_embd > kMaxEmbd || n_ff % kMxBlock != 0) {
+ std::fprintf(stderr, "ds4 gguf %s: geometry not covered (k %d n_embd %d n_ff %d)\n", who, k, n_embd, n_ff);
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+ return false;
+ }
+ return true;
+}
+
+} // namespace
+
+// ---------------------------------------------------------------- entry points
+bool prepare(cudaStream_t s) { return scratch(s) != nullptr; }
+
+int launch_errors() { return g_errors.load(std::memory_order_relaxed); }
+
+size_t type_row_bytes(int type, int64_t k) {
+ if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;
+ return iq::supported(type) && k % 256 == 0 ? iq::row_bytes(type, k) : 0;
+}
+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? 256 : 0; }
+const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }
+
+namespace {
+size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row
+ if (m.ld > 0) return (size_t) m.ld;
+ return m.fmt == kFmtMXFP4 ? mx_row_bytes(m.k)
+ : m.fmt == kFmtQ8_0 ? q8_row_bytes(m.k)
+ : m.fmt == kFmtGG ? type_row_bytes(m.K, m.k)
+ : (size_t) m.k;
+}
+// a layer's low-bit experts as ds4_iq's arguments (false: its types not covered - printed)
+bool iq_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit, iq::Args& ia, const char* who) {
+ ia = iq::Args{};
+ for (int r = 0; r < 3; ++r) ia.tr[r] = lay.trellis[r];
+ ia.n_embd = n_embd;
+ ia.n_ff = n_ff;
+ ia.limit = limit;
+ ia.tg = expert_type(lay, 0);
+ ia.td = expert_type(lay, 2);
+ if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % 256 != 0 ||
+ n_ff % 256 != 0) {
+ std::fprintf(stderr, "ds4 gguf %s: expert types %s / %s / %s (n_embd %d, n_ff %d) not covered\n", who,
+ type_name(expert_type(lay, 0)), type_name(expert_type(lay, 1)), type_name(expert_type(lay, 2)),
+ n_embd, n_ff);
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+ return false;
+ }
+ return true;
+}
+template <int NC>
+void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {
+ const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);
+ if (f == 1) {
+ if (vec) mx_mv_kernel<NC, true, false><<<grid, kThreads, 0, s>>>(J);
+ else mx_mv_kernel<NC, false, false><<<grid, kThreads, 0, s>>>(J);
+ } else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)
+ if (vec) q8_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);
+ else q8_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);
+ } else {
+ if (vec) bf16_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);
+ else bf16_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);
+ }
+}
+void mv_set(int f, const BfJobs& J, int nj, int max_n, int max_c, bool vec, cudaStream_t s) {
+ if (nj == 0) return;
+ if (max_c <= 1) mv_launch<1>(f, J, nj, max_n, vec, s);
+ else if (max_c <= 2) mv_launch<2>(f, J, nj, max_n, vec, s);
+ else if (max_c <= 4) mv_launch<4>(f, J, nj, max_n, vec, s);
+ else if (max_c <= 8) mv_launch<8>(f, J, nj, max_n, vec, s);
+ else mv_launch<16>(f, J, nj, max_n, vec, s);
+}
+} // namespace
+
+x3::Mat view(const x3::Mat& m, int c0, int nc) {
+ x3::Mat v = m;
+ const size_t ld = row_stride(m);
+ v.ld = (int) ld;
+ v.raw = m.fmt != kFmtBF16 ? (const void*) ((const uint8_t*) m.raw + (size_t) c0 * ld)
+ : (const void*) ((const uint16_t*) m.raw + (size_t) c0 * ld);
+ v.n = nc;
+ return v;
+}
+
+bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
+ if (n <= 0 || n > x3::kMaxMvJobs) return false;
+ BfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch each
+ int nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};
+ bool vec[3] = {true, true, true};
+ iq::MvJob IJ[x3::kMaxMvJobs]; // the low-bit ones (ds4_iq: a launch a type)
+ int nij = 0;
+ for (int i = 0; i < n; ++i) {
+ const x3::MvJob& q = jobs[i];
+ const x3::Mat& m = *q.m;
+ if (m.fmt == kFmtGG) { // the activations rounded to int8 (job i's rows of the scratch), then ds4_iq's GEMV
+ Scratch* sc = scratch(s);
+ if (sc == nullptr || m.raw == nullptr || q.xf == nullptr || q.y == nullptr || q.ncol < 1 ||
+ q.ncol > x3::kMaxMvCols || m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {
+ std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (type %s k %d n %d)\n", i, type_name(m.K), m.k, m.n);
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+ return false;
+ }
+ const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
+ q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ iq::MvJob& b = IJ[nij++];
+ b.w = (const uint8_t*) m.raw;
+ b.ld = row_stride(m);
+ b.type = m.K;
+ b.k = m.k;
+ b.n = m.n;
+ b.ncol = q.ncol;
+ b.xq = sc->mq + at;
+ b.xd = sc->md + at / 32;
+ b.y = q.y;
+ b.ycs = q.y_cs;
+ b.bias = q.bias;
+ b.alpha = q.alpha;
+ continue;
+ }
+ const int f = m.fmt == kFmtMXFP4 ? 1 : m.fmt == kFmtQ8_0 ? 2 : 0;
+ if ((m.fmt != kFmtBF16 && m.fmt != kFmtMXFP4 && m.fmt != kFmtQ8_0) || m.raw == nullptr || q.xf == nullptr ||
+ q.y == nullptr || q.ncol < 1 || q.ncol > x3::kMaxMvCols || m.k <= 0 || m.n <= 0 ||
+ (f != 0 && m.k % kMxBlock != 0)) {
+ std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (fmt %d k %d n %d, %d activations)\n", i, m.fmt, m.k,
+ m.n, q.ncol);
+ g_errors.fetch_add(1, std::memory_order_relaxed);
+ return false;
+ }
+ BfJob& b = J[f].j[nj[f]++];
+ b.w = m.raw;
+ b.ld = row_stride(m);
+ b.x = q.xf;
+ b.xcs = q.xf_cs;
+ b.y = q.y;
+ b.ycs = q.y_cs;
+ b.bias = q.bias;
+ b.alpha = q.alpha;
+ b.k = m.k;
+ b.n = m.n;
+ b.ncol = q.ncol;
+ max_n[f] = std::max(max_n[f], m.n);
+ max_c[f] = std::max(max_c[f], q.ncol);
+ const bool xa = ((uintptr_t) b.x & 15u) == 0 && (q.ncol == 1 || b.xcs % 4 == 0);
+ if (f == 2) { // the activations rounded to int8 first (each job its own rows of the scratch)
+ Scratch* sc = scratch(s);
+ if (sc == nullptr || m.k > kMaxQK) {
+ fail("mv: Q8_0 inputs past the scratch");
+ return false;
+ }
+ const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
+ b.xq = sc->mq + at;
+ b.xd = sc->md + at / 32;
+ q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ vec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;
+ } else {
+ vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));
+ }
+ }
+ for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);
+ if (nij > 0 && !iq::mv(IJ, nij, s)) {
+ fail("mv: a low-bit job not covered");
+ return false;
+ }
+ check("mv");
+ return true;
+}
+
+void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
+ if ((m.fmt != kFmtBF16 && m.fmt != kFmtQ8_0) || m.raw == nullptr) {
+ fail("rows: not a BF16 or Q8_0 matrix");
+ return;
+ }
+ const size_t ld = row_stride(m);
+ const size_t total = (size_t) m.n * (size_t) m.k;
+ const unsigned blocks = (unsigned) std::min<size_t>((total + 255) / 256, 65535);
+ if (m.fmt == kFmtQ8_0) q8_rows_kernel<<<blocks, 256, 0, s>>>((const uint8_t*) m.raw, ld, m.k, m.n, dst, bf16);
+ else bf16_rows_kernel<<<blocks, 256, 0, s>>>((const uint16_t*) m.raw, ld, m.k, m.n, dst, bf16);
+ check("rows");
+}
+
+namespace {
+// (staged when the matrix's rows are 4-byte aligned: 17 nb % 4 == 0; the blobs' offsets are)
+template <int NE>
+struct GateUpK {
+ static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {
+ if (mx_row_bytes(m.n_embd) % 4 == 0) mx_gate_up_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);
+ else mx_gate_up_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);
+ }
+};
+template <int NE>
+struct DownK {
+ static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {
+ if (mx_row_bytes(m.n_ff) % 4 == 0) mx_down_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);
+ else mx_down_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);
+ }
+};
+template <template <int> class Kern>
+void launch_groups(const MxArgs& m, int rows, int groups, int ne_max, cudaStream_t s) {
+ const dim3 grid((unsigned) ((rows + kWarps - 1) / kWarps), (unsigned) groups);
+ if (ne_max <= 1) Kern<1>::run(m, grid, s);
+ else if (ne_max <= 4) Kern<4>::run(m, grid, s);
+ else if (ne_max <= 8) Kern<8>::run(m, grid, s);
+ else Kern<16>::run(m, grid, s);
+}
+MxArgs mx_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit) {
+ MxArgs m{};
+ for (int r = 0; r < 3; ++r) m.tr[r] = lay.trellis[r];
+ m.n_embd = n_embd;
+ m.n_ff = n_ff;
+ m.limit = limit;
+ return m;
+}
+} // namespace
+
+void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s) {
+ if (!mx_geometry("moe_gate_up", a.k, a.n_embd, a.n_ff) || h == nullptr) return;
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return;
+ if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's kernels
+ iq::Args ia;
+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_gate_up")) return;
+ ia.plan_ptr = a.plan_ptr;
+ ia.h = h;
+ ia.xq = sc->xq;
+ ia.xd = sc->xd;
+ q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);
+ iq::gate_up(ia, a.k, 1, s);
+ check("moe_gate_up");
+ return;
+ }
+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
+ m.plan_ptr = a.plan_ptr;
+ m.x = x;
+ m.h = h;
+ m.xq = sc->xq;
+ m.xd = sc->xd;
+ q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);
+ launch_groups<GateUpK>(m, a.n_ff, a.k, 1, s);
+ check("moe_gate_up");
+}
+
+void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
+ if (!mx_geometry("moe_down", a.k, a.n_embd, a.n_ff) || h == nullptr) return;
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return;
+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
+ m.plan_ptr = a.plan_ptr;
+ m.h = const_cast<float*>(h);
+ m.part = sc->part;
+ m.hq = sc->hq;
+ m.hd = sc->hd;
+ q8_quant(h, (size_t) a.n_ff, a.k, a.n_ff, sc->hq, sc->hd, s);
+ if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's down, then the same combine
+ iq::Args ia;
+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_down")) return;
+ ia.plan_ptr = a.plan_ptr;
+ ia.part = sc->part;
+ ia.hq = sc->hq;
+ ia.hd = sc->hd;
+ iq::down(ia, a.k, 1, s);
+ } else {
+ launch_groups<DownK>(m, a.n_embd, a.k, 1, s);
+ }
+ CombArgs c{};
+ c.plan_ptr = a.plan_ptr;
+ c.plan_w = a.plan_w;
+ c.part = sc->part;
+ c.cpu_part = a.cpu_part;
+ c.cpu_stride = (size_t) a.n_embd;
+ c.cpu_flag = a.cpu_flag;
+ c.sh_out = sh_out;
+ c.out = out;
+ c.out_stride = (size_t) a.n_embd;
+ c.k = a.k;
+ c.n_embd = a.n_embd;
+ mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s>>>(c);
+ check("moe_down");
+}
+
+bool win_experts(const x3::WinArgs& a, cudaStream_t s) {
+ if (a.T < 1 || a.T > kMaxEnt || !mx_geometry("win_experts", a.k, a.n_embd, a.n_ff) || a.h == nullptr ||
+ a.part == nullptr || a.out == nullptr) {
+ std::fprintf(stderr, "ds4 gguf win_experts: not covered (T %d)\n", a.T);
+ return false;
+ }
+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);
+ m.grp_ptr = a.grp_ptr;
+ m.grp_start = a.grp_start;
+ m.grp_ent = a.grp_ent;
+ m.n_groups = a.n_groups;
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ m.x = a.x;
+ m.x_stride = a.x_stride;
+ m.h = a.h;
+ m.part = a.part;
+ m.xq = sc->xq;
+ m.xd = sc->xd;
+ m.hq = sc->hq;
+ m.hd = sc->hd;
+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)
+ const bool mx = mx_experts(a.lay);
+ iq::Args ia;
+ if (!mx) { // low-bit experts: ds4_iq's kernels on the same groups and scratch
+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "win_experts")) return false;
+ ia.grp_ptr = a.grp_ptr;
+ ia.grp_start = a.grp_start;
+ ia.grp_ent = a.grp_ent;
+ ia.n_groups = a.n_groups;
+ ia.h = a.h;
+ ia.part = a.part;
+ ia.xq = sc->xq;
+ ia.xd = sc->xd;
+ ia.hq = sc->hq;
+ ia.hd = sc->hd;
+ }
+ q8_quant(a.x, a.x_stride, a.T, a.n_embd, sc->xq, sc->xd, s);
+ if (mx) launch_groups<GateUpK>(m, a.n_ff, groups, a.T, s);
+ else iq::gate_up(ia, groups, a.T, s);
+ q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)
+ if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);
+ else iq::down(ia, groups, a.T, s);
+ if (a.wait_ptr != nullptr) mx_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);
+ CombArgs c{};
+ c.plan_ptr = a.plan_ptr;
+ c.plan_w = a.plan_w;
+ c.part = a.part;
+ c.cpu_part = a.cpu_part;
+ c.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;
+ c.cpu_flag = a.cpu_flag;
+ c.sh_out = a.sh_out;
+ c.out = a.out;
+ c.out_stride = a.out_stride;
+ c.k = a.k;
+ c.n_embd = a.n_embd;
+ mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s>>>(c);
+ check("win_experts");
+ return true;
+}
+
+namespace {
+iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (the inputs / outputs and types to fill in)
+ iq::SetArgs q;
+ q.wbase = sa.wbase;
+ q.stride = sa.stride;
+ for (int r = 0; r < 3; ++r) q.tr[r] = sa.lay.trellis[r];
+ q.bounds = sa.bounds;
+ q.row0 = sa.row0;
+ q.max_rows = sa.max_rows;
+ q.n_exp = sa.n_exp;
+ q.n_embd = sa.n_embd;
+ q.n_ff = sa.n_ff;
+ q.limit = sa.limit;
+ return q;
+}
+} // namespace
+
+void set_gate_up(const x3::SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
+ cudaStream_t s) {
+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
+ if (!mx_geometry("set_gate_up", 1, sa.n_embd, sa.n_ff)) return;
+ if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets
+ iq::Args ia;
+ if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_gate_up")) return;
+ iq::SetArgs q = iq_set(sa);
+ q.xf = x;
+ q.idx = row_idx;
+ q.ldx = ldx;
+ q.h = h16;
+ q.ldo = ldh;
+ q.tg = ia.tg;
+ q.td = ia.td;
+ if (!iq::set_gate_up(q, s)) fail("set_gate_up: not covered");
+ check("set_gate_up");
+ return;
+ }
+ int skip = 0;
+ { // the experts of at most 16 rows: the light kernels (each weight read once; ds4_iq.cuh), the others the tiles
+ iq::SetArgs q = iq_set(sa);
+ q.xf = x;
+ q.idx = row_idx;
+ q.ldx = ldx;
+ q.h = h16;
+ q.ldo = ldh;
+ q.tg = q.td = kTypeMXFP4;
+ if (iq::light_gate_up(q, s)) {
+ if (sa.max_rows <= 16) {
+ check("set_gate_up");
+ return;
+ }
+ skip = 16;
+ }
+ }
+ SetK k{};
+ k.skip_le = skip;
+ k.wbase = sa.wbase;
+ k.stride = sa.stride;
+ for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];
+ k.bounds = sa.bounds;
+ k.row0 = sa.row0;
+ k.n_embd = sa.n_embd;
+ k.n_ff = sa.n_ff;
+ k.limit = sa.limit;
+ k.xf = x;
+ k.idx = row_idx;
+ k.ldx = ldx;
+ k.h = (__half*) h16;
+ k.ldo = ldh;
+ const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_ff / BN), (unsigned) sa.n_exp);
+#if !defined(STRATA_USE_HIP)
+ if (tc_ok()) mx_set_tc_kernel<true><<<grid, kThreads, 0, s>>>(k);
+#else
+ if (tc_ok()) mx_set_wmma_kernel<true><<<grid, kThreads, 0, s>>>(k);
+#endif
+ else
+ mx_set_kernel<true><<<grid, kThreads, 0, s>>>(k);
+ check("set_gate_up");
+}
+
+void set_down(const x3::SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {
+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
+ if (!mx_geometry("set_down", 1, sa.n_embd, sa.n_ff)) return;
+ if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets
+ iq::Args ia;
+ if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_down")) return;
+ iq::SetArgs q = iq_set(sa);
+ q.xh = h16;
+ q.ldx = ldh;
+ q.y = y;
+ q.ldo = ldy;
+ q.tg = ia.tg;
+ q.td = ia.td;
+ if (!iq::set_down(q, s)) fail("set_down: not covered");
+ check("set_down");
+ return;
+ }
+ int skip = 0;
+ { // (as set_gate_up's)
+ iq::SetArgs q = iq_set(sa);
+ q.xh = h16;
+ q.ldx = ldh;
+ q.y = y;
+ q.ldo = ldy;
+ q.tg = q.td = kTypeMXFP4;
+ if (iq::light_down(q, s)) {
+ if (sa.max_rows <= 16) {
+ check("set_down");
+ return;
+ }
+ skip = 16;
+ }
+ }
+ SetK k{};
+ k.skip_le = skip;
+ k.wbase = sa.wbase;
+ k.stride = sa.stride;
+ for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];
+ k.bounds = sa.bounds;
+ k.row0 = sa.row0;
+ k.n_embd = sa.n_embd;
+ k.n_ff = sa.n_ff;
+ k.xh = (const __half*) h16;
+ k.ldx = ldh;
+ k.y = y;
+ k.ldo = ldy;
+ const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_embd / BN), (unsigned) sa.n_exp);
+#if !defined(STRATA_USE_HIP)
+ if (tc_ok()) mx_set_tc_kernel<false><<<grid, kThreads, 0, s>>>(k);
+#else
+ if (tc_ok()) mx_set_wmma_kernel<false><<<grid, kThreads, 0, s>>>(k);
+#endif
+ else
+ mx_set_kernel<false><<<grid, kThreads, 0, s>>>(k);
+ check("set_down");
+}
+
+} // namespace strata::ds4::gguf
diff --git a/src/dsv41/ds4_gguf.cuh b/src/dsv41/ds4_gguf.cuh
new file mode 100644
index 000000000..6257c8a7f
--- /dev/null
+++ b/src/dsv41/ds4_gguf.cuh
@@ -0,0 +1,76 @@
+// src/ds4/ds4_gguf.cuh - a GGUF's weights on the device for maya-ds4 (unsloth's DeepSeek-V4.1-Flash-1017 GGUFs,
+// tools/ds4/pack_gguf.py): the dense matrices in BF16 and the routed experts in MXFP4. They stand in for
+// strata::kernels::exl3's matrices and experts (an x3::Mat with fmt kFmtBF16, an x3::ExpertLayout with fmt kFmtMXFP4)
+// with the same arguments and the same results' layout, so the engine's paths (decode, verify windows, chunked prompts,
+// tensor parallel) take either format.
+//
+// MXFP4 is ggml's block_mxfp4: 32 values in 17 bytes, an E8M0 scale e then 16 bytes of E2M1 codes - value j of the
+// block in byte j's low nibble (j < 16) or byte j - 16's high nibble, worth kvalues[code] * 2^(e - 128) with kvalues
+// {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12} (the E2M1 values doubled). A matrix is [out][in / 32] blocks
+// (row r = output r), as the GGUF stores it, so an expert's blob is its gate, up and down matrices one after another.
+// Every product is an F32 sum of exactly dequantised weights times F32 activations, and a window's entry gets the bits
+// the token path gives it (the same kernel, the same order).
+#pragma once
+
+#include "strata/kernels/exl3.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4::gguf {
+
+namespace x3 = strata::kernels::exl3;
+
+constexpr int kFmtBF16 = 1; // x3::Mat::fmt
+constexpr int kFmtMXFP4 = 2; // x3::ExpertLayout::fmt (and an expert's matrix as an x3::Mat)
+constexpr int kFmtQ8_0 = 3; // x3::Mat::fmt: ggml's Q8_0, an FP16 scale and 32 int8 in 34 bytes (UD-Q4_K_XL's dense)
+constexpr int kFmtGG = 4; // x3::Mat::fmt: ggml's low-bit block type m.K (ds4_iq.cuh's: IQ2_XXS .. Q4_K), [n][k / 256]
+constexpr int kMxBlock = 32, kMxBytes = 17, kQ8Bytes = 34;
+constexpr int kTypeMXFP4 = 39; // ggml's type id
+
+/// Bytes of an MXFP4 row of k values (k % 32 == 0), and of a matrix of n such rows.
+__host__ __device__ inline size_t mx_row_bytes(int64_t k) { return (size_t) (k / kMxBlock) * kMxBytes; }
+__host__ __device__ inline size_t mx_bytes(int64_t n, int64_t k) { return (size_t) n * mx_row_bytes(k); }
+__host__ __device__ inline size_t q8_row_bytes(int64_t k) { return (size_t) (k / 32) * kQ8Bytes; }
+
+/// A GGUF expert layer (x3::ExpertLayout::fmt kFmtMXFP4: block-quantized experts read in place) keeps each role's ggml
+/// type in lay.K[r] (r: gate, up, down; 0 = MXFP4): MXFP4 takes these kernels, the low-bit types ds4_iq.cuh's (gate
+/// and up of one type). Roles of one layer may differ (down of another type); MXFP4 and a low-bit type do not mix.
+inline int expert_type(const x3::ExpertLayout& l, int r) { return l.K[r] == 0 ? kTypeMXFP4 : l.K[r]; }
+inline bool mx_experts(const x3::ExpertLayout& l) {
+ return expert_type(l, 0) == kTypeMXFP4 && expert_type(l, 1) == kTypeMXFP4 && expert_type(l, 2) == kTypeMXFP4;
+}
+/// Bytes of a row of k values of an expert type (0: not one the device kernels take, or k not whole blocks).
+size_t type_row_bytes(int type, int64_t k);
+/// Values a block of an expert type (32: MXFP4, 256: the low-bit ones; 0: neither).
+int type_block(int type);
+/// The type's name (ggml's), "?" when unknown.
+const char* type_name(int type);
+
+/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).
+bool prepare(cudaStream_t s);
+
+/// x3::mv's contract for GGUF matrices (F32 activations): each job's m->fmt kFmtBF16 (raw = BF16 [n][ld], ld values a
+/// row), kFmtMXFP4 (raw = MXFP4 [n][k / 32] blocks, ld bytes a row; 0: mx_row_bytes(k)) - an expert's matrix in VRAM -
+/// or kFmtQ8_0 (raw = Q8_0 [n][k / 32] blocks, ld bytes a row; 0: q8_row_bytes(k)).
+bool mv(const x3::MvJob* jobs, int n, cudaStream_t s);
+/// The columns [c0, c0 + nc) of such a matrix: its rows c0 .. c0 + nc - 1, a contiguous range.
+x3::Mat view(const x3::Mat& m, int c0, int nc);
+/// x3::reconstruct_rows' layout from a BF16 or Q8_0 matrix: dst[r][c] = W[c][r] (n rows of k values), F16 or BF16
+/// bits (a Q8_0 weight rounded to them).
+void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);
+
+/// x3::moe_gate_up / moe_down / win_experts / set_gate_up / set_down for a GGUF's experts (lay.fmt == kFmtMXFP4; the
+/// roles' types in lay.K, see expert_type).
+void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s);
+void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
+bool win_experts(const x3::WinArgs& a, cudaStream_t s);
+void set_gate_up(const x3::SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
+ cudaStream_t s);
+void set_down(const x3::SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);
+
+/// Launch failures of these kernels since the process started (each is also printed).
+int launch_errors();
+
+} // namespace strata::ds4::gguf
diff --git a/src/dsv41/ds4_gguf_cpu.cpp b/src/dsv41/ds4_gguf_cpu.cpp
new file mode 100644
index 000000000..beae19dcf
--- /dev/null
+++ b/src/dsv41/ds4_gguf_cpu.cpp
@@ -0,0 +1,416 @@
+// src/ds4/ds4_gguf_cpu.cpp - a GGUF's MXFP4 experts on the CPU lane (see ds4_gguf_cpu.hpp).
+#include "ds4_gguf_cpu.hpp"
+
+#include "strata/kernels/cpu/exl3_cpu.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define MX_X86 1
+#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+
+// a function's instruction set (GCC / Clang, clang-cl too; MSVC compiles any intrinsic as it is): the file is built for
+// the baseline x86-64, and each tier's code runs only where detect() says the CPU and the OS take it
+#if defined(MX_X86) && (defined(__GNUC__) || defined(__clang__))
+#define MX_TARGET(s) __attribute__((target(s)))
+#else
+#define MX_TARGET(s)
+#endif
+#define MX_AVX2 MX_TARGET("avx2,fma")
+// AVX-VNNI's intrinsics: GCC 11, Clang 12, MSVC 2022
+#if defined(MX_X86) && (defined(_MSC_VER) || (defined(__clang__) && __clang_major__ >= 12) || \
+ (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 11))
+#define MX_AVXVNNI 1
+#endif
+
+namespace strata::ds4::gguf {
+namespace {
+
+constexpr int kBlock = 32, kBytes = 17;
+constexpr float kValues[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f,
+ 0.0f, -1.0f, -2.0f, -3.0f, -4.0f, -6.0f, -8.0f, -12.0f};
+constexpr int kCodes[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+
+float e8m0_half(uint32_t e) { // 2^(e - 128), as the device's
+ const uint32_t bits = e < 2 ? 0x00200000u << e : (e - 1) << 23;
+ float f;
+ std::memcpy(&f, &bits, 4);
+ return f;
+}
+struct E8m0Table {
+ float v[256];
+ E8m0Table() {
+ for (int e = 0; e < 256; ++e) v[e] = e8m0_half((uint32_t) e);
+ }
+};
+const E8m0Table& e8() {
+ static const E8m0Table t;
+ return t;
+}
+
+void rows_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* p = w + (size_t) r * rb;
+ float acc[kMxCpuMaxN] = {};
+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+ const float d = e8m0_half(p[0]);
+ float wv[kBlock];
+ for (int j = 0; j < 16; ++j) {
+ wv[j] = kValues[p[1 + j] & 15] * d;
+ wv[j + 16] = kValues[p[1 + j] >> 4] * d;
+ }
+ for (int i = 0; i < n; ++i) {
+ const float* xb = x[i] + b * kBlock;
+ float s = 0.0f;
+ for (int j = 0; j < kBlock; ++j) s += wv[j] * xb[j];
+ acc[i] += s;
+ }
+ }
+ for (int i = 0; i < n; ++i) y[i][r - r0] = acc[i];
+ }
+}
+
+#if defined(MX_X86)
+MX_AVX2 float hsum(__m256 v) {
+ const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));
+ const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));
+ return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));
+}
+
+// a block: the 16 code bytes -> 32 int8 values through a shuffle table (low nibbles: values 0..15, high: 16..31),
+// widened to F32 in 4 x 8, times the block's scale (exact), FMAs against each activation
+MX_AVX2 void rows_avx2(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n,
+ float* const* y) {
+ const __m128i lut = _mm_setr_epi8(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12);
+ const __m128i m4 = _mm_set1_epi8(0x0f);
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* p = w + (size_t) r * rb;
+ __m256 acc[kMxCpuMaxN];
+ for (int i = 0; i < n; ++i) acc[i] = _mm256_setzero_ps();
+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+ const __m256 d = _mm256_set1_ps(e8m0_half(p[0]));
+ const __m128i q = _mm_loadu_si128((const __m128i*) (p + 1));
+ const __m128i lo = _mm_shuffle_epi8(lut, _mm_and_si128(q, m4));
+ const __m128i hi = _mm_shuffle_epi8(lut, _mm_and_si128(_mm_srli_epi16(q, 4), m4));
+ const __m256 w0 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo)));
+ const __m256 w1 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo, 8))));
+ const __m256 w2 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi)));
+ const __m256 w3 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi, 8))));
+ for (int i = 0; i < n; ++i) {
+ const float* xb = x[i] + b * kBlock;
+ __m256 a = acc[i];
+ a = _mm256_fmadd_ps(w0, _mm256_loadu_ps(xb), a);
+ a = _mm256_fmadd_ps(w1, _mm256_loadu_ps(xb + 8), a);
+ a = _mm256_fmadd_ps(w2, _mm256_loadu_ps(xb + 16), a);
+ a = _mm256_fmadd_ps(w3, _mm256_loadu_ps(xb + 24), a);
+ acc[i] = a;
+ }
+ }
+ for (int i = 0; i < n; ++i) y[i][r - r0] = hsum(acc[i]);
+ }
+}
+
+// ---- the int8 path's vector tiers (ds4_mx_kern.inc)
+#define MXK_NS mx_avx2
+#define MXK_ATTR MX_TARGET("avx2,fma")
+#define MXK_256 1
+#include "ds4_mx_kern.inc"
+#if defined(MX_AVXVNNI)
+#define MXK_NS mx_avxvnni
+#define MXK_ATTR MX_TARGET("avx2,fma,avxvnni")
+#define MXK_256 1
+#define MXK_VNNI 1
+#include "ds4_mx_kern.inc"
+#endif
+#define MXK_NS mx_avx512
+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,fma")
+#define MXK_512 1
+#include "ds4_mx_kern.inc"
+#define MXK_NS mx_avx512vnni
+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,avx512vnni,fma")
+#define MXK_512 1
+#define MXK_VNNI 1
+#include "ds4_mx_kern.inc"
+#endif
+
+// ---- the int8 path's scalar tier: ds4_mx_kern.inc's order (8 accumulators, block j of a group of 8 into L(j)), so it
+// gives the vector tiers' bits - also the tests' check on them. The activation's layout (ds4_mx_kern.inc), for any k
+// % 32 == 0: the int8 values in quads (a partial last quad padded), then per group of 8 blocks the scales and the
+// offset corrections in lane order (a partial last group padded)
+inline int act_lane(int b) { return 8 * (b >> 3) + ((b & 7) & 1) * 4 + ((b & 7) >> 1); }
+inline size_t act_qbytes(int k) { return (size_t) 128 * (size_t) ((k / kBlock + 3) / 4); }
+inline size_t act_groups(int k) { return (size_t) ((k / kBlock + 7) / 8); }
+inline int act_q(const uint8_t* act, int b, int j) { // block b's value j (0..31)
+ return ((const int8_t*) act)[(b >> 2) * 128 + (j >= 16 ? 64 : 0) + 16 * (b & 3) + (j & 15)];
+}
+inline float act_d(const uint8_t* act, int k, int b) { return ((const float*) (act + act_qbytes(k)))[act_lane(b)]; }
+
+void quant_scalar(const float* x, int b0, int b1, uint8_t* act, int k) {
+ int8_t* q = (int8_t*) act;
+ float* dx = (float*) (act + act_qbytes(k));
+ int32_t* corr = (int32_t*) (act + act_qbytes(k) + 32 * act_groups(k));
+ for (int b = b0; b < b1; ++b, x += kBlock) {
+ float am = 0.0f;
+ for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(x[j]));
+ const float sc = am / 127.0f;
+ int8_t v[kBlock];
+ int s = 0;
+ for (int j = 0; j < kBlock; ++j) {
+ v[j] = am > 0.0f ? (int8_t) (int) std::nearbyint(x[j] / sc) : (int8_t) 0;
+ s += v[j];
+ }
+ int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);
+ for (int j = 0; j < 16; ++j) {
+ lo[j] = v[j];
+ lo[64 + j] = v[16 + j];
+ }
+ dx[act_lane(b)] = sc;
+ corr[act_lane(b)] = -12 * s;
+ }
+}
+
+void rows_q_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {
+ const float* t = e8().v;
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* p = w + (size_t) r * rb;
+ float acc[kMxCpuMaxN][8] = {};
+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {
+ int wv[kBlock];
+ for (int j = 0; j < 16; ++j) {
+ wv[j] = kCodes[p[1 + j] & 15];
+ wv[j + 16] = kCodes[p[1 + j] >> 4];
+ }
+ const int l = act_lane(b) & 7;
+ for (int i = 0; i < n; ++i) {
+ const uint8_t* a = (const uint8_t*) act[i];
+ int s = 0;
+ for (int j = 0; j < kBlock; ++j) s += wv[j] * act_q(a, b, j);
+ acc[i][l] = std::fma((float) s, t[p[0]] * act_d(a, k, b), acc[i][l]);
+ }
+ }
+ for (int i = 0; i < n; ++i) {
+ const float* A = acc[i];
+ y[i][r - r0] = ((A[0] + A[4]) + (A[2] + A[6])) + ((A[1] + A[5]) + (A[3] + A[7]));
+ }
+ }
+}
+
+// ---- which tiers this CPU runs
+struct Cpu {
+ bool avx2 = false, vnni256 = false, avx512 = false, vnni512 = false, vbmi = false, bmi2 = false;
+};
+Cpu detect() {
+ Cpu c;
+#if defined(MX_X86)
+ unsigned r[4] = {0, 0, 0, 0};
+ auto cpuid = [&](unsigned leaf, unsigned sub) {
+#if defined(_MSC_VER)
+ int x[4];
+ __cpuidex(x, (int) leaf, (int) sub);
+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+#endif
+ };
+ cpuid(0, 0);
+ const unsigned max_leaf = r[0];
+ if (max_leaf < 7) return c;
+ cpuid(1, 0);
+ const bool fma = (r[2] >> 12) & 1u, osxsave = (r[2] >> 27) & 1u, avx = (r[2] >> 28) & 1u;
+ if (!fma || !osxsave || !avx) return c;
+#if defined(_MSC_VER)
+ const unsigned long long xcr0 = _xgetbv(0);
+#else
+ unsigned lo = 0, hi = 0;
+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;
+#endif
+ if ((xcr0 & 0x6) != 0x6) return c; // the OS saves the YMM state
+ cpuid(7, 0);
+ const unsigned b7 = r[1], c7 = r[2];
+ c.avx2 = (b7 >> 5) & 1u;
+ const bool zmm_os = (xcr0 & 0xE6) == 0xE6;
+ c.avx512 = c.avx2 && zmm_os && ((b7 >> 16) & 1u) && ((b7 >> 30) & 1u); // F, BW
+ c.vnni512 = c.avx512 && ((c7 >> 11) & 1u);
+ c.vbmi = c.avx512 && ((c7 >> 1) & 1u); // AVX-512 VBMI
+ c.bmi2 = (b7 >> 8) & 1u;
+ cpuid(7, 1);
+ c.vnni256 = c.avx2 && ((r[0] >> 4) & 1u); // AVX-VNNI
+#endif
+ return c;
+}
+const Cpu& cpu() {
+ static const Cpu c = detect();
+ return c;
+}
+
+// The tier for this CPU, measured: every vector tier it runs times the same small warm job (64 rows of a 5120-wide
+// matrix, in L2, against 1, 2 and 4 activations - the compute a row takes, which is where the tiers differ; streaming
+// from RAM they all reach the memory's speed) in interleaved rounds, the best round of each counting. The choice never
+// changes an answer (the tiers give the same bits), only the speed. DS4_MX_TIER=<name> picks one by hand.
+std::string g_tier_report;
+int measure_tier(const std::vector<int>& cand);
+int pick_tier() {
+ if (const char* f = std::getenv("DS4_MX_TIER"))
+ for (int t = 0; t < kMxTiers; ++t)
+ if (std::string(f) == mx_tier_name(t) && mx_tier_ok(t)) {
+ g_tier_report = std::string("DS4_MX_TIER=") + f;
+ return t;
+ }
+ std::vector<int> cand;
+ for (int t = 1; t < kMxTiers; ++t)
+ if (mx_tier_ok(t)) cand.push_back(t);
+ if (cand.empty()) return kMxScalar;
+ if (cand.size() == 1) return cand[0];
+ return measure_tier(cand);
+}
+int measure_tier(const std::vector<int>& cand) {
+ constexpr int K = 5120, R = 64, NT = 4;
+ const size_t rb = (size_t) K / kBlock * kBytes;
+ std::vector<uint8_t> w(rb * R), act(mx_act_bytes(K) * NT);
+ uint32_t s = 0x9E3779B9u;
+ for (size_t i = 0; i < w.size(); ++i) {
+ s = s * 1664525u + 1013904223u;
+ w[i] = (uint8_t) (s >> 24);
+ }
+ for (size_t b = 0; b < w.size() / kBytes; ++b) w[b * kBytes] = (uint8_t) (118 + b % 6);
+ std::vector<float> x(K);
+ for (int i = 0; i < NT; ++i) {
+ for (int j = 0; j < K; ++j) {
+ s = s * 1664525u + 1013904223u;
+ x[(size_t) j] = (float) ((int) (s >> 16) - 32768) / 32768.0f;
+ }
+ quant_scalar(x.data(), 0, K / kBlock, act.data() + (size_t) i * mx_act_bytes(K), K);
+ }
+ const void* ap[NT];
+ for (int i = 0; i < NT; ++i) ap[i] = act.data() + (size_t) i * mx_act_bytes(K);
+ float y[NT][R];
+ float* yp[NT] = {y[0], y[1], y[2], y[3]};
+ std::vector<double> best(cand.size(), 1e30);
+ for (int round = 0; round < 6; ++round)
+ for (size_t c = 0; c < cand.size(); ++c) {
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int rep = 0; rep < 8; ++rep)
+ for (int n : {1, 2, 4}) mx_rows_tier(cand[c], w.data(), rb, K, 0, R, ap, n, yp, 0);
+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+ if (round > 0) best[c] = std::min(best[c], us); // (round 0: the clock and the caches warm)
+ }
+ size_t bi = 0;
+ for (size_t c = 1; c < cand.size(); ++c)
+ if (best[c] < best[bi]) bi = c;
+ char buf[64];
+ for (size_t c = 0; c < cand.size(); ++c) {
+ std::snprintf(buf, sizeof buf, "%s%s %.0f us", c ? ", " : "", mx_tier_name(cand[c]), best[c]);
+ g_tier_report += buf;
+ }
+ return cand[bi];
+}
+int prefetch_bytes() {
+ const char* f = std::getenv("DS4_MX_PF");
+ return f != nullptr ? std::max(0, std::atoi(f)) : 0;
+}
+
+} // namespace
+
+void q8_round(const float* x, int k, float* out) {
+ for (int b = 0; b < k / kBlock; ++b) {
+ const float* xb = x + b * kBlock;
+ float am = 0.0f;
+ for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(xb[j]));
+ const float sc = am / 127.0f;
+ for (int j = 0; j < kBlock; ++j)
+ out[b * kBlock + j] = am > 0.0f ? std::nearbyint(xb[j] / sc) * sc : 0.0f;
+ }
+}
+
+void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {
+#if defined(MX_X86)
+ static const bool avx2 = strata::kernels::cpu::exl3_cpu_tier() >= 1;
+ if (avx2) return rows_avx2(w, rb, k, r0, r1, x, n, y);
+#endif
+ rows_scalar(w, rb, k, r0, r1, x, n, y);
+}
+
+// ---- the int8 path
+bool mx_tier_ok(int t) {
+ switch (t) {
+ case kMxScalar: return true;
+ case kMxAvx2: return cpu().avx2;
+#if defined(MX_AVXVNNI)
+ case kMxAvxVnni: return cpu().vnni256;
+#endif
+ case kMxAvx512: return cpu().avx512;
+ case kMxAvx512Vnni: return cpu().vnni512;
+ default: return false;
+ }
+}
+const char* mx_tier_name(int t) {
+ switch (t) {
+ case kMxScalar: return "scalar";
+ case kMxAvx2: return "avx2";
+ case kMxAvxVnni: return "avxvnni";
+ case kMxAvx512: return "avx512";
+ case kMxAvx512Vnni: return "avx512vnni";
+ default: return "?";
+ }
+}
+int mx_tier() {
+ static const int t = pick_tier();
+ return t;
+}
+const char* mx_tier_report() {
+ (void) mx_tier();
+ return g_tier_report.c_str();
+}
+
+void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k) {
+#if defined(MX_X86)
+ if (cpu().avx2 && k % 256 == 0) return mx_avx2::quant(x, b0, b1, (uint8_t*) act, k);
+#endif
+ quant_scalar(x, b0, b1, (uint8_t*) act, k);
+}
+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k) {
+ quant_scalar(x, b0, b1, (uint8_t*) act, k);
+}
+int mx_act_value(const void* act, int b, int j) { return act_q((const uint8_t*) act, b, j); }
+float mx_act_scale(const void* act, int k, int b) { return act_d((const uint8_t*) act, k, b); }
+
+void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+ float* const* y, int pf) {
+ if (k % 256 != 0 || !mx_tier_ok(t)) t = kMxScalar; // (the vector tiers take a group of 8 blocks a step)
+ const float* tb = e8().v;
+#if defined(MX_X86)
+ switch (t) {
+ case kMxAvx2: return mx_avx2::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+#if defined(MX_AVXVNNI)
+ case kMxAvxVnni: return mx_avxvnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+#endif
+ case kMxAvx512: return mx_avx512::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+ case kMxAvx512Vnni: return mx_avx512vnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);
+ default: break;
+ }
+#endif
+ (void) tb, (void) pf;
+ for (int i0 = 0; i0 < n; i0 += kMxCpuMaxN)
+ rows_q_scalar(w, rb, k, r0, r1, act + i0, std::min(kMxCpuMaxN, n - i0), y + i0);
+}
+
+void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {
+ static const int pf = prefetch_bytes();
+ mx_rows_tier(mx_tier(), w, rb, k, r0, r1, act, n, y, pf);
+}
+
+} // namespace strata::ds4::gguf
diff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hpp
new file mode 100644
index 000000000..170c0f499
--- /dev/null
+++ b/src/dsv41/ds4_gguf_cpu.hpp
@@ -0,0 +1,75 @@
+// src/ds4/ds4_gguf_cpu.hpp - a GGUF's MXFP4 experts on the CPU lane of maya-ds4 (the format: ds4_gguf.cuh). The
+// weights are read where they lie (the GGUF file's mapping), each block decoded once for all the activations of a
+// call (mx_rows: int8 dots, the CPU's best vector tier at run time; a scalar loop otherwise).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+namespace strata::ds4::gguf {
+
+constexpr int kMxCpuMaxN = 8; // activations a call
+
+/// y[i][r - r0] = (row r of W) . x[i] for r0 <= r < r1 and i < n (n <= kMxCpuMaxN): W's rows hold k values
+/// (k % 32 == 0) as MXFP4 blocks, rb bytes apart. (F32 activations: the tests' and the benchmark's old path.)
+void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y);
+
+/// A GGUF's low-bit experts (ds4_iq.cuh's types) on the CPU lane - ggml-cpu's arithmetic through the GLM CPU lane's
+/// native experts (strata::kernels::cpu::native_*: its AVX-512 / AVX2 multi-token rows where they are ggml's bit for bit,
+/// ggml's dots otherwise; ds4_iq_cpu.cpp): an activation quantized once to the weight type's vec_dot_type (Q8_K: an F32
+/// scale per 256 values - the device takes one per 32, so the two sides agree to that rounding), each row one dot.
+struct IqCpu {
+ int tg = 0, td = 0, n_embd = 0, n_ff = 0;
+ int gu_act = 0, d_act = 0; // the activations' ggml types
+ size_t gu_row = 0, d_row = 0; // bytes a weight row
+ size_t act_bytes = 0, h_bytes = 0; // bytes of x (n_embd) and of h (n_ff) as those
+};
+/// The layer's geometry; false (err) when this build or ggml-cpu cannot take it.
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+/// x (k values) as the activation type `act` (f.gu_act / f.d_act) into dst.
+void iq_cpu_quant(int act, const float* x, int k, void* dst);
+/// ff[t][r] = clamped swiglu(gate_r . a[t], up_r . a[t]) for rows [r0, r1) (limit 0: no clamp), nt tokens.
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,
+ float* const* ff, int r0, int r1, float limit);
+/// out[t][r] = down_r . hq[t] for rows [r0, r1).
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32
+/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then
+/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of the
+/// blocks' sums. The vector tiers (ds4_mx_kern.inc): AVX2 vpmaddubsw, AVX-VNNI and AVX-512 (VNNI) vpdpbusd; the CPU's
+/// fastest is taken at run time (DS4_MX_TIER=<name> picks another one it runs). Every tier, the scalar one included,
+/// sums in the same order: they give the same bits, so the choice never changes an answer.
+enum MxTier { kMxScalar = 0, kMxAvx2, kMxAvxVnni, kMxAvx512, kMxAvx512Vnni, kMxTiers };
+/// Bytes of an activation of k values (k % 32 == 0) for mx_rows: the int8 values in quads of blocks, then a scale and
+/// an offset correction a block in groups of 8 (ds4_mx_kern.inc) - 1.25 k for k % 256 == 0.
+inline size_t mx_act_bytes(int k) {
+ const size_t nb = (size_t) k / 32;
+ return 128 * ((nb + 3) / 4) + 64 * ((nb + 7) / 8);
+}
+/// (tests) value j (0..31) of block b of an activation, and the block's scale
+int mx_act_value(const void* act, int b, int j);
+float mx_act_scale(const void* act, int k, int b);
+/// Blocks [b0, b1) of a k-value activation (x at block b0's first value: x[32 (b - b0) + j]) into act.
+void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k);
+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k); // (the same bytes, no vectors)
+inline void mx_quant(const float* x, int k, void* act) { mx_quant_blocks(x, 0, k / 32, act, k); }
+/// y[i][r - r0] = (row r of W) . act[i] for r0 <= r < r1, i < n (any n; act[i] mx_quant's of k values).
+void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y);
+/// The same with tier t (the scalar one when the CPU does not run t, or k % 256 != 0); pf: bytes prefetched ahead.
+void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+ float* const* y, int pf = 0);
+int mx_tier(); // the tier mx_rows takes (measured on first use: the fastest a core, ds4_gguf_cpu.cpp)
+const char* mx_tier_report(); // the measurement behind it (each candidate's time), or how it was picked
+bool mx_tier_ok(int t); // this CPU runs tier t
+const char* mx_tier_name(int t);
+/// Whether the rows of `type` are ds4's own here (IQ3_XXS / IQ3_S on AVX2 / AVX-512 VBMI: ggml-cpu's numbers bit for
+/// bit, checked against its dot on first use; DS4_IQ_OWN=0 turns them off) rather than ggml's / the GLM lane's.
+bool iq_cpu_own_rows(int type);
+
+/// x rounded as the device's MXFP4 kernels round their activations (q8_quant: per 32 values sc = max|x| / 127, q =
+/// round-half-even(x / sc), the same F32 operations, so the same q), into out as q sc - an expert gives the same sums
+/// on the CPU lane as in VRAM (up to their order), whichever side the expert cache puts it on.
+void q8_round(const float* x, int k, float* out);
+
+} // namespace strata::ds4::gguf
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
new file mode 100644
index 000000000..22ff70b05
--- /dev/null
+++ b/src/dsv41/ds4_iq.cu
@@ -0,0 +1,1018 @@
+// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are
+// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.
+#include "ds4_iq.cuh"
+
+#include <cuda_fp16.h>
+#if !defined(STRATA_USE_HIP)
+#include <mma.h>
+#endif
+
+#define GGML_COMMON_DECL_CUDA
+#define GGML_COMMON_IMPL_CUDA
+#include "ggml-common.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+#include <unordered_map>
+
+namespace strata::ds4::gguf::iq {
+namespace {
+
+constexpr int kWarps = 8, kThreads = kWarps * 32;
+
+// ---------------------------------------------------------------- the types
+template <int T>
+struct Ty;
+template <> struct Ty<kIQ2_XXS> { static constexpr int B = 66, TAB = 512, MIN = 0; };
+template <> struct Ty<kIQ2_S> { static constexpr int B = 82, TAB = 2048, MIN = 0; };
+template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN = 0; };
+template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };
+template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };
+template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };
+template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
+template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
+// MXFP4 (ds4_gguf.cu's own kernels everywhere else): a block of 32 values, an E8M0 scale then 16 code bytes; the light
+// sets take it too
+constexpr int kMXFP4 = 39;
+template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };
+template <int T> constexpr int kQK = T == kMXFP4 ? 32 : 256; // values a block
+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_s) == 82 && sizeof(block_iq3_xxs) == 98 &&
+ sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&
+ sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110,
+ "ggml's block sizes");
+
+template <int T>
+__host__ __device__ __forceinline__ size_t row_bytes_t(int k) { return (size_t) (k / kQK<T>) * Ty<T>::B; }
+
+// the type's codebook into shared memory (u32 words; IQ4_XS: its 16 values, from bytes - an int8 table's alignment)
+template <int T>
+__device__ __forceinline__ void load_tab(uint32_t* tab) {
+ if constexpr (T == kIQ4_XS) {
+ if (threadIdx.x < 4) {
+ const uint8_t* kv = (const uint8_t*) kvalues_iq4nl + 4 * threadIdx.x;
+ tab[threadIdx.x] = (uint32_t) kv[0] | (uint32_t) kv[1] << 8 | (uint32_t) kv[2] << 16 | (uint32_t) kv[3] << 24;
+ }
+ } else if constexpr (Ty<T>::TAB > 0) {
+ const uint32_t* src = T == kIQ2_XXS ? (const uint32_t*) iq2xxs_grid
+ : T == kIQ2_S ? (const uint32_t*) iq2s_grid
+ : T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid
+ : (const uint32_t*) iq3s_grid;
+ for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];
+ }
+}
+
+__device__ __forceinline__ uint32_t ld2(const uint8_t* p) { // 4 bytes from a 2-byte aligned address
+ const uint16_t* q = (const uint16_t*) p;
+ return (uint32_t) q[0] | ((uint32_t) q[1] << 16);
+}
+__device__ __forceinline__ uint32_t ld4(const uint8_t* p) { return *(const uint32_t*) p; }
+__device__ __forceinline__ float h2f(const uint8_t* p) { return __half2float(__ushort_as_half(*(const uint16_t*) p)); }
+// 7 sign bits (bits 0..6 of v) -> the 8 signs, bit 7 their parity (ggml's ksigns_iq2xs)
+__device__ __forceinline__ uint32_t ksigns8(uint32_t v) {
+ v &= 127u;
+ return v | ((__popc(v) & 1u) << 7);
+}
+// 4 grid bytes with the signs of bits 0..3 of b applied (bit i: byte i negative) - llama.cpp's HIP form, cheap where the
+// SIMD byte instructions (__vcmpne4 / __vsub4) are emulated (Volta, RDNA): every grid byte is nonzero and <= 62, so
+// (g ^ 0xff) + 1 = -g in each byte with no carry into the next
+__device__ __forceinline__ int sgn4(uint32_t g, uint32_t b) {
+ const uint32_t lo = (((b & 15u) * 0x10204080u) & 0x80808080u) >> 7;
+ return (int) ((g ^ (lo * 255u)) + lo);
+}
+__device__ __forceinline__ int2 tab16(uint32_t q4, const uint32_t* t) { // vecdotq's get_int_from_table_16
+ uint32_t tmp[2];
+ const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ const uint32_t sh = 16 * i;
+ const uint32_t lo = __byte_perm(t[0], t[1], q4 >> sh), hi = __byte_perm(t[2], t[3], q4 >> sh);
+ tmp[i] = __byte_perm(lo, hi, sel >> sh);
+ }
+ return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));
+}
+__device__ __forceinline__ int q3k_scale(const uint8_t* sc, int is) {
+ const int lo = is < 8 ? sc[is] & 15 : sc[is - 8] >> 4;
+ const int hi = (sc[8 + (is & 3)] >> (2 * (is >> 2))) & 3;
+ return (lo | (hi << 4)) - 32;
+}
+__device__ __forceinline__ void q4k_scale(const uint8_t* q, int j, int& d, int& m) { // ggml's get_scale_min_k4
+ if (j < 4) {
+ d = q[j] & 63;
+ m = q[j + 4] & 63;
+ } else {
+ d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
+ m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4);
+ }
+}
+
+// a 32-value sub-block (kb: its index in the row) as 32 int8 codes and its scales: weight i = d s q_i - m, s and m of
+// its half (i < 16: s0, m0; else s1, m1)
+struct Sub {
+ int q[8];
+ float d, m0, m1;
+ int s0, s1;
+};
+template <int T>
+__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {
+ const int ib = kb & 7;
+ const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;
+ if constexpr (T == kMXFP4) { // codes: value j in byte j's low nibble, j + 16 in its high one (any alignment)
+ const uint32_t t[4] = {0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu}; // ggml's kvalues_mxfp4
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const uint8_t* q = b + 1 + 4 * i;
+ const int2 v = tab16((uint32_t) q[0] | (uint32_t) q[1] << 8 | (uint32_t) q[2] << 16 | (uint32_t) q[3] << 24, t);
+ s.q[i] = v.x;
+ s.q[4 + i] = v.y;
+ }
+ const uint32_t e = b[0]; // 2^(e - 128): ggml's GGML_E8M0_TO_FP32_HALF (the codes are the E2M1 values doubled)
+ s.d = __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);
+ s.s0 = s.s1 = 1;
+ (void) ib;
+ (void) tab;
+ } else if constexpr (T == kIQ2_XXS) {
+ const uint8_t* q = b + 2 + 8 * ib;
+ const uint32_t a0 = ld2(q), a1 = ld2(q + 4);
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint2 g = ((const uint2*) tab)[(a0 >> (8 * l)) & 255u];
+ const uint32_t sg = ksigns8(a1 >> (7 * l));
+ s.q[2 * l] = sgn4(g.x, sg);
+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
+ }
+ s.s0 = s.s1 = 2 * (int) (a1 >> 28) + 1;
+ s.d = h2f(b) * 0.125f;
+ } else if constexpr (T == kIQ2_S) {
+ const uint8_t* qs = b + 2;
+ const uint32_t i4 = ld2(qs + 4 * ib), sg4 = ld2(qs + 32 + 4 * ib), h = b[66 + ib], sc = b[74 + ib];
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint2 g = ((const uint2*) tab)[((i4 >> (8 * l)) & 255u) | ((h << (8 - 2 * l)) & 0x300u)];
+ const uint32_t sg = sg4 >> (8 * l);
+ s.q[2 * l] = sgn4(g.x, sg);
+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);
+ }
+ s.s0 = 2 * (int) (sc & 15) + 1;
+ s.s1 = 2 * (int) (sc >> 4) + 1;
+ s.d = h2f(b) * 0.125f;
+ } else if constexpr (T == kIQ3_XXS) {
+ const uint8_t* qs = b + 2;
+ const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), aux = ld2(qs + 64 + 4 * ib);
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);
+ const uint32_t sg = ksigns8(aux >> (7 * l));
+ s.q[2 * l] = sgn4(tab[(w >> sh) & 255u], sg);
+ s.q[2 * l + 1] = sgn4(tab[(w >> (sh + 8)) & 255u], sg >> 4);
+ }
+ s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;
+ s.d = h2f(b) * 0.25f;
+ } else if constexpr (T == kIQ3_S) {
+ const uint8_t* qs = b + 2;
+ const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);
+#pragma unroll
+ for (int l = 0; l < 4; ++l) {
+ const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);
+ const uint32_t i1 = ((w >> sh) & 255u) | ((h << (8 - 2 * l)) & 256u);
+ const uint32_t i2 = ((w >> (sh + 8)) & 255u) | ((h << (7 - 2 * l)) & 256u);
+ const uint32_t sg = s4 >> (8 * l);
+ s.q[2 * l] = sgn4(tab[i1], sg);
+ s.q[2 * l + 1] = sgn4(tab[i2], sg >> 4);
+ }
+ s.s0 = s.s1 = 1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15);
+ s.d = h2f(b);
+ } else if constexpr (T == kIQ4_XS) {
+ const uint32_t sh = *(const uint16_t*) (b + 2);
+ const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);
+ const uint8_t* qs = b + 8 + 16 * ib;
+#pragma unroll
+ for (int t = 0; t < 4; ++t) {
+ const int2 v = tab16(ld4(qs + 4 * t), tab);
+ s.q[t] = v.x;
+ s.q[4 + t] = v.y;
+ }
+ s.s0 = s.s1 = ls - 32;
+ s.d = h2f(b);
+ } else if constexpr (T == kQ4_K) {
+ int sc, m;
+ q4k_scale(b + 4, ib, sc, m);
+ const uint8_t* qs = b + 16 + 32 * (ib >> 1);
+ const int sh = 4 * (ib & 1);
+#pragma unroll
+ for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu);
+ s.s0 = s.s1 = sc;
+ s.d = h2f(b);
+ s.m0 = s.m1 = h2f(b + 2) * (float) m;
+ } else if constexpr (T == kQ2_K) {
+ const int n = ib >> 2, j = ib & 3;
+ const uint8_t* qs = b + 16 + 32 * n;
+#pragma unroll
+ for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> (2 * j)) & 0x03030303u);
+ const int c0 = b[8 * n + 2 * j], c1 = b[8 * n + 2 * j + 1];
+ const float dmin = h2f(b + 82);
+ s.s0 = c0 & 15;
+ s.s1 = c1 & 15;
+ s.d = h2f(b + 80);
+ s.m0 = dmin * (float) (c0 >> 4);
+ s.m1 = dmin * (float) (c1 >> 4);
+ } else { // Q3_K
+ const int n = ib >> 2, j = ib & 3;
+ const uint8_t* qs = b + 32 + 32 * n;
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t ql = (ld2(qs + 4 * t) >> (2 * j)) & 0x03030303u;
+ const uint32_t neg = ((~ld2(b + 4 * t)) >> (4 * n + j)) & 0x01010101u;
+ s.q[t] = (int) (ql | (neg * 0xFCu)); // (q - 4 in a byte: q + 0xFC, no carry for q <= 3)
+ }
+ s.s0 = q3k_scale(b + 96, 8 * n + 2 * j);
+ s.s1 = q3k_scale(b + 96, 8 * n + 2 * j + 1);
+ s.d = h2f(b + 108);
+ }
+}
+// the sub-block against 32 int8 activations (xa: values 0..15, xb: 16..31), before their scale: an exact integer dot,
+// its scales in F32 (explicit roundings: every instantiation the same bits)
+template <int T>
+__device__ __forceinline__ float sub_dot(const Sub& s, const int4& xa, const int4& xb) {
+ int lo = __dp4a(s.q[0], xa.x, 0);
+ lo = __dp4a(s.q[1], xa.y, lo);
+ lo = __dp4a(s.q[2], xa.z, lo);
+ lo = __dp4a(s.q[3], xa.w, lo);
+ int hi = __dp4a(s.q[4], xb.x, 0);
+ hi = __dp4a(s.q[5], xb.y, hi);
+ hi = __dp4a(s.q[6], xb.z, hi);
+ hi = __dp4a(s.q[7], xb.w, hi);
+ float v = __fmul_rn(s.d, (float) (s.s0 * lo + s.s1 * hi));
+ if constexpr (Ty<T>::MIN) {
+ int sl = __dp4a(0x01010101, xa.x, 0);
+ sl = __dp4a(0x01010101, xa.y, sl);
+ sl = __dp4a(0x01010101, xa.z, sl);
+ sl = __dp4a(0x01010101, xa.w, sl);
+ int sh = __dp4a(0x01010101, xb.x, 0);
+ sh = __dp4a(0x01010101, xb.y, sh);
+ sh = __dp4a(0x01010101, xb.z, sh);
+ sh = __dp4a(0x01010101, xb.w, sh);
+ v = __fsub_rn(v, __fadd_rn(__fmul_rn(s.m0, (float) sl), __fmul_rn(s.m1, (float) sh)));
+ }
+ return v;
+}
+
+// values [8 p, 8 p + 8) of sub-block kb, as F32 (the prompt sets' tiles): ggml's dequantize_row_* values
+template <int T>
+__device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, int p, const uint32_t* tab, float* w) {
+ const int ib = kb & 7;
+ const uint8_t* b = row + (size_t) (kb >> 3) * Ty<T>::B;
+ if constexpr (T == kIQ2_XXS) {
+ const uint8_t* q = b + 2 + 8 * ib;
+ const uint32_t a1 = ld2(q + 4);
+ const uint2 g = ((const uint2*) tab)[q[p]];
+ const uint32_t v = (a1 >> (7 * p)) & 127u, sg = v | ((__popc(v) & 1u) << 7);
+ const float dl = h2f(b) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
+ }
+ } else if constexpr (T == kIQ2_S) {
+ const uint8_t* qs = b + 2;
+ const uint32_t idx = qs[4 * ib + p] | ((((uint32_t) b[66 + ib]) << (8 - 2 * p)) & 0x300u);
+ const uint32_t sg = qs[32 + 4 * ib + p], sc = b[74 + ib];
+ const uint2 g = ((const uint2*) tab)[idx];
+ const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
+ }
+ } else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {
+ const uint8_t* qs = b + 2;
+ uint32_t g1, g2, sg;
+ float dl;
+ if constexpr (T == kIQ3_XXS) {
+ const uint32_t aux = ld2(qs + 64 + 4 * ib), v = (aux >> (7 * p)) & 127u;
+ g1 = tab[qs[8 * ib + 2 * p]];
+ g2 = tab[qs[8 * ib + 2 * p + 1]];
+ sg = v | ((__popc(v) & 1u) << 7);
+ dl = h2f(b) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;
+ } else {
+ const uint32_t h = b[66 + ib];
+ g1 = tab[qs[8 * ib + 2 * p] | ((h << (8 - 2 * p)) & 256u)];
+ g2 = tab[qs[8 * ib + 2 * p + 1] | ((h << (7 - 2 * p)) & 256u)];
+ sg = b[74 + 4 * ib + p];
+ dl = h2f(b) * (float) (1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15));
+ }
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const float gv = (float) ((j < 4 ? g1 >> (8 * j) : g2 >> (8 * (j - 4))) & 255u);
+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
+ }
+ } else if constexpr (T == kIQ4_XS) {
+ const uint32_t sh = *(const uint16_t*) (b + 2);
+ const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);
+ const float dl = h2f(b) * (float) (ls - 32);
+ const uint8_t* qs = b + 8 + 16 * ib + 8 * (p & 1);
+ const int8_t* kv = (const int8_t*) tab;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];
+ } else if constexpr (T == kQ4_K) {
+ int sc, m;
+ q4k_scale(b + 4, ib, sc, m);
+ const float d1 = h2f(b) * (float) sc, m1 = h2f(b + 2) * (float) m;
+ const uint8_t* qs = b + 16 + 32 * (ib >> 1) + 8 * p;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) w[j] = d1 * (float) ((ib & 1) ? qs[j] >> 4 : qs[j] & 15) - m1;
+ } else if constexpr (T == kQ2_K) {
+ const int n = ib >> 2, jj = ib & 3;
+ const int c = b[8 * n + 2 * jj + (p >> 1)];
+ const float dl = h2f(b + 80) * (float) (c & 15), ml = h2f(b + 82) * (float) (c >> 4);
+ const uint8_t* qs = b + 16 + 32 * n + 8 * p;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) w[j] = dl * (float) ((qs[j] >> (2 * jj)) & 3) - ml;
+ } else { // Q3_K
+ const int n = ib >> 2, jj = ib & 3;
+ const float dl = h2f(b + 108) * (float) q3k_scale(b + 96, 8 * n + 2 * jj + (p >> 1));
+ const uint8_t* qs = b + 32 + 32 * n + 8 * p;
+ const uint8_t* hm = b + 8 * p;
+#pragma unroll
+ for (int j = 0; j < 8; ++j)
+ w[j] = dl * (float) ((int) ((qs[j] >> (2 * jj)) & 3) - (((hm[j] >> (4 * n + jj)) & 1) ? 0 : 4));
+ }
+}
+
+__device__ __forceinline__ float warp_sum(float v) {
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+ return v;
+}
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) { // (ds4_gguf.cu's)
+ if (limit > 0.0f) {
+ gv = fminf(gv, limit);
+ uv = fminf(fmaxf(uv, -limit), limit);
+ }
+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+__device__ __forceinline__ bool group(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {
+ if (a.grp_ptr == nullptr) {
+ const unsigned long long p = a.plan_ptr[grp];
+ if (p == 0ull) return false;
+ blob = (const uint8_t*) p;
+ m0 = grp;
+ m1 = grp + 1;
+ return true;
+ }
+ if (grp >= *a.n_groups) return false;
+ blob = (const uint8_t*) a.grp_ptr[grp];
+ m0 = a.grp_start[grp];
+ m1 = a.grp_start[grp + 1];
+ return true;
+}
+
+// ---------------------------------------------------------------- token / window experts
+// RW rows a warp, 32 / RW lanes a row (a lane a sub-block a step), their sums over those lanes
+template <int RW>
+__device__ __forceinline__ float lanes_sum(float v) {
+#pragma unroll
+ for (int o = 16 / RW; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+ return v;
+}
+constexpr int kGuRW = 1, kDnRW = 2; // rows a warp: gate / up (n_embd 5120: 10 sub-blocks a lane), down (2304: 9)
+
+// grid (n_ff / (8 RW), groups): RW rows of the gate and of the up a warp, NE of the group's entries a pass (each
+// sub-block decoded once for them) -> h = the clamped swiglu
+template <int T, int NE, int RW = kGuRW>
+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {
+ constexpr int LPR = 32 / RW;
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ int es[NE], xr[NE];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!group(a, blockIdx.y, blob, m0, m1)) return; // (uniform)
+ load_tab<T>(tab);
+ const int lane32 = threadIdx.x & 31;
+ const int row = (blockIdx.x * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
+ const size_t rb = row_bytes_t<T>(a.n_embd);
+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;
+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;
+ const int nsb = a.n_embd / 32;
+ for (int mm = m0; mm < m1; mm += NE) {
+ const int ne = min(NE, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) {
+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ es[threadIdx.x] = e;
+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;
+ }
+ __syncthreads();
+ if (row >= a.n_ff) continue; // (a whole warp: n_ff % (8 RW) == 0)
+ float ag[NE], au[NE];
+#pragma unroll
+ for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;
+ for (int kb = lane; kb < nsb; kb += LPR) {
+ Sub sg, su;
+ decode<T>(g, kb, tab, sg);
+ decode<T>(u, kb, tab, su);
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) kb * 32);
+ const int4 xa = xp[0], xb = xp[1];
+ const float dx = a.xd[(size_t) xr[i] * nsb + kb];
+ ag[i] = fmaf(dx, sub_dot<T>(sg, xa, xb), ag[i]);
+ au[i] = fmaf(dx, sub_dot<T>(su, xa, xb), au[i]);
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const float gv = lanes_sum<RW>(ag[i]), uv = lanes_sum<RW>(au[i]);
+ if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);
+ }
+ }
+}
+
+// grid (n_embd / (8 RW), groups): RW rows of the down a warp for NE entries a pass, against their int8 h -> part
+template <int T, int NE, int RW = kDnRW>
+__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {
+ constexpr int LPR = 32 / RW;
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ int es[NE];
+ const uint8_t* blob;
+ int m0, m1;
+ if (!group(a, blockIdx.y, blob, m0, m1)) return;
+ load_tab<T>(tab);
+ const int lane32 = threadIdx.x & 31;
+ const int row = (blockIdx.x * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;
+ const uint8_t* d = blob + a.tr[2] + (size_t) row * row_bytes_t<T>(a.n_ff);
+ const int nsb = a.n_ff / 32;
+ for (int mm = m0; mm < m1; mm += NE) {
+ const int ne = min(NE, m1 - mm);
+ __syncthreads();
+ if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;
+ __syncthreads();
+ if (row >= a.n_embd) continue;
+ float ad[NE];
+#pragma unroll
+ for (int i = 0; i < NE; ++i) ad[i] = 0.0f;
+ for (int kb = lane; kb < nsb; kb += LPR) {
+ Sub sd;
+ decode<T>(d, kb, tab, sd);
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) kb * 32);
+ const int4 ha = hp[0], hb = hp[1];
+ ad[i] = fmaf(a.hd[(size_t) es[i] * nsb + kb], sub_dot<T>(sd, ha, hb), ad[i]);
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const float v = lanes_sum<RW>(ad[i]);
+ if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;
+ }
+ }
+}
+
+// ---------------------------------------------------------------- the prompt path's sets
+constexpr int BM = 64, BN = 64, TM = 4, TN = 4;
+static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");
+
+// F32 FMAs (any device): a block RM rows (64, or 16 for experts of a few rows: a short prompt's) x BN outputs, a 32-value
+// sub-block of K a step, the weights dequantised into shared memory (each thread 8 values of a row), each thread RT x TN
+// outputs. (deq8 is the tile loader a tensor-core form takes too.)
+template <int T, bool GATE_UP, int RM = BM>
+__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {
+ constexpr int RT = RM / (kThreads / (BN / TN));
+ static_assert(RT >= 1 && RT * (kThreads / (BN / TN)) == RM, "the block's rows");
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ float wg[32][BN + 1];
+ __shared__ float wu[GATE_UP ? 32 : 1][BN + 1];
+ __shared__ float xs[RM][33];
+ __shared__ int src[RM];
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * RM;
+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
+ load_tab<T>(tab);
+ const int nr = min(RM, hi - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GATE_UP ? a.n_embd : a.n_ff;
+ const size_t rb = row_bytes_t<T>(K);
+ const int tid = (int) threadIdx.x;
+ if (tid < RM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
+ __syncthreads();
+ const int ty = tid / (BN / TN), tx = tid % (BN / TN);
+ const int tr = tid >> 2, tq = tid & 3;
+ float ag[RT][TN], au[RT][TN];
+#pragma unroll
+ for (int i = 0; i < RT; ++i)
+#pragma unroll
+ for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;
+ for (int kb = 0; kb < K / 32; ++kb) {
+ float w8[8];
+ deq8<T>(blob + a.tr[GATE_UP ? 0 : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) wg[tq * 8 + j][tr] = w8[j];
+ if (GATE_UP) {
+ deq8<T>(blob + a.tr[1] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) wu[tq * 8 + j][tr] = w8[j];
+ }
+ for (int i = tid; i < RM * 32; i += kThreads) {
+ const int r = i / 32, c = i % 32, sr = src[r];
+ float v = 0.0f;
+ if (sr >= 0) {
+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + c;
+ v = GATE_UP ? a.xf[at] : __half2float(__ushort_as_half(a.xh[at]));
+ }
+ xs[r][c] = v;
+ }
+ __syncthreads();
+#pragma unroll 8
+ for (int kk = 0; kk < 32; ++kk) {
+ float xv[RT], gw[TN], uw[TN];
+#pragma unroll
+ for (int i = 0; i < RT; ++i) xv[i] = xs[ty * RT + i][kk];
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ gw[j] = wg[kk][tx * TN + j];
+ if (GATE_UP) uw[j] = wu[kk][tx * TN + j];
+ }
+#pragma unroll
+ for (int i = 0; i < RT; ++i)
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);
+ if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);
+ }
+ }
+ __syncthreads();
+ }
+#pragma unroll
+ for (int i = 0; i < RT; ++i) {
+ const int r = ty * RT + i;
+ if (r >= nr) continue;
+ const size_t lr = (size_t) (r0 + r);
+#pragma unroll
+ for (int j = 0; j < TN; ++j) {
+ const int c = c0 + tx * TN + j;
+ if (GATE_UP) a.h[lr * a.ldo + c] = __half_as_ushort(__float2half_rn(swiglu(ag[i][j], au[i][j], a.limit)));
+ else a.y[lr * a.ldo + c] = ag[i][j];
+ }
+ }
+}
+
+#if !defined(STRATA_USE_HIP)
+// the same on the tensor cores (sm_70+): 8 warps each 16 rows x 32 outputs, the tiles F16 in shared memory
+namespace wm = nvcuda::wmma;
+constexpr int kLdT = 40;
+template <int T, bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) set_tc_kernel(const SetArgs a) {
+ struct Tiles {
+ __half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];
+ };
+ struct Out {
+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];
+ };
+ __shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ int src[BM];
+ Tiles& Tl = *reinterpret_cast<Tiles*>(smem);
+ Out& O = *reinterpret_cast<Out*>(smem);
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)
+ load_tab<T>(tab);
+ const int nr = min(BM, hi - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GATE_UP ? a.n_embd : a.n_ff;
+ const size_t rb = row_bytes_t<T>(K);
+ const int tid = (int) threadIdx.x, warp = tid >> 5;
+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;
+ __syncthreads();
+ const int wm_ = warp >> 1, wn = (warp & 1) * 2;
+ wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];
+ for (int j = 0; j < 2; ++j) {
+ wm::fill_fragment(fg[j], 0.0f);
+ wm::fill_fragment(fu[j], 0.0f);
+ }
+ const int tr = tid >> 2, tq = tid & 3;
+ for (int kb = 0; kb < K / 32; ++kb) {
+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {
+ float w8[8];
+ deq8<T>(blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);
+ __half* dst = m == 0 ? &Tl.wg[tr][tq * 8] : &Tl.wu[tr][tq * 8];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) dst[j] = __float2half_rn(w8[j]);
+ }
+ const int sr = src[tr];
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ __half v = __float2half_rn(0.0f);
+ if (sr >= 0) {
+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + tq * 8 + j;
+ v = GATE_UP ? __float2half_rn(a.xf[at]) : __ushort_as_half(a.xh[at]);
+ }
+ Tl.x[tr][tq * 8 + j] = v;
+ }
+ __syncthreads();
+#pragma unroll
+ for (int kk = 0; kk < 32; kk += 16) {
+ wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;
+ wm::load_matrix_sync(fa, &Tl.x[wm_ * 16][kk], kLdT);
+#pragma unroll
+ for (int j = 0; j < 2; ++j) {
+ wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;
+ wm::load_matrix_sync(fb, &Tl.wg[(wn + j) * 16][kk], kLdT);
+ wm::mma_sync(fg[j], fa, fb, fg[j]);
+ if (GATE_UP) {
+ wm::load_matrix_sync(fb, &Tl.wu[(wn + j) * 16][kk], kLdT);
+ wm::mma_sync(fu[j], fa, fb, fu[j]);
+ }
+ }
+ }
+ __syncthreads();
+ }
+ for (int j = 0; j < 2; ++j) {
+ wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);
+ if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);
+ }
+ __syncthreads();
+ for (int i = tid; i < BM * BN; i += kThreads) {
+ const int r = i / BN, c = i % BN;
+ if (r >= nr) continue;
+ const size_t lr = (size_t) (r0 + r);
+ if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __half_as_ushort(__float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit)));
+ else a.y[lr * a.ldo + c0 + c] = O.g[r][c];
+ }
+}
+#endif
+
+// ---------------------------------------------------------------- GEMVs
+constexpr int kMaxJobs = 8;
+struct Jobs {
+ MvJob j[kMaxJobs];
+};
+template <int T, int NC>
+__global__ void __launch_bounds__(kThreads) mv_kernel(const Jobs J) {
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ load_tab<T>(tab);
+ __syncthreads();
+ const MvJob& b = J.j[blockIdx.y];
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (row >= b.n) return;
+ const uint8_t* w = b.w + (size_t) row * b.ld;
+ const int nsb = b.k / 32;
+ float acc[NC];
+#pragma unroll
+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (int kb = lane; kb < nsb; kb += 32) {
+ Sub s;
+ decode<T>(w, kb, tab, s);
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);
+ acc[c] = fmaf(b.xd[(size_t) c * nsb + kb], sub_dot<T>(s, xp[0], xp[1]), acc[c]);
+ }
+ }
+#pragma unroll
+ for (int c = 0; c < NC; ++c)
+ if (c < b.ncol) {
+ const float v = warp_sum(acc[c]);
+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- the prompt sets' light experts
+// A set's experts of a few rows (<= kLightRows: a prompt of a few hundred tokens spreads ~10 over most of 384 experts)
+// in the token kernels' form: each weight sub-block decoded once and dotted with every row's int8 input (rounded per 32
+// values as the token path rounds them) - the weights read once at the memory's speed, where a 64-row GEMM tile did 64
+// rows' work for them. Each expert by its own rows: the light kernels take the experts of <= kLightRows rows, the tiles
+// the others (SetArgs::skip_le: they skip the light ones) - every expert's sums in one fixed order whatever the set.
+// Expert e's row i is input row e kLightRows + i of the scratch, output (local) row bounds[e] - row0 + i.
+constexpr int kLightRows = 16;
+constexpr int kLightMaxExp = 512; // a set's experts at most (the scratch: kLightMaxExp kLightRows rows)
+
+// grid (k / 32 / 8, n_exp kLightRows): the light experts' rows of k values (gate / up: F32 x[idx[lr]]; down: F16 h[lr],
+// ldx apart) -> int8 q[e kLightRows + i][k], scales d[..][k / 32]
+__global__ void __launch_bounds__(kThreads) light_q8_kernel(const float* __restrict__ xf, const uint16_t* __restrict__ xh,
+ const int* __restrict__ idx, int ldx, int k,
+ const int* __restrict__ bounds, int row0,
+ int8_t* __restrict__ q, float* __restrict__ d) {
+ const int e = blockIdx.y / kLightRows, i = blockIdx.y % kLightRows;
+ const int rows = bounds[e + 1] - bounds[e];
+ const int b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (i >= rows || rows > kLightRows || b * 32 >= k) return;
+ const int lr = bounds[e] - row0 + i;
+ const size_t at = (size_t) b * 32 + lane, qr = (size_t) blockIdx.y;
+ const float v = xf != nullptr ? xf[(size_t) idx[lr] * ldx + at] : __half2float(__ushort_as_half(xh[(size_t) lr * ldx + at]));
+ float am = fabsf(v);
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+ const float sc = am / 127.0f; // (ds4_gguf.cu's q8_quant: the same operations)
+ q[qr * k + at] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
+ if (lane == 0) d[qr * (k / 32) + b] = sc;
+}
+
+// grid (n_ff / 8 or n_embd / 8, n_exp): a light expert's rows, a warp a row of the weights (gate and up, or down), every
+// row of the expert a pass -> h16 (the F16 swiglu) or y
+template <int T, int NE, bool GATE_UP>
+__global__ void __launch_bounds__(kThreads) light_kernel(const SetArgs a, const int8_t* __restrict__ xq,
+ const float* __restrict__ xd) {
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ const int e = blockIdx.y;
+ const int lo = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];
+ if (ne <= 0 || ne > NE) return; // (uniform: the tiles take an expert of more rows)
+ load_tab<T>(tab);
+ __syncthreads();
+ const int K = GATE_UP ? a.n_embd : a.n_ff, N = GATE_UP ? a.n_ff : a.n_embd, nsb = K / 32;
+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ if (row >= N) return;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const size_t rb = row_bytes_t<T>(K);
+ const uint8_t* w0 = blob + a.tr[GATE_UP ? 0 : 2] + (size_t) row * rb;
+ const uint8_t* w1 = blob + a.tr[1] + (size_t) row * rb;
+ const size_t q0 = (size_t) e * kLightRows;
+ float a0[NE], a1[NE];
+#pragma unroll
+ for (int i = 0; i < NE; ++i) a0[i] = a1[i] = 0.0f;
+ for (int kb = lane; kb < nsb; kb += 32) {
+ Sub s0, s1;
+ decode<T>(w0, kb, tab, s0);
+ if (GATE_UP) decode<T>(w1, kb, tab, s1);
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const int4* xp = (const int4*) (xq + (q0 + i) * K + (size_t) kb * 32);
+ const int4 xa = xp[0], xb = xp[1];
+ const float dx = xd[(q0 + i) * nsb + kb];
+ a0[i] = fmaf(dx, sub_dot<T>(s0, xa, xb), a0[i]);
+ if (GATE_UP) a1[i] = fmaf(dx, sub_dot<T>(s1, xa, xb), a1[i]);
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < NE; ++i)
+ if (i < ne) {
+ const float v0 = warp_sum(a0[i]);
+ const float v1 = GATE_UP ? warp_sum(a1[i]) : 0.0f;
+ if (lane == 0) {
+ const size_t lr = (size_t) (lo + i);
+ if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(v0, v1, a.limit)));
+ else a.y[lr * a.ldo + row] = v0;
+ }
+ }
+}
+
+bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps the F32 kernel
+#if defined(STRATA_USE_HIP)
+ return false;
+#else
+ static const bool ok = [] {
+ int dev = 0, major = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ const char* v = std::getenv("STRATA_DS4_NO_TC");
+ return major >= 7 && !(v != nullptr && v[0] == '1');
+ }();
+ return ok;
+#endif
+}
+
+// ---------------------------------------------------------------- dispatch
+// F(T) for the type t at run time; false when it is not one of these
+#define DS4_IQ_SWITCH(t, F) \
+ switch (t) { \
+ case kIQ2_XXS: F(kIQ2_XXS); break; \
+ case kIQ2_S: F(kIQ2_S); break; \
+ case kIQ3_XXS: F(kIQ3_XXS); break; \
+ case kIQ3_S: F(kIQ3_S); break; \
+ case kIQ4_XS: F(kIQ4_XS); break; \
+ case kQ4_K: F(kQ4_K); break; \
+ case kQ2_K: F(kQ2_K); break; \
+ case kQ3_K: F(kQ3_K); break; \
+ default: return false; \
+ }
+
+template <int T, int NE>
+void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { gate_up_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }
+template <int T, int NE>
+void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { down_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }
+template <int T>
+void gu_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {
+ if (ne <= 1) launch_gu<T, 1>(a, grid, s);
+ else if (ne <= 4) launch_gu<T, 4>(a, grid, s);
+ else if (ne <= 8) launch_gu<T, 8>(a, grid, s);
+ else launch_gu<T, 16>(a, grid, s);
+}
+template <int T>
+void dn_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {
+ if (ne <= 1) launch_dn<T, 1>(a, grid, s);
+ else if (ne <= 4) launch_dn<T, 4>(a, grid, s);
+ else if (ne <= 8) launch_dn<T, 8>(a, grid, s);
+ else launch_dn<T, 16>(a, grid, s);
+}
+template <int T, bool GU>
+void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {
+#if !defined(STRATA_USE_HIP)
+ if (tc_ok()) {
+ set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);
+ return;
+ }
+#endif
+ if (a.max_rows <= 16) { // (a block of 64 rows would do 64 rows' FMAs for the few there are)
+ grid.x = (unsigned) ((a.max_rows + 15) / 16);
+ set_kernel<T, GU, 16><<<grid, kThreads, 0, s>>>(a);
+ return;
+ }
+ set_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);
+}
+// the light sets' int8 rows, per stream (kLightMaxExp kLightRows rows of k)
+struct LightBuf {
+ int8_t* q = nullptr;
+ float* d = nullptr;
+ size_t cap = 0;
+};
+std::mutex g_lmu;
+std::unordered_map<cudaStream_t, LightBuf> g_light;
+LightBuf* light_buf(cudaStream_t s, int k) {
+ std::lock_guard<std::mutex> lk(g_lmu);
+ LightBuf& b = g_light[s];
+ const size_t need = (size_t) kLightMaxExp * kLightRows * (size_t) std::max(k, 8192);
+ if (b.cap < need) {
+ if (b.q != nullptr) cudaFree(b.q);
+ if (b.d != nullptr) cudaFree(b.d);
+ b = LightBuf{};
+ if (cudaMalloc(&b.q, need) != cudaSuccess || cudaMalloc(&b.d, need / 32 * sizeof(float)) != cudaSuccess) {
+ std::fprintf(stderr, "ds4 iq: the light sets' scratch (%zu MB) failed\n", need >> 20);
+ return nullptr;
+ }
+ b.cap = need;
+ }
+ return &b;
+}
+bool light_on() { // STRATA_DS4_NO_LIGHT=1: every set through the GEMM tiles
+ static const bool on = [] {
+ const char* v = std::getenv("STRATA_DS4_NO_LIGHT");
+ return !(v != nullptr && v[0] == '1');
+ }();
+ return on;
+}
+template <int T, bool GU>
+void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {
+ const dim3 grid((unsigned) (((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps), (unsigned) a.n_exp);
+ if (a.max_rows <= 4) light_kernel<T, 4, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);
+ else if (a.max_rows <= 8) light_kernel<T, 8, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);
+ else light_kernel<T, kLightRows, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);
+}
+bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {
+ const int K = gu ? a.n_embd : a.n_ff;
+ LightBuf* b = light_buf(s, K);
+ if (b == nullptr) return false;
+ const dim3 gq((unsigned) ((K / 32 + kWarps - 1) / kWarps), (unsigned) (a.n_exp * kLightRows));
+ light_q8_kernel<<<gq, kThreads, 0, s>>>(gu ? a.xf : nullptr, gu ? nullptr : a.xh, a.idx, a.ldx, K, a.bounds,
+ a.row0, b->q, b->d);
+ if (gu) {
+ if (a.tg == kMXFP4) {
+ launch_light<kMXFP4, true>(a, *b, s);
+ return true;
+ }
+#define LG_(T) launch_light<T, true>(a, *b, s)
+ DS4_IQ_SWITCH(a.tg, LG_)
+#undef LG_
+ } else {
+ if (a.td == kMXFP4) {
+ launch_light<kMXFP4, false>(a, *b, s);
+ return true;
+ }
+#define LD_(T) launch_light<T, false>(a, *b, s)
+ DS4_IQ_SWITCH(a.td, LD_)
+#undef LD_
+ }
+ return true;
+}
+
+template <int T>
+void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
+ const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);
+ if (max_c <= 1) mv_kernel<T, 1><<<grid, kThreads, 0, s>>>(J);
+ else if (max_c <= 4) mv_kernel<T, 4><<<grid, kThreads, 0, s>>>(J);
+ else mv_kernel<T, 16><<<grid, kThreads, 0, s>>>(J);
+}
+
+} // namespace
+
+bool supported(int t) { return block_bytes(t) > 0; }
+
+int block_bytes(int t) {
+ switch (t) {
+ case kIQ2_XXS: return Ty<kIQ2_XXS>::B;
+ case kIQ2_S: return Ty<kIQ2_S>::B;
+ case kIQ3_XXS: return Ty<kIQ3_XXS>::B;
+ case kIQ3_S: return Ty<kIQ3_S>::B;
+ case kIQ4_XS: return Ty<kIQ4_XS>::B;
+ case kQ4_K: return Ty<kQ4_K>::B;
+ case kQ2_K: return Ty<kQ2_K>::B;
+ case kQ3_K: return Ty<kQ3_K>::B;
+ default: return 0;
+ }
+}
+
+const char* name(int t) {
+ switch (t) {
+ case kIQ2_XXS: return "IQ2_XXS";
+ case kIQ2_S: return "IQ2_S";
+ case kIQ3_XXS: return "IQ3_XXS";
+ case kIQ3_S: return "IQ3_S";
+ case kIQ4_XS: return "IQ4_XS";
+ case kQ4_K: return "Q4_K";
+ case kQ2_K: return "Q2_K";
+ case kQ3_K: return "Q3_K";
+ default: return "?";
+ }
+}
+
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;
+ const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);
+#define GU_(T) gu_ne<T>(a, grid, ne_max, s)
+ DS4_IQ_SWITCH(a.tg, GU_)
+#undef GU_
+ return true;
+}
+
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;
+ const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);
+#define DN_(T) dn_ne<T>(a, grid, ne_max, s)
+ DS4_IQ_SWITCH(a.td, DN_)
+#undef DN_
+ return true;
+}
+
+bool light_gate_up(const SetArgs& a, cudaStream_t s) {
+ if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
+ return light_set(a, true, s);
+}
+bool light_down(const SetArgs& a, cudaStream_t s) {
+ if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;
+ return light_set(a, false, s);
+}
+
+bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+ if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;
+ SetArgs t = a;
+ if (light_gate_up(a, s)) { // the experts of a few rows; the tiles the others
+ if (a.max_rows <= kLightRows) return true;
+ t.skip_le = kLightRows;
+ }
+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_ff / BN), (unsigned) a.n_exp);
+#define SG_(T) launch_set<T, true>(t, grid, s)
+ DS4_IQ_SWITCH(a.tg, SG_)
+#undef SG_
+ return true;
+}
+
+bool set_down(const SetArgs& a, cudaStream_t s) {
+ if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;
+ SetArgs t = a;
+ if (light_down(a, s)) {
+ if (a.max_rows <= kLightRows) return true;
+ t.skip_le = kLightRows;
+ }
+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_embd / BN), (unsigned) a.n_exp);
+#define SD_(T) launch_set<T, false>(t, grid, s)
+ DS4_IQ_SWITCH(a.td, SD_)
+#undef SD_
+ return true;
+}
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+ // the jobs by type, up to kMaxJobs a launch
+ bool done[64] = {};
+ if (n > 64) return false;
+ for (int i = 0; i < n; ++i)
+ while (!done[i]) {
+ const int t = jobs[i].type;
+ Jobs J{};
+ int nj = 0, max_n = 0, max_c = 1;
+ for (int k = i; k < n && nj < kMaxJobs; ++k) {
+ if (done[k] || jobs[k].type != t) continue;
+ if (jobs[k].k % 256 != 0 || jobs[k].ncol < 1 || jobs[k].ncol > 16) return false;
+ J.j[nj++] = jobs[k];
+ max_n = std::max(max_n, jobs[k].n);
+ max_c = std::max(max_c, jobs[k].ncol);
+ done[k] = true;
+ }
+#define MV_(T) mv_nc<T>(J, nj, max_n, max_c, s)
+ DS4_IQ_SWITCH(t, MV_)
+#undef MV_
+ }
+ return true;
+}
+
+} // namespace strata::ds4::gguf::iq
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
new file mode 100644
index 000000000..1b4486d69
--- /dev/null
+++ b/src/dsv41/ds4_iq.cuh
@@ -0,0 +1,102 @@
+// src/ds4/ds4_iq.cuh - a GGUF's routed experts in ggml's low-bit block types on the device for maya-ds4: the i-quants
+// IQ2_XXS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS and the K-quants Q2_K, Q3_K, Q4_K (256 values a block, ggml's layouts:
+// ggml-common.h, llama.cpp MIT). ds4_gguf.cu sends an expert layer here when its experts are not MXFP4
+// (x3::ExpertLayout::fmt kFmtMXFP4 with K[r] = role r's ggml type; K[r] == kTypeMXFP4: ds4_gguf.cu's own kernels).
+//
+// Every weight is decoded exactly (ggml's dequantize_row_*: a block scale times an integer scale times a grid or code
+// value) and every product takes the activations rounded to int8 per 32 values with an F32 scale (ds4_gguf.cu's
+// q8_quant): a 32-value sub-block's sum is an exact integer dot (dp4a) times its scales in F32 - no integer division as
+// llama.cpp's vecdotq takes (its sums truncate), so these are the double reference's sums to F32 rounding. A window's
+// entry gets the token path's bits (the same kernel, the same order).
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4::gguf::iq {
+
+// ggml type ids
+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,
+ kIQ4_XS = 23;
+
+/// Whether the device kernels take this ggml type.
+bool supported(int type);
+/// Bytes of a block (256 values) of the type (0: not one of these).
+int block_bytes(int type);
+/// Bytes of a row of k values (k % 256 == 0).
+inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / 256) * (size_t) block_bytes(type); }
+/// The type's name ("?" when unknown).
+const char* name(int type);
+
+/// The token / window experts (ds4_gguf.cu's MxArgs, with the roles' types): group g = an expert (grp_ptr null: expert g
+/// of the plan, plan_ptr[g] its blob or 0, its entry g, input row 0; else blob grp_ptr[g], entries grp_ent[grp_start[g]
+/// ..], entry e's input row e >> 3). x / h as int8 rows with an F32 scale per 32 (xq, xd: rows of n_embd; hq, hd: row e
+/// = entry e, rows of n_ff).
+struct Args {
+ const unsigned long long* plan_ptr = nullptr;
+ const unsigned long long* grp_ptr = nullptr;
+ const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+ size_t tr[3] = {0, 0, 0}; // the roles' offsets in a blob
+ int n_embd = 0, n_ff = 0;
+ float limit = 0.0f;
+ const int8_t* xq = nullptr;
+ const float* xd = nullptr;
+ const int8_t* hq = nullptr;
+ const float* hd = nullptr;
+ float* h = nullptr; // [entries][n_ff]: the clamped swiglu
+ float* part = nullptr; // [entries][n_embd]: each entry's down output
+ int tg = 0, td = 0; // gate / up's type, down's
+};
+/// grid (n_ff / 8, groups): h of every entry (ne_max: a group's most entries). false: a type not covered.
+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);
+/// grid (n_embd / 8, groups): part of every entry.
+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);
+
+/// The prompt path's expert sets (ds4_gguf.cu's SetK): expert i's blob at wbase + i stride owns the sorted rows
+/// [bounds[i], bounds[i + 1]) (local row = sorted row - row0); gate / up: h16[local] = F16 swiglu over x[idx[local]]
+/// (F32 rows, ldx); down: y[local] = down(h16[local]) (F16 rows in, ldx; F32 out, ldo). A set's experts of at most 16
+/// rows take the token kernels' form (each weight read once, the rows' inputs rounded to int8 per 32 as the token path
+/// rounds them; STRATA_DS4_NO_LIGHT=1: never), the others GEMM tiles (tensor cores on CUDA, F16 operands).
+struct SetArgs {
+ const uint8_t* wbase = nullptr;
+ size_t stride = 0;
+ size_t tr[3] = {0, 0, 0};
+ const int* bounds = nullptr;
+ int row0 = 0, max_rows = 0, n_exp = 0;
+ int n_embd = 0, n_ff = 0;
+ float limit = 0.0f;
+ const float* xf = nullptr;
+ const int* idx = nullptr;
+ const uint16_t* xh = nullptr;
+ int ldx = 0;
+ uint16_t* h = nullptr;
+ float* y = nullptr;
+ int ldo = 0;
+ int tg = 0, td = 0;
+ int skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them
+};
+bool set_gate_up(const SetArgs& a, cudaStream_t s);
+bool set_down(const SetArgs& a, cudaStream_t s);
+/// The light form alone (the set's experts of at most 16 rows; the others untouched), for any of these types and MXFP4
+/// (tg / td 39: ds4_gguf.cu's experts): false (nothing launched) when it is switched off - the caller's tiles take every
+/// expert then; true: they take those of more rows (skip_le 16).
+bool light_gate_up(const SetArgs& a, cudaStream_t s);
+bool light_down(const SetArgs& a, cudaStream_t s);
+
+/// GEMV jobs on such matrices (an expert's views on the host-routed path): y[c][r] = alpha (row r . x_c) + bias[r] for
+/// ncol activations, x as int8 rows (xq + c k, scales xd + c k / 32).
+struct MvJob {
+ const uint8_t* w = nullptr;
+ size_t ld = 0; // bytes a row
+ int type = 0, k = 0, n = 0, ncol = 1;
+ const int8_t* xq = nullptr;
+ const float* xd = nullptr;
+ float* y = nullptr;
+ size_t ycs = 0;
+ const float* bias = nullptr;
+ float alpha = 1.0f;
+};
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+
+} // namespace strata::ds4::gguf::iq
diff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cpp
new file mode 100644
index 000000000..b2d8f2409
--- /dev/null
+++ b/src/dsv41/ds4_iq_cpu.cpp
@@ -0,0 +1,766 @@
+// src/ds4/ds4_iq_cpu.cpp - a GGUF's low-bit experts on the CPU lane (see ds4_gguf_cpu.hpp): the GLM CPU lane's native
+// experts (src/kernels/cpu/native_expert.cpp), whose arithmetic is ggml-cpu's - and for IQ3_XXS / IQ3_S (Maya-S's
+// down, Maya-M's gate / up / down) rows of our own, the same numbers bit for bit, faster.
+#include "ds4_gguf_cpu.hpp"
+
+#if defined(STRATA_NATIVE_EXPERTS)
+#include "strata/kernels/cpu/native_expert.hpp"
+
+#define GGML_COMMON_DECL_CPP
+#define GGML_COMMON_IMPL_CPP
+#include "ggml-common.h"
+#include "ggml-cpu.h"
+#include "ggml.h"
+#endif
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define IQ_X86 1
+#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+#if defined(IQ_X86) && (defined(__GNUC__) || defined(__clang__))
+#define IQ_AVX2 __attribute__((target("avx2,fma")))
+#else
+#define IQ_AVX2
+#endif
+
+namespace strata::ds4::gguf {
+
+#if defined(STRATA_NATIVE_EXPERTS)
+namespace kc = strata::kernels::cpu;
+namespace {
+kc::NativeFmt native(const IqCpu& f) { // (the fields the rows read; no lookups)
+ kc::NativeFmt n;
+ n.gu_type = f.tg;
+ n.d_type = f.td;
+ n.gu_act = f.gu_act;
+ n.d_act = f.d_act;
+ n.n_embd = f.n_embd;
+ n.n_ff = f.n_ff;
+ n.gu_row = f.gu_row;
+ n.d_row = f.d_row;
+ n.up_off = f.gu_row * (size_t) f.n_ff;
+ n.down_off = 2 * n.up_off;
+ n.bytes = n.down_off + f.d_row * (size_t) f.n_embd;
+ n.act_bytes = f.act_bytes;
+ n.h_bytes = f.h_bytes;
+ return n;
+}
+
+// ---- what this CPU runs: AVX2 + FMA (the OS saving the YMM state), and AVX-512 F / BW + VBMI with BMI2 (the ZMM state)
+struct IqCpuIsa {
+ bool avx2 = false, w512 = false;
+};
+IqCpuIsa iq_isa() {
+ IqCpuIsa c;
+#if defined(IQ_X86)
+ unsigned r[4] = {0, 0, 0, 0};
+ auto cpuid = [&](unsigned leaf, unsigned sub) {
+#if defined(_MSC_VER)
+ int x[4];
+ __cpuidex(x, (int) leaf, (int) sub);
+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+#endif
+ };
+ cpuid(0, 0);
+ if (r[0] < 7) return c;
+ cpuid(1, 0);
+ if (!((r[2] >> 12) & 1u) || !((r[2] >> 27) & 1u) || !((r[2] >> 28) & 1u)) return c; // FMA, OSXSAVE, AVX
+#if defined(_MSC_VER)
+ const unsigned long long xcr0 = _xgetbv(0);
+#else
+ unsigned lo = 0, hi = 0;
+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;
+#endif
+ if ((xcr0 & 0x6) != 0x6) return c;
+ cpuid(7, 0);
+ c.avx2 = (r[1] >> 5) & 1u;
+ c.w512 = c.avx2 && (xcr0 & 0xE6) == 0xE6 && ((r[1] >> 16) & 1u) && ((r[1] >> 30) & 1u) && ((r[2] >> 1) & 1u) &&
+ ((r[1] >> 8) & 1u); // AVX-512 F, BW, VBMI; BMI2
+#endif
+ return c;
+}
+const IqCpuIsa& isa() {
+ static const IqCpuIsa c = iq_isa();
+ return c;
+}
+
+// ---- IQ3_XXS / IQ3_S rows of our own. ggml-cpu's AVX2 dots build each 32-value chunk's grid magnitudes from eight
+// scalar table loads and inserts, and redo it for every token; here the grid comes in with one vpgatherdd from the
+// same table, the signs are built in registers (the GLM rows' sgn7_vec / sgn_vec), and every token of a call applies
+// the decoded chunk. The arithmetic is ggml's to the bit: the same exact int32 lanes per block (lane l: the chunks'
+// grid entries l, each times its 2 s + 1), then for each block fma(d_x d_y, lanes, acc) and hsum_float_8 (IQ3_XXS:
+// times 0.25) - checked against ggml-cpu's own dot once per type (iq3_check), else ggml's rows are used.
+inline float h2f(uint16_t h) { // F16 -> F32, exactly
+ const uint32_t s = (uint32_t) (h & 0x8000u) << 16, e = (h >> 10) & 0x1fu, m = h & 0x3ffu;
+ uint32_t b;
+ if (e == 0) {
+ const float f = (float) m * 5.9604644775390625e-8f; // m 2^-24 (exact)
+ std::memcpy(&b, &f, 4);
+ b |= s;
+ } else if (e == 31) {
+ b = s | 0x7f800000u | (m << 13);
+ } else {
+ b = s | ((e + 112u) << 23) | (m << 13);
+ }
+ float f;
+ std::memcpy(&f, &b, 4);
+ return f;
+}
+inline uint32_t u32(const uint8_t* p) {
+ uint32_t v;
+ std::memcpy(&v, p, 4);
+ return v;
+}
+
+#if defined(IQ_X86)
+IQ_AVX2 inline float hsum_ggml(__m256 x) { // ggml's hsum_float_8, the same additions in the same order
+ __m128 r = _mm256_extractf128_ps(x, 1);
+ r = _mm_add_ps(r, _mm256_castps256_ps128(x));
+ r = _mm_add_ps(r, _mm_movehl_ps(r, r));
+ r = _mm_add_ss(r, _mm_movehdup_ps(r));
+ return _mm_cvtss_f32(r);
+}
+// IQ2_S / IQ3_S: 32 explicit sign bits -> 32 bytes of -1 / +1 (ggml's bit selector)
+IQ_AVX2 inline __m256i sgn32(uint32_t m) {
+ const __m256i sh = _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, //
+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);
+ const __m256i sel = _mm256_set1_epi64x((long long) 0x8040201008040201ull);
+ const __m256i b = _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) m), sh), sel);
+ return _mm256_or_si256(_mm256_cmpeq_epi8(b, sel), _mm256_set1_epi8(1));
+}
+// table entries into a vector without the shuffle port (ggml's _mm256_set_epi64x / _epi32 of table loads ends up in
+// inserts, one shuffle each; here each entry is a broadcast load and the lanes are put together by dword blends)
+IQ_AVX2 inline __m256i bq(const void* p) { return _mm256_castpd_si256(_mm256_broadcast_sd((const double*) p)); }
+IQ_AVX2 inline __m256i bd(const void* p) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*) p)); }
+IQ_AVX2 inline __m256i tab4(const uint64_t* t, unsigned a, unsigned b, unsigned c, unsigned d) { // qwords t[a] t[b] t[c] t[d]
+ return _mm256_blend_epi32(_mm256_blend_epi32(bq(t + a), bq(t + b), 0x0C), _mm256_blend_epi32(bq(t + c), bq(t + d), 0xC0),
+ 0xF0);
+}
+IQ_AVX2 inline __m256i tab8(const uint32_t* t, const unsigned* i) { // dwords t[i[0]] .. t[i[7]]
+ const __m256i a = _mm256_blend_epi32(bd(t + i[0]), bd(t + i[1]), 0x02), b = _mm256_blend_epi32(bd(t + i[2]), bd(t + i[3]), 0x08);
+ const __m256i c = _mm256_blend_epi32(bd(t + i[4]), bd(t + i[5]), 0x20), e = _mm256_blend_epi32(bd(t + i[6]), bd(t + i[7]), 0x80);
+ return _mm256_blend_epi32(_mm256_blend_epi32(a, b, 0x0C), _mm256_blend_epi32(c, e, 0xC0), 0xF0);
+}
+// the 8 sign bytes (-1 / +1) of each 7-bit sign index with its parity bit: ggml's keven_signs_q2xs, as +-1 bytes
+alignas(64) uint64_t g_ks64[128];
+void build_ks64() {
+ for (int i = 0; i < 128; ++i) {
+ int par = 0;
+ for (int b = 0; b < 7; ++b) par ^= (i >> b) & 1;
+ const int s = i | (par << 7);
+ uint64_t v = 0;
+ for (int j = 0; j < 8; ++j) v |= (uint64_t) (((s >> j) & 1) ? 0xFFu : 0x01u) << (8 * j);
+ g_ks64[i] = v;
+ }
+}
+IQ_AVX2 inline __m256i sgn7t(uint32_t w) { // a chunk's four 7-bit sign indices (bits 0..27 of w)
+ return tab4(g_ks64, w & 127u, (w >> 7) & 127u, (w >> 14) & 127u, (w >> 21) & 127u);
+}
+IQ_AVX2 inline __m256i sc16(int a, int b) { // int16 lanes 0-7 a (values 0-15), 8-15 b (values 16-31)
+ return _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi16((short) a)), _mm_set1_epi16((short) b), 1);
+}
+bool g_gather18 = false, g_gather21 = false; // the IQ3_XXS / IQ3_S grids by vpgatherdd (own(): if faster here)
+// IQ2_XXS's grid entry with its signs applied, for every grid index g and sign index i, at g << 7 | i (256 KB, built on
+// first use): one lookup instead of two; the products are then (q8 + 128) x signed grid less 128 x the grid's pair sums
+// (vpmaddubsw takes u8 x s8), the same integers as ggml's |grid| x signed q8
+alignas(64) uint64_t g_ct2x[256 * 128];
+bool g_ct16 = false; // (own(): if faster here)
+void build_ct2x() {
+ for (int g = 0; g < 256; ++g)
+ for (int i = 0; i < 128; ++i) {
+ const uint64_t sg = g_ks64[i]; // (+-1 bytes)
+ uint64_t v = 0;
+ for (int j = 0; j < 8; ++j) {
+ const int gb = (int) ((iq2xxs_grid[g] >> (8 * j)) & 255u);
+ const int sb = ((sg >> (8 * j)) & 255u) == 0xFFu ? -gb : gb;
+ v |= (uint64_t) (uint8_t) (int8_t) sb << (8 * j);
+ }
+ g_ct2x[(size_t) g << 7 | (size_t) i] = v;
+ }
+}
+
+// rows [r0, r1) (out[t][r - r0]) of NT tokens; TY 16 = IQ2_XXS, 22 = IQ2_S, 18 = IQ3_XXS, 21 = IQ3_S. A pair of
+// 32-value chunks a step: grid magnitudes g1 / g2, sign vectors s1 / s2 (-1 / +1), scales c1 / c2 - decoded once - then
+// every token: q8 times the signs, vpmaddubsw against the grid, vpmaddwd by the scales, into ggml's 8 int32 lanes
+template <int TY, int NT>
+IQ_AVX2 void iq_rows_avx2(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,
+ int r1) {
+ const __m256i sub = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
+ const bool gat = TY == 18 ? g_gather18 : TY == 21 ? g_gather21 : false;
+ const bool ct = TY == 16 && g_ct16;
+ const __m256i k80 = _mm256_set1_epi8((char) 0x80);
+ constexpr float K = TY == 18 ? 0.25f : TY == 21 ? 1.0f : 0.125f;
+ constexpr size_t BB = TY == 16 ? sizeof(block_iq2_xxs) : TY == 22 ? sizeof(block_iq2_s) :
+ TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* row = w + (size_t) r * rb;
+ __m256 accf[NT];
+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = row + (size_t) i * BB;
+ __m256i sumi[NT];
+ for (int t = 0; t < NT; ++t) sumi[t] = _mm256_setzero_si256();
+ uint16_t dh;
+ std::memcpy(&dh, blk, 2);
+ for (int ib = 0; ib < QK_K / 32; ib += 2) {
+ __m256i g1, g2, s1, s2, c1, c2;
+ if (TY == 16) { // IQ2_XXS: per chunk a word of 4 grid indices, a word of 4 sign indices + the scale
+ const uint8_t* q = blk + 2 + 8 * ib;
+ const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);
+ if (ct) { // (g1 / g2 the signed grids; s1 / s2 their 128 x pair sums)
+ g1 = tab4(g_ct2x, (a0 & 255u) << 7 | (a1 & 127u), ((a0 >> 8) & 255u) << 7 | ((a1 >> 7) & 127u),
+ ((a0 >> 16) & 255u) << 7 | ((a1 >> 14) & 127u), (a0 >> 24) << 7 | ((a1 >> 21) & 127u));
+ g2 = tab4(g_ct2x, (b0 & 255u) << 7 | (b1 & 127u), ((b0 >> 8) & 255u) << 7 | ((b1 >> 7) & 127u),
+ ((b0 >> 16) & 255u) << 7 | ((b1 >> 14) & 127u), (b0 >> 24) << 7 | ((b1 >> 21) & 127u));
+ s1 = _mm256_maddubs_epi16(k80, g1);
+ s2 = _mm256_maddubs_epi16(k80, g2);
+ } else {
+ g1 = tab4(iq2xxs_grid, a0 & 255u, (a0 >> 8) & 255u, (a0 >> 16) & 255u, a0 >> 24);
+ g2 = tab4(iq2xxs_grid, b0 & 255u, (b0 >> 8) & 255u, (b0 >> 16) & 255u, b0 >> 24);
+ s1 = sgn7t(a1);
+ s2 = sgn7t(b1);
+ }
+ c1 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));
+ c2 = _mm256_set1_epi16((short) (2 * (b1 >> 28) + 1));
+ } else if (TY == 22) { // IQ2_S: 10-bit grid indices (qs | 2 bits of qh), explicit signs, 2 scales a chunk
+ const uint8_t* qs = blk + 2 + 4 * ib;
+ const uint8_t* qh = blk + 2 + QK_K / 4;
+ const uint8_t* sg = blk + 2 + QK_K / 8 + 4 * ib;
+ const uint8_t* scl = blk + 2 + QK_K / 4 + QK_K / 32;
+ const unsigned h0 = qh[ib], h1 = qh[ib + 1];
+ g1 = tab4(iq2s_grid, qs[0] | ((h0 << 8) & 0x300u), qs[1] | ((h0 << 6) & 0x300u), qs[2] | ((h0 << 4) & 0x300u),
+ qs[3] | ((h0 << 2) & 0x300u));
+ g2 = tab4(iq2s_grid, qs[4] | ((h1 << 8) & 0x300u), qs[5] | ((h1 << 6) & 0x300u), qs[6] | ((h1 << 4) & 0x300u),
+ qs[7] | ((h1 << 2) & 0x300u));
+ s1 = sgn32(u32(sg));
+ s2 = sgn32(u32(sg + 4));
+ c1 = sc16(2 * (scl[ib] & 0xf) + 1, 2 * (scl[ib] >> 4) + 1);
+ c2 = sc16(2 * (scl[ib + 1] & 0xf) + 1, 2 * (scl[ib + 1] >> 4) + 1);
+ } else if (TY == 18) { // IQ3_XXS: 8 grid indices a chunk, then the chunks' sign / scale words
+ const uint8_t* q3 = blk + 2 + 8 * ib;
+ const uint8_t* gas = blk + 2 + QK_K / 4 + 4 * ib;
+ if (gat) {
+ g1 = _mm256_i32gather_epi32((const int*) iq3xxs_grid, _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) q3)), 4);
+ g2 = _mm256_i32gather_epi32((const int*) iq3xxs_grid,
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (q3 + 8))), 4);
+ } else {
+ unsigned i1[8], i2[8];
+ for (int k = 0; k < 8; ++k) {
+ i1[k] = q3[k];
+ i2[k] = q3[8 + k];
+ }
+ g1 = tab8(iq3xxs_grid, i1);
+ g2 = tab8(iq3xxs_grid, i2);
+ }
+ const uint32_t a0 = u32(gas), a1 = u32(gas + 4);
+ s1 = sgn7t(a0);
+ s2 = sgn7t(a1);
+ c1 = _mm256_set1_epi16((short) (2 * (a0 >> 28) + 1));
+ c2 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));
+ } else { // IQ3_S: 9-bit grid indices (qs | a bit of qh), explicit signs
+ const uint8_t* qs = blk + 2 + 8 * ib;
+ const uint8_t* qh = blk + 2 + QK_K / 4;
+ const uint8_t* sg = blk + 2 + QK_K / 4 + QK_K / 32 + 4 * ib;
+ const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + ib / 2];
+ const unsigned h0 = qh[ib], h1 = qh[ib + 1];
+ if (gat) {
+ const __m256i hb1 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h0), sub), _mm256_set1_epi32(1)), 8);
+ const __m256i hb2 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h1), sub), _mm256_set1_epi32(1)), 8);
+ g1 = _mm256_i32gather_epi32((const int*) iq3s_grid,
+ _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) qs)), hb1), 4);
+ g2 = _mm256_i32gather_epi32((const int*) iq3s_grid,
+ _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8))), hb2), 4);
+ } else {
+ unsigned i1[8], i2[8];
+ for (int k = 0; k < 8; ++k) {
+ i1[k] = qs[k] | (((h0 >> k) & 1u) << 8);
+ i2[k] = qs[8 + k] | (((h1 >> k) & 1u) << 8);
+ }
+ g1 = tab8(iq3s_grid, i1);
+ g2 = tab8(iq3s_grid, i2);
+ }
+ s1 = sgn32(u32(sg));
+ s2 = sgn32(u32(sg + 4));
+ c1 = _mm256_set1_epi16((short) (2 * (sc & 0xf) + 1));
+ c2 = _mm256_set1_epi16((short) (2 * (sc >> 4) + 1));
+ }
+ if (ct) {
+ for (int t = 0; t < NT; ++t) {
+ const int8_t* q8 = y[t][i].qs + 32 * ib;
+ const __m256i u1 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) q8), k80); // (q8 + 128)
+ const __m256i u2 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) (q8 + 32)), k80);
+ const __m256i d1 = _mm256_sub_epi16(_mm256_maddubs_epi16(u1, g1), s1);
+ const __m256i d2 = _mm256_sub_epi16(_mm256_maddubs_epi16(u2, g2), s2);
+ sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));
+ }
+ continue;
+ }
+ for (int t = 0; t < NT; ++t) {
+ const int8_t* q8 = y[t][i].qs + 32 * ib;
+ const __m256i d1 = _mm256_maddubs_epi16(g1, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) q8), s1));
+ const __m256i d2 = _mm256_maddubs_epi16(g2, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) (q8 + 32)), s2));
+ sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));
+ }
+ }
+ const float dx = h2f(dh);
+ for (int t = 0; t < NT; ++t)
+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(sumi[t]), accf[t]);
+ }
+ for (int t = 0; t < NT; ++t) out[t][r - r0] = K * hsum_ggml(accf[t]);
+ }
+}
+
+// ---- the same rows in 512-bit lanes (AVX-512 BW + VBMI, BMI2: Ice Lake and later Intel, Zen 4 / Zen 5), the GLM
+// lane's one-token IQ2_XXS scheme (src/kernels/cpu/iq_avx512_one.cpp) for 3-bit grids: every grid byte is one of 8
+// levels, so an entry is four 3-bit codes (a u16) and the grid's codes fit in registers - IQ3_XXS's 256 in 8 zmm,
+// IQ3_S's 512 in 16; a block's 64 indices are looked up 32 at a time with two-register word permutes and blends, each
+// 64 values' 16 codes spread to their bytes by a word permute, a multishift, an and and a byte shuffle; IQ3_XXS's signs
+// come from pdep and a parity fold, IQ3_S's are a mask register's bits. Two chunks a register, folded to ggml's 8
+// lanes, then ggml's float steps: the same numbers to the bit (own_check).
+#if defined(__GNUC__) || defined(__clang__)
+#define IQ_512 __attribute__((target("avx512f,avx512bw,avx512vbmi,bmi2,avx2,fma")))
+#else
+#define IQ_512
+#endif
+alignas(64) uint16_t g_code3x[256], g_code3s[512];
+bool g_codes_ok = false;
+
+IQ_512 inline uint32_t signs4(uint32_t w) { // ksigns_iq2xs for the four 7-bit fields of w: i | parity(i) << 7
+ const uint64_t s = _pdep_u64(w & 0x0FFFFFFFu, 0x7F7F7F7Full);
+ uint64_t p = s ^ ((s >> 4) & 0x0F0F0F0Full);
+ p ^= (p >> 2) & 0x03030303ull;
+ p ^= (p >> 1) & 0x01010101ull;
+ return (uint32_t) (s | ((p & 0x01010101ull) << 7));
+}
+// each grid entry as four 3-bit level codes; false if a byte is not one of the 8 levels (a grid this file does not know)
+bool build_codes3() {
+ static const int lx[8] = {4, 12, 20, 28, 36, 44, 52, 62}, ls[8] = {1, 3, 5, 7, 9, 11, 13, 15};
+ auto code = [](uint32_t e, const int* lv) -> int {
+ int c = 0;
+ for (int k = 0; k < 4; ++k) {
+ const int b = (int) ((e >> (8 * k)) & 255u);
+ int l = -1;
+ for (int j = 0; j < 8; ++j)
+ if (lv[j] == b) l = j;
+ if (l < 0) return -1;
+ c |= l << (3 * k);
+ }
+ return c;
+ };
+ for (int i = 0; i < 256; ++i) {
+ const int c = code(iq3xxs_grid[i], lx);
+ if (c < 0) return false;
+ g_code3x[i] = (uint16_t) c;
+ }
+ for (int i = 0; i < 512; ++i) {
+ const int c = code(iq3s_grid[i], ls);
+ if (c < 0) return false;
+ g_code3s[i] = (uint16_t) c;
+ }
+ return true;
+}
+IQ_512 inline __m256i fold(__m512i a) { // a block's 16 int32 lanes -> ggml's 8 (lane k + lane k + 8), exact
+ return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1));
+}
+IQ_512 inline __m512i scales4(int s0, int s1, int s2, int s3) { // int16 lanes 0-7 s0, 8-15 s1, 16-23 s2, 24-31 s3
+ const __m512i idx = _mm512_set_epi16(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, //
+ 0, 0, 0, 0, 0, 0, 0, 0);
+ const uint64_t packed = (uint64_t) (uint16_t) s0 | ((uint64_t) (uint16_t) s1 << 16) |
+ ((uint64_t) (uint16_t) s2 << 32) | ((uint64_t) (uint16_t) s3 << 48);
+ return _mm512_permutexvar_epi16(idx, _mm512_set1_epi64((long long) packed));
+}
+// the codes of 32 word indices: 64 entries a register pair, NP pairs, blended by the index's bits 6, 7 (and 8)
+template <int NP>
+IQ_512 inline __m512i lookup(const __m512i* T, __m512i idx) {
+ __m512i r[NP];
+ for (int j = 0; j < NP; ++j) r[j] = _mm512_permutex2var_epi16(T[2 * j], idx, T[2 * j + 1]);
+ const __mmask32 m6 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(64));
+ const __mmask32 m7 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(128));
+ __m512i a = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[0], r[1]), _mm512_mask_blend_epi16(m6, r[2], r[3]));
+ if (NP == 8) {
+ const __mmask32 m8 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(256));
+ const __m512i b = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[NP > 4 ? 4 : 0], r[NP > 5 ? 5 : 0]),
+ _mm512_mask_blend_epi16(m6, r[NP > 6 ? 6 : 0], r[NP > 7 ? 7 : 0]));
+ a = _mm512_mask_blend_epi16(m8, a, b);
+ }
+ return a;
+}
+
+template <int TY, int NT>
+IQ_512 void iq3_rows_512(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,
+ int r1) {
+ constexpr int NTAB = TY == 18 ? 8 : 16;
+ constexpr size_t BB = TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);
+ __m512i T[NTAB];
+ for (int k = 0; k < NTAB; ++k)
+ T[k] = _mm512_load_si512((const void*) ((TY == 18 ? g_code3x : g_code3s) + 32 * k));
+ // code -> grid byte (IQ3_XXS 4 .. 62, IQ3_S 1 .. 15) in every 128-bit lane; byte b of an output qword <- 3 bits at
+ // [0, 3, 6, 9] (its first entry's code, word 0) or [32, 35, 38, 41] (the second's, word 2)
+ const __m512i lut = TY == 18 ? _mm512_set4_epi32(0, 0, 0x3E342C24, 0x1C140C04) : _mm512_set4_epi32(0, 0, 0x0F0D0B09, 0x07050301);
+ const __m512i ctl = _mm512_set1_epi64(0x2926232009060300LL);
+ const __m512i seven = _mm512_set1_epi8(7), zero = _mm512_setzero_si512();
+ __m512i pidx[2]; // a 64-value group h of 32 codes: output qword m <- entries 16 h + 2 m (word 0), + 1 (word 2)
+ for (int h = 0; h < 2; ++h) {
+ alignas(64) uint16_t p[32];
+ for (int m = 0; m < 8; ++m) {
+ p[4 * m] = p[4 * m + 1] = (uint16_t) (16 * h + 2 * m);
+ p[4 * m + 2] = p[4 * m + 3] = (uint16_t) (16 * h + 2 * m + 1);
+ }
+ pidx[h] = _mm512_load_si512((const void*) p);
+ }
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* row = w + (size_t) r * rb;
+ __m256 accf[NT];
+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();
+ for (int i = 0; i < nb; ++i) {
+ const uint8_t* blk = row + (size_t) i * BB;
+ __m512i codes[2];
+ if (TY == 18) {
+ const __m512i q = _mm512_loadu_si512((const void*) (blk + 2));
+ codes[0] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(q)));
+ codes[1] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(q, 1)));
+ } else {
+ const uint8_t* qh = blk + 2 + QK_K / 4;
+ const __m512i hi = _mm512_set1_epi16(256);
+ const __m512i ia = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 2)));
+ const __m512i ib = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 34)));
+ // (index k of chunk c: qs[8 c + k] | bit k of qh[c] << 8 - the 32 bits of qh[0..3] are chunks 0..3's)
+ codes[0] = lookup<8>(T, _mm512_or_si512(ia, _mm512_maskz_mov_epi16((__mmask32) u32(qh), hi)));
+ codes[1] = lookup<8>(T, _mm512_or_si512(ib, _mm512_maskz_mov_epi16((__mmask32) u32(qh + 4), hi)));
+ }
+ __m512i acc[NT];
+ for (int t = 0; t < NT; ++t) acc[t] = _mm512_setzero_si512();
+ for (int p = 0; p < 4; ++p) {
+ const __m512i rep = _mm512_permutexvar_epi16(pidx[p & 1], codes[p >> 1]);
+ const __m512i g = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_multishift_epi64_epi8(ctl, rep), seven));
+ uint64_t sm;
+ int sa, sb;
+ if (TY == 18) {
+ const uint8_t* sw = blk + 2 + QK_K / 4 + 8 * p;
+ const uint32_t wa = u32(sw), wb = u32(sw + 4);
+ sm = (uint64_t) signs4(wa) | ((uint64_t) signs4(wb) << 32);
+ sa = 2 * (int) (wa >> 28) + 1;
+ sb = 2 * (int) (wb >> 28) + 1;
+ } else {
+ std::memcpy(&sm, blk + 2 + QK_K / 4 + QK_K / 32 + 8 * p, 8);
+ const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + p];
+ sa = 2 * (sc & 0xf) + 1;
+ sb = 2 * (sc >> 4) + 1;
+ }
+ const __mmask64 m = _cvtu64_mask64(sm);
+ const __m512i S = scales4(sa, sa, sb, sb);
+ for (int t = 0; t < NT; ++t) {
+ const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * p));
+ const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);
+ acc[t] = _mm512_add_epi32(acc[t], _mm512_madd_epi16(_mm512_maddubs_epi16(g, ys), S));
+ }
+ }
+ uint16_t dh;
+ std::memcpy(&dh, blk, 2);
+ const float dx = h2f(dh);
+ for (int t = 0; t < NT; ++t)
+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold(acc[t])), accf[t]);
+ }
+ for (int t = 0; t < NT; ++t) out[t][r - r0] = (TY == 18 ? 0.25f : 1.0f) * hsum_ggml(accf[t]);
+ }
+}
+
+template <int TY>
+void iq_rows_n(const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+ static const bool w512 = [] {
+ const char* v = std::getenv("DS4_IQ_AVX2"); // (=1: the AVX2 rows on an AVX-512 CPU, to compare)
+ return (TY == 18 || TY == 21) && g_codes_ok && isa().w512 && !(v != nullptr && std::atoi(v) != 0);
+ }();
+ const int nb = n / QK_K;
+ for (int t0 = 0; t0 < nt; t0 += 4) {
+ const block_q8_K* y[4];
+ float* o[4];
+ const int c = nt - t0 < 4 ? nt - t0 : 4;
+ for (int t = 0; t < c; ++t) {
+ y[t] = (const block_q8_K*) act[t0 + t];
+ o[t] = out[t0 + t];
+ }
+ if constexpr (TY == 18 || TY == 21) {
+ if (w512) {
+ switch (c) {
+ case 1: iq3_rows_512<TY, 1>(w, rb, nb, y, o, r0, r1); break;
+ case 2: iq3_rows_512<TY, 2>(w, rb, nb, y, o, r0, r1); break;
+ case 3: iq3_rows_512<TY, 3>(w, rb, nb, y, o, r0, r1); break;
+ default: iq3_rows_512<TY, 4>(w, rb, nb, y, o, r0, r1); break;
+ }
+ continue;
+ }
+ }
+ switch (c) {
+ case 1: iq_rows_avx2<TY, 1>(w, rb, nb, y, o, r0, r1); break;
+ case 2: iq_rows_avx2<TY, 2>(w, rb, nb, y, o, r0, r1); break;
+ case 3: iq_rows_avx2<TY, 3>(w, rb, nb, y, o, r0, r1); break;
+ default: iq_rows_avx2<TY, 4>(w, rb, nb, y, o, r0, r1); break;
+ }
+ }
+}
+#endif // IQ_X86
+
+// out[t][r - r0] for the rows of `type` (an own type only)
+void own_rows(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0,
+ int r1) {
+#if defined(IQ_X86)
+ if (type == GGML_TYPE_IQ2_XXS) return iq_rows_n<16>(w, rb, n, act, nt, out, r0, r1);
+ if (type == GGML_TYPE_IQ2_S) return iq_rows_n<22>(w, rb, n, act, nt, out, r0, r1);
+ if (type == GGML_TYPE_IQ3_XXS) return iq_rows_n<18>(w, rb, n, act, nt, out, r0, r1);
+ if (type == GGML_TYPE_IQ3_S) return iq_rows_n<21>(w, rb, n, act, nt, out, r0, r1);
+#endif
+ (void) type, (void) w, (void) rb, (void) n, (void) act, (void) nt, (void) out, (void) r0, (void) r1;
+}
+
+// our rows against ggml-cpu's dot, bit for bit: random blocks of the type (its F16 scale a small normal value), Q8_K
+// activations from ggml's quantizer, 1, 2, 3, 5 and 9 tokens, 13 rows of 8 blocks
+bool own_check(int type) {
+ ggml_cpu_init();
+ const ggml_type t = (ggml_type) type;
+ const int n = 2048, rows = 13;
+ const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);
+ uint64_t st = 0x9E3779B97F4A7C15ull;
+ auto next = [&st] {
+ st ^= st << 13;
+ st ^= st >> 7;
+ st ^= st << 17;
+ return st;
+ };
+ std::vector<uint8_t> w(rb * rows);
+ for (auto& b : w) b = (uint8_t) next();
+ for (size_t o = 0; o < w.size(); o += bb) {
+ const ggml_fp16_t d = ggml_fp32_to_fp16(0.002f + 0.001f * (float) (next() % 16));
+ std::memcpy(&w[o], &d, sizeof d);
+ }
+ const auto* q8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);
+ const auto dot = ggml_get_type_traits_cpu(t)->vec_dot;
+ for (int nt : {1, 2, 3, 5, 9}) {
+ std::vector<uint8_t> a(ab * (size_t) nt);
+ std::vector<float> x((size_t) n), got((size_t) nt * rows), ref((size_t) nt * rows);
+ std::vector<const void*> ap((size_t) nt);
+ std::vector<float*> op((size_t) nt);
+ for (int k = 0; k < nt; ++k) {
+ for (auto& v : x) v = ((float) (next() % 2001) - 1000.0f) * 1e-3f;
+ q8->from_float(x.data(), a.data() + ab * (size_t) k, n);
+ ap[(size_t) k] = a.data() + ab * (size_t) k;
+ op[(size_t) k] = got.data() + (size_t) k * rows;
+ }
+ own_rows(type, w.data(), rb, n, ap.data(), nt, op.data(), 0, rows);
+ for (int k = 0; k < nt; ++k)
+ for (int r = 0; r < rows; ++r) dot(n, &ref[(size_t) k * rows + r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) k], 0, 1);
+ if (std::memcmp(got.data(), ref.data(), got.size() * sizeof(float)) != 0) return false;
+ }
+ return true;
+}
+
+// the time (us, best of 5 rounds) of 32 rows of 16 blocks of the type against one token: ours with the IQ3 grids as
+// broadcasts + blends (variant 1) or by vpgatherdd (2), or the native lane's rows (0: the GLM lane's / ggml-cpu's)
+double time_variant(int type, int variant) {
+ const ggml_type t = (ggml_type) type;
+ const int n = 4096, rows = 32;
+ const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);
+ std::vector<uint8_t> w(rb * rows), a(ab);
+ uint32_t s = 12345u;
+ for (auto& b : w) b = (uint8_t) ((s = s * 1664525u + 1013904223u) >> 24);
+ for (size_t o = 0; o < w.size(); o += bb) {
+ const ggml_fp16_t d = ggml_fp32_to_fp16(0.003f);
+ std::memcpy(&w[o], &d, sizeof d);
+ }
+ std::vector<float> x((size_t) n), y((size_t) rows);
+ for (auto& v : x) v = (float) ((int) ((s = s * 1664525u + 1013904223u) >> 16) - 32768) / 32768.0f;
+ ggml_get_type_traits_cpu(GGML_TYPE_Q8_K)->from_float(x.data(), a.data(), n);
+ const void* ap[1] = {a.data()};
+ float* yp[1] = {y.data()};
+ kc::NativeFmt nf; // (the native rows timed as a gate / up pair of the same rows: 2 x rows dots, as ours below)
+ std::string err;
+ if (variant == 0 && !kc::native_fmt(type, type, n, QK_K, nf, err)) return 1e30;
+ bool* gflag = type == GGML_TYPE_IQ3_XXS ? &g_gather18 : type == GGML_TYPE_IQ3_S ? &g_gather21 :
+ type == GGML_TYPE_IQ2_XXS ? &g_ct16 : nullptr;
+ const bool keep = gflag != nullptr && *gflag;
+ if (gflag != nullptr) *gflag = variant >= 2;
+ double best = 1e30;
+ for (int round = 0; round < 6; ++round) {
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int rep = 0; rep < 4; ++rep) {
+ if (variant == 0) {
+ kc::native_gu_rows_split(nf, w.data(), w.data(), ap, 1, yp, 0, rows, std::numeric_limits<float>::infinity());
+ } else {
+ own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);
+ own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);
+ }
+ }
+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+ if (round > 0) best = std::min(best, us);
+ }
+ if (gflag != nullptr) *gflag = keep;
+ return best;
+}
+
+// our rows for this type, here: IQ3_XXS / IQ3_S on AVX2 or AVX-512 VBMI, IQ2_XXS / IQ2_S on AVX2 - checked once
+// against ggml-cpu's dot (each variant), then the fastest of ours and the native lane's measured on this CPU (they
+// give the same bits, so the choice changes no answer). DS4_IQ_OWN=0: the native lane's rows; =1: ours without
+// measuring (DS4_IQ_GATHER=1: the IQ3 grids by vpgatherdd); DS4_IQ_AVX2=1: our AVX2 rows on an AVX-512 CPU too.
+bool own(int type) {
+#if defined(IQ_X86)
+ int k;
+ switch (type) {
+ case GGML_TYPE_IQ3_XXS: k = 0; break;
+ case GGML_TYPE_IQ3_S: k = 1; break;
+ case GGML_TYPE_IQ2_XXS: k = 2; break;
+ case GGML_TYPE_IQ2_S: k = 3; break;
+ default: return false;
+ }
+ static int mode = -1; // DS4_IQ_OWN: -1 measured, 0 never, 1 always
+ static bool force_avx2 = false, force_gather = false;
+ static const bool allowed = [] {
+ const char* v = std::getenv("DS4_IQ_OWN");
+ mode = v == nullptr ? -1 : std::atoi(v) != 0 ? 1 : 0;
+ const char* a2 = std::getenv("DS4_IQ_AVX2");
+ force_avx2 = a2 != nullptr && std::atoi(a2) != 0;
+ const char* gt = std::getenv("DS4_IQ_GATHER");
+ force_gather = gt != nullptr && std::atoi(gt) != 0;
+ g_codes_ok = build_codes3(); // (the 512-bit rows' tables; without them those rows are not used)
+ build_ks64();
+ build_ct2x();
+ return mode != 0 && isa().avx2;
+ }();
+ // (IQ2 on an AVX-512 CPU: the GLM lane's AVX-512 rows serve them, unless DS4_IQ_AVX2)
+ if (!allowed || (k >= 2 && isa().w512 && !force_avx2)) return false;
+ static std::once_flag once[4];
+ static bool ok[4];
+ std::call_once(once[k], [&] {
+ bool* gflag = k == 0 ? &g_gather18 : k == 1 ? &g_gather21 : k == 2 ? &g_ct16 : nullptr; // (the second form)
+ bool good = own_check(type); // (the blend form; the 512-bit rows where they serve)
+ bool good_g = false;
+ if (gflag != nullptr) {
+ *gflag = true;
+ good_g = own_check(type);
+ *gflag = false;
+ }
+ if (mode == 1) {
+ if (gflag != nullptr) *gflag = force_gather && good_g;
+ ok[k] = good || (force_gather && good_g);
+ return;
+ }
+ // the variants timed in turns, the best of each kept (other work on the machine - the load's read-ahead, the
+ // prompt - then hits them alike), and ours taken only when clearly faster (10 %): the choice changes no answer,
+ // a wrong one costs time (Uranus' Golden Cove: one-shot timings 30-80 % apart run to run, the choice flipping)
+ double tn = 1e30, tb = 1e30, tg = 1e30;
+ for (int it = 0; it < 5; ++it) {
+ tn = std::min(tn, time_variant(type, 0));
+ if (good) tb = std::min(tb, time_variant(type, 1));
+ if (good_g) tg = std::min(tg, time_variant(type, 2));
+ }
+ if (gflag != nullptr) *gflag = tg < tb;
+ ok[k] = std::min(tb, tg) < 0.9 * tn;
+ if (std::getenv("DS4_IQ_VERBOSE") != nullptr)
+ std::fprintf(stderr, "ds4: CPU lane %s rows: native %.0f us, ours %.0f us%s -> %s\n", ggml_type_name((ggml_type) type),
+ tn, tb, gflag != nullptr ? (std::string(" (second form: gather / one table ") + std::to_string((int) tg) + " us)").c_str() : "",
+ ok[k] ? (gflag != nullptr && *gflag ? "ours (second form)" : "ours") : "native");
+ });
+ return ok[k];
+#else
+ (void) type;
+ return false;
+#endif
+}
+} // namespace
+
+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+ kc::NativeFmt n;
+ if (!kc::native_fmt(tg, td, n_embd, n_ff, n, err)) return false;
+ f.tg = tg;
+ f.td = td;
+ f.n_embd = n_embd;
+ f.n_ff = n_ff;
+ f.gu_act = n.gu_act;
+ f.d_act = n.d_act;
+ f.gu_row = n.gu_row;
+ f.d_row = n.d_row;
+ f.act_bytes = n.act_bytes;
+ f.h_bytes = n.h_bytes;
+ return true;
+}
+
+void iq_cpu_quant(int act, const float* x, int k, void* dst) {
+ kc::NativeFmt n;
+ n.gu_act = act;
+ n.n_embd = k;
+ kc::native_quant_act(n, x, dst);
+}
+
+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,
+ float* const* ff, int r0, int r1, float limit) {
+ if (r1 > r0 && own(f.tg)) { // (native_gu_rows_split's epilogue, on the same values)
+ const int nr = r1 - r0;
+ thread_local std::vector<float> buf;
+ thread_local std::vector<float*> go, uo;
+ buf.resize((size_t) 2 * nt * nr);
+ go.resize((size_t) nt);
+ uo.resize((size_t) nt);
+ for (int t = 0; t < nt; ++t) {
+ go[(size_t) t] = buf.data() + (size_t) t * nr;
+ uo[(size_t) t] = buf.data() + (size_t) (nt + t) * nr;
+ }
+ own_rows(f.tg, gate, f.gu_row, f.n_embd, act, nt, go.data(), r0, r1);
+ own_rows(f.tg, up, f.gu_row, f.n_embd, act, nt, uo.data(), r0, r1);
+ for (int t = 0; t < nt; ++t)
+ for (int r = 0; r < nr; ++r) {
+ float g = go[(size_t) t][r], u = uo[(size_t) t][r];
+ if (limit > 0.0f) {
+ g = std::fmin(g, limit);
+ u = std::fmin(std::fmax(u, -limit), limit);
+ }
+ ff[t][r0 + r] = (g / (1.f + std::exp(-g))) * u;
+ }
+ return;
+ }
+ kc::native_gu_rows_split(native(f), gate, up, act, nt, ff, r0, r1,
+ limit > 0.0f ? limit : std::numeric_limits<float>::infinity());
+}
+
+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+ if (r1 > r0 && own(f.td)) {
+ thread_local std::vector<float*> o;
+ o.resize((size_t) nt);
+ for (int t = 0; t < nt; ++t) o[(size_t) t] = out[t] + r0;
+ own_rows(f.td, down, f.d_row, f.n_ff, hq, nt, o.data(), r0, r1);
+ return;
+ }
+ kc::native_down_rows_split(native(f), down, hq, nt, out, r0, r1);
+}
+
+bool iq_cpu_own_rows(int type) { return own(type); }
+#else
+bool iq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
+ err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts' CPU lane is not in it";
+ return false;
+}
+void iq_cpu_quant(int, const float*, int, void*) {}
+void iq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int,
+ float) {}
+void iq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+bool iq_cpu_own_rows(int) { return false; }
+#endif
+
+} // namespace strata::ds4::gguf
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
new file mode 100644
index 000000000..688fe1701
--- /dev/null
+++ b/src/dsv41/ds4_kernels.cu
@@ -0,0 +1,2060 @@
+// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary
+// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.
+#include "ds4_kernels.cuh"
+#include "ds4_numerics.cuh"
+#include "ds4_q6k.cuh"
+
+#include <cuda_fp16.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cfloat>
+#include <cstdio>
+
+namespace strata::ds4 {
+namespace {
+
+constexpr int kThreads = 256;
+
+__device__ __forceinline__ float bf16f(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }
+
+__device__ __forceinline__ uint16_t f2bf16(float f) { // round to nearest even (the values here are exact anyway)
+ uint32_t u = __float_as_uint(f);
+ if ((u & 0x7f800000u) == 0x7f800000u) return (uint16_t) (u >> 16);
+ u += 0x7fffu + ((u >> 16) & 1u);
+ return (uint16_t) (u >> 16);
+}
+
+__device__ __forceinline__ float warp_sum(float v) {
+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+ return v;
+}
+__device__ __forceinline__ float warp_max(float v) {
+ for (int o = 16; o > 0; o >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, o));
+ return v;
+}
+
+// block-wide sum for blockDim.x <= 1024 (a multiple of 32); every thread gets the result
+__device__ float block_sum(float v) {
+ __shared__ float red[32];
+ __shared__ float total;
+ const int lane = threadIdx.x & 31, w = threadIdx.x >> 5;
+ v = warp_sum(v);
+ if (lane == 0) red[w] = v;
+ __syncthreads();
+ if (w == 0) {
+ float t = lane < (int) (blockDim.x >> 5) ? red[lane] : 0.0f;
+ t = warp_sum(t);
+ if (lane == 0) total = t;
+ }
+ __syncthreads();
+ const float r = total;
+ __syncthreads();
+ return r;
+}
+
+// 2^ceil(log2(x)) for a positive normal x (the reference's fast_round_scale)
+__device__ __forceinline__ float pow2_ceil(float x) {
+ const uint32_t b = __float_as_uint(x);
+ const int e = (int) ((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);
+ return __uint_as_float((uint32_t) (e + 127) << 23);
+}
+
+// a load of memory another device or the host writes, never from a stale cache line (CUDA's __ldcv; HIP: volatile
+// 32-bit loads)
+template <class T>
+__device__ __forceinline__ T ld_fresh(const T* p) {
+#if defined(STRATA_USE_HIP)
+ T v;
+ const volatile uint32_t* s = (const volatile uint32_t*) p;
+ uint32_t* d = (uint32_t*) &v;
+#pragma unroll
+ for (int i = 0; i < (int) (sizeof(T) / 4); ++i) d[i] = s[i];
+ return v;
+#else
+ return __ldcv(p);
+#endif
+}
+
+__device__ __forceinline__ float fp8_rt(float v) { return e4m3_round(v); } // E4M3, nearest even (input clamped)
+
+__device__ __forceinline__ float e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even
+ const float a = fabsf(v);
+ float r;
+ if (a <= 0.25f) r = 0.0f;
+ else if (a < 0.75f) r = 0.5f;
+ else if (a <= 1.25f) r = 1.0f;
+ else if (a < 1.75f) r = 1.5f;
+ else if (a <= 2.5f) r = 2.0f;
+ else if (a < 3.5f) r = 3.0f;
+ else if (a <= 5.0f) r = 4.0f;
+ else r = 6.0f;
+ return copysignf(r, v);
+}
+
+// ---------------------------------------------------------------- the caches' packed rows (kv_row_bytes)
+// the E2M1 code of a value e2m1_rt returned (bit 3 the sign, -0 kept; magnitudes 0 .5 1 1.5 2 3 4 6 as 0 .. 7)
+__device__ __forceinline__ uint32_t e2m1_code(float q) {
+ const float a = fabsf(q);
+ const uint32_t m = a <= 1.5f ? (uint32_t) (a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;
+ return m | ((__float_as_uint(q) >> 31) << 3);
+}
+// the E4M3 code of v / 2^k for a v whose quotient is an E4M3 value (|v / 2^k| <= 448, at most 4 significant bits, a
+// multiple of 2^-9; v 0 or a normal F32): integer arithmetic, exact, -0 kept
+__device__ __forceinline__ uint32_t e4m3_code(float v, int k = 0) {
+ const uint32_t b = __float_as_uint(v), sg = (b >> 24) & 0x80u, a = b & 0x7fffffffu;
+ if (a == 0u) return sg;
+ const int e = (int) (a >> 23) - 127 - k; // the quotient's binade
+ if (e >= -6) return sg | ((uint32_t) (e + 7) << 3) | ((a >> 20) & 7u);
+ return sg | (((a & 0x7fffffu) | 0x800000u) >> (14 - e)); // a subnormal: m 2^-9
+}
+// an E2M1 code's value times 2^-14, by the conversion of an FP16 (the code's sign, exponent and mantissa bit are an
+// FP16's at bits 15 and 11..9: exact, subnormals too) - the readers' decode: (value 2^-14) (scale 2^14) is value * scale
+// exactly, both products exact
+__device__ __forceinline__ float e2m1_val14(uint32_t c) {
+ return __half2float(__ushort_as_half((unsigned short) (((c & 8u) << 12) | ((c & 7u) << 9))));
+}
+// a UE8M0 scale byte's value (the byte the F32 exponent: 2^(b - 127); 0 -> 0, an unwritten row's)
+__device__ __forceinline__ float ue8m0_val(uint32_t b) { return __uint_as_float(b << 23); }
+
+// ---------------------------------------------------------------- embed
+__global__ void embed_kernel(const uint16_t* __restrict__ rows, float* __restrict__ h, int dim, int hc) {
+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (e >= dim) return;
+ const float v = bf16f(rows[(size_t) t * dim + e]);
+ for (int s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;
+}
+
+// ---------------------------------------------------------------- finish
+constexpr int kFinMax = 32; // values a thread holds: len <= 32 * 256
+
+// kPack: the rounded rows packed to f.pdst (f.quant's cache format) instead of written to x or f.dst
+template <bool kPack>
+__global__ void __launch_bounds__(kThreads) finish_kernel(float* x, int len, long long stride, Finish f) {
+ float* v = x + (long long) blockIdx.x * stride;
+ float r[kFinMax];
+ const int nj = (len + kThreads - 1) / kThreads;
+ float ss = 0.0f;
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ r[j] = e < len ? v[e] : 0.0f;
+ ss += r[j] * r[j];
+ }
+ if (f.w != nullptr) {
+ const float tot = block_sum(ss);
+ const float inv = rsqrtf(tot / (float) len + f.eps);
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ if (e < len) r[j] = r[j] * inv * f.w[e];
+ }
+ }
+ if (f.rd > 0) {
+ const int r0 = len - f.rd;
+ const int pos = f.pos + (int) (blockIdx.x / f.vpp) * f.pos_step;
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ const float other = __shfl_xor_sync(0xffffffffu, r[j], 1);
+ if (e >= r0 && e < len) {
+ const int i = (e - r0) >> 1;
+ const float ang = (float) pos * f.freqs[i];
+ const float c = cosf(ang);
+ const float sn = f.inverse ? -sinf(ang) : sinf(ang);
+ if (((e - r0) & 1) == 0) r[j] = r[j] * c - other * sn; // re = a c - b s
+ else r[j] = other * sn + r[j] * c; // im = a s + b c
+ }
+ }
+ }
+ if (f.quant != 0 && !kPack) {
+ const int g = f.quant == 2 ? 16 : 32;
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ float a = e < len ? fabsf(r[j]) : 0.0f;
+ for (int o = g / 2; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o));
+ if (e >= len) continue;
+ if (f.quant == 1) {
+ const float s = pow2_ceil(fmaxf(a, 1e-4f) * (1.0f / 448.0f));
+ r[j] = fp8_rt(fminf(fmaxf(r[j] / s, -448.0f), 448.0f)) * s;
+ } else if (f.quant == 2) {
+ const float s = fp8_rt(fmaxf(a, 6.0f * 0.001953125f) / 6.0f);
+ r[j] = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f)) * s;
+ } else {
+ const float s = pow2_ceil(fmaxf(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ r[j] = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f)) * s;
+ }
+ }
+ }
+ if (kPack) { // the same roundings, written as their codes and each group's scale (the readers' q * s: r exactly)
+ const int g = f.quant == 2 ? 16 : 32;
+ uint8_t* pr = f.pdst + (long long) blockIdx.x * f.pdst_stride;
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ float a = e < len ? fabsf(r[j]) : 0.0f;
+ for (int o = g / 2; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o));
+ float q = 0.0f, s = 1.0f;
+ if (e < len) {
+ if (f.quant == 1) {
+ s = pow2_ceil(fmaxf(a, 1e-4f) * (1.0f / 448.0f));
+ q = fp8_rt(fminf(fmaxf(r[j] / s, -448.0f), 448.0f));
+ } else if (f.quant == 2) {
+ s = fp8_rt(fmaxf(a, 6.0f * 0.001953125f) / 6.0f);
+ q = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f));
+ } else {
+ s = pow2_ceil(fmaxf(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ q = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f));
+ }
+ r[j] = q * s;
+ }
+ if (f.quant == 1) {
+ if (e < len) pr[e] = (uint8_t) e4m3_code(q);
+ } else {
+ const uint32_t c = e2m1_code(q);
+ const uint32_t hi = __shfl_down_sync(0xffffffffu, c, 1); // (value e + 1: the next lane's)
+ if (e < len && (e & 1) == 0) pr[e >> 1] = (uint8_t) (c | (hi << 4));
+ }
+ if (e < len && e % g == 0)
+ pr[(f.quant == 1 ? len : len / 2) + e / g] =
+ (uint8_t) (f.quant == 2 ? e4m3_code(s) : __float_as_uint(s) >> 23);
+ }
+ }
+ uint16_t* d = f.dst != nullptr ? f.dst + (long long) blockIdx.x * f.dst_stride : nullptr;
+#pragma unroll
+ for (int j = 0; j < kFinMax; ++j) {
+ if (j >= nj) break;
+ const int e = threadIdx.x + j * kThreads;
+ if (e >= len) continue;
+ if (d != nullptr) d[e] = f2bf16(r[j]);
+ else if (!kPack) v[e] = r[j];
+ }
+}
+
+// ---------------------------------------------------------------- hyper-connections
+__global__ void __launch_bounds__(kThreads) hc_mix_kernel(const float* __restrict__ h, const float* __restrict__ fn,
+ int hcdim, float eps, float* __restrict__ mixes) {
+ const int m = blockIdx.x, t = blockIdx.y;
+ const float* w = fn + (size_t) m * hcdim;
+ const float* x = h + (size_t) t * hcdim;
+ float acc = 0.0f, ss = 0.0f;
+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {
+ const float4 hv = *(const float4*) (x + k);
+ const float4 wv = *(const float4*) (w + k);
+ acc += hv.x * wv.x + hv.y * wv.y + hv.z * wv.z + hv.w * wv.w;
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ acc = block_sum(acc);
+ ss = block_sum(ss);
+ if (threadIdx.x == 0) mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);
+}
+
+// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone)
+__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,
+ const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,
+ float* comb) {
+ const int t = blockIdx.x, lane = threadIdx.x;
+ const float* mx = mixes + (size_t) t * 32;
+ pre += (size_t) t * 8;
+ post += (size_t) t * 8;
+ comb += (size_t) t * 64;
+ if (lane < hc) {
+ pre[lane] = 1.0f / (1.0f + expf(-(mx[lane] * scale[0] + base[lane]))) + eps;
+ post[lane] = 2.0f / (1.0f + expf(-(mx[hc + lane] * scale[1] + base[hc + lane])));
+ }
+ if (hc == 4) {
+ // rows are the 4-lane groups (j = lane / 4), columns the lanes 4 apart (k = lane % 4)
+ const bool on = lane < 16;
+ float c = on ? mx[8 + lane] * scale[2] + base[8 + lane] : -FLT_MAX;
+ auto row_sum = [&](float v) {
+ v += __shfl_xor_sync(0xffffffffu, v, 1);
+ return v + __shfl_xor_sync(0xffffffffu, v, 2);
+ };
+ auto col_sum = [&](float v) {
+ v += __shfl_xor_sync(0xffffffffu, v, 4);
+ return v + __shfl_xor_sync(0xffffffffu, v, 8);
+ };
+ float m = fmaxf(c, __shfl_xor_sync(0xffffffffu, c, 1));
+ m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, 2));
+ c = on ? expf(c - m) : 0.0f;
+ c = c / row_sum(c) + eps; // softmax over each row, + eps
+ c = on ? c : 0.0f;
+ c = c / (col_sum(c) + eps); // each destination column over its sources
+ for (int it = 1; it < iters; ++it) {
+ c = c / (row_sum(c) + eps);
+ c = c / (col_sum(c) + eps);
+ }
+ if (on) comb[lane] = c;
+ return;
+ }
+ if (lane != 0) return;
+ float c[8][8];
+ for (int j = 0; j < hc; ++j) {
+ float m = -FLT_MAX;
+ for (int k = 0; k < hc; ++k) {
+ c[j][k] = mx[2 * hc + j * hc + k] * scale[2] + base[2 * hc + j * hc + k];
+ m = fmaxf(m, c[j][k]);
+ }
+ float s = 0.0f;
+ for (int k = 0; k < hc; ++k) {
+ c[j][k] = expf(c[j][k] - m);
+ s += c[j][k];
+ }
+ for (int k = 0; k < hc; ++k) c[j][k] = c[j][k] / s + eps;
+ }
+ for (int k = 0; k < hc; ++k) {
+ float s = 0.0f;
+ for (int j = 0; j < hc; ++j) s += c[j][k];
+ for (int j = 0; j < hc; ++j) c[j][k] /= s + eps;
+ }
+ for (int it = 1; it < iters; ++it) {
+ for (int j = 0; j < hc; ++j) {
+ float s = 0.0f;
+ for (int k = 0; k < hc; ++k) s += c[j][k];
+ for (int k = 0; k < hc; ++k) c[j][k] /= s + eps;
+ }
+ for (int k = 0; k < hc; ++k) {
+ float s = 0.0f;
+ for (int j = 0; j < hc; ++j) s += c[j][k];
+ for (int j = 0; j < hc; ++j) c[j][k] /= s + eps;
+ }
+ }
+ for (int j = 0; j < hc; ++j)
+ for (int k = 0; k < hc; ++k) comb[j * hc + k] = c[j][k];
+}
+
+__global__ void __launch_bounds__(1024) hc_pre_norm_kernel(const float* __restrict__ h, const float* __restrict__ mix,
+ const float* __restrict__ w, int dim, int hc, float eps,
+ float* __restrict__ x) {
+ const int t = blockIdx.x;
+ h += (size_t) t * hc * dim;
+ mix += (size_t) t * 8;
+ x += (size_t) t * dim;
+ float r[8];
+ float ss = 0.0f;
+ int n = 0;
+ for (int e = threadIdx.x; e < dim; e += blockDim.x, ++n) {
+ float a = 0.0f;
+ for (int s = 0; s < hc; ++s) a += mix[s] * h[(size_t) s * dim + e];
+ r[n] = a;
+ ss += a * a;
+ }
+ const float inv = rsqrtf(block_sum(ss) / (float) dim + eps);
+ n = 0;
+ for (int e = threadIdx.x; e < dim; e += blockDim.x, ++n) x[e] = w != nullptr ? r[n] * inv * w[e] : r[n];
+}
+
+__global__ void hc_post_kernel(const float* __restrict__ y, const float* __restrict__ res,
+ const float* __restrict__ post, const float* __restrict__ comb, int dim, int hc,
+ float* __restrict__ out) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (i >= dim * hc) return;
+ const int k = i / dim, e = i - k * dim;
+ const float* r = res + (size_t) t * hc * dim;
+ float a = post[(size_t) t * 8 + k] * y[(size_t) t * dim + e];
+ for (int j = 0; j < hc; ++j) a += comb[(size_t) t * 64 + j * hc + k] * r[(size_t) j * dim + e];
+ out[(size_t) t * hc * dim + i] = a;
+}
+
+// ---------------------------------------------------------------- gemv over F16 weights, a warp a row
+__global__ void __launch_bounds__(kThreads) gemv_f16_kernel(const uint16_t* __restrict__ W, const float* __restrict__ x,
+ int n, int k, float alpha, float* __restrict__ y) {
+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ const int lane = threadIdx.x & 31, t = blockIdx.y;
+ if (row >= n) return;
+ const uint16_t* w = W + (size_t) row * k;
+ const float* xv = x + (size_t) t * k;
+ float acc = 0.0f;
+ for (int c = lane * 8; c < k; c += 32 * 8) {
+ const uint4 q = *(const uint4*) (w + c);
+ const __half2* hp = (const __half2*) &q;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = __half22float2(hp[i]);
+ acc += f.x * xv[c + 2 * i] + f.y * xv[c + 2 * i + 1];
+ }
+ }
+ acc = warp_sum(acc);
+ if (lane == 0) y[(size_t) t * n + row] = alpha * acc;
+}
+// ... for T activations (rows ldx apart, 16-byte aligned; outputs ldy apart), kRowsT of them a block (grid y): each
+// weight row read once for the block's activations, theirs in float4s - each activation's sum gemv_f16_kernel's (the
+// same products in the same order), so its bits too. (A block for all of them, 16 at most, was slower than a launch
+// a token: its accumulators and guards kept the loop from unrolling - 66 against 29 us at 5 tokens on an RTX 5060 Ti,
+// this 9)
+constexpr int kRowsT = 2;
+__global__ void __launch_bounds__(kThreads) gemv_f16_rows_kernel(const uint16_t* __restrict__ W,
+ const float* __restrict__ x, size_t ldx, int n, int k,
+ int T, float alpha, float* __restrict__ y, size_t ldy) {
+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ const int lane = threadIdx.x & 31;
+ const int t0 = blockIdx.y * kRowsT, nt = min(kRowsT, T - t0);
+ if (row >= n) return;
+ const uint16_t* w = W + (size_t) row * k;
+ float acc[kRowsT];
+#pragma unroll
+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;
+ for (int c = lane * 8; c < k; c += 32 * 8) {
+ const uint4 q = *(const uint4*) (w + c);
+ const __half2* hp = (const __half2*) &q;
+ float xv[kRowsT][8];
+#pragma unroll
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ const float* xp = x + (size_t) (t0 + t) * ldx + c;
+ const float4 a = *(const float4*) xp, b = *(const float4*) (xp + 4);
+ xv[t][0] = a.x, xv[t][1] = a.y, xv[t][2] = a.z, xv[t][3] = a.w;
+ xv[t][4] = b.x, xv[t][5] = b.y, xv[t][6] = b.z, xv[t][7] = b.w;
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = __half22float2(hp[i]);
+#pragma unroll
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ const float a = warp_sum(acc[t]);
+ if (lane == 0) y[(size_t) (t0 + t) * ldy + row] = alpha * a;
+ }
+}
+
+// ---------------------------------------------------------------- gemv over F32 weights, a warp a row (the router)
+__global__ void __launch_bounds__(kThreads) gemv_f32_kernel(const float* __restrict__ W, const float* __restrict__ x,
+ int n, int k, float* __restrict__ y, int ldy) {
+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ const int lane = threadIdx.x & 31, t = blockIdx.y;
+ if (row >= n) return;
+ const float4* w = (const float4*) (W + (size_t) row * k);
+ const float4* xv = (const float4*) (x + (size_t) t * k);
+ float acc = 0.0f;
+ for (int c = lane; c < k / 4; c += 32) {
+ const float4 a = w[c], b = xv[c];
+ acc = fmaf(a.x, b.x, acc);
+ acc = fmaf(a.y, b.y, acc);
+ acc = fmaf(a.z, b.z, acc);
+ acc = fmaf(a.w, b.w, acc);
+ }
+ acc = warp_sum(acc);
+ if (lane == 0) y[(size_t) t * ldy + row] = acc;
+}
+
+// ---------------------------------------------------------------- the router
+__global__ void __launch_bounds__(512) route_kernel(const float* __restrict__ logits, const float* __restrict__ bias,
+ int n, int k, float route_scale, int* idx, float* w,
+ const float* __restrict__ bias_vl, const VlBits vl) {
+ __shared__ float sc[1024], bs[1024];
+ __shared__ float bv[32];
+ __shared__ int bi[32];
+ __shared__ int pick[16];
+ const int t = blockIdx.x;
+ logits += (size_t) t * 1024;
+ if (bias_vl != nullptr && ((vl.w[t >> 5] >> (t & 31)) & 1u)) bias = bias_vl;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
+ const float l = logits[i];
+ const float sp = l > 20.0f ? l : log1pf(expf(l));
+ sc[i] = sqrtf(sp);
+ bs[i] = sc[i] + bias[i];
+ }
+ __syncthreads();
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ for (int r = 0; r < k; ++r) {
+ float best = -FLT_MAX;
+ int bix = 0x7fffffff;
+ for (int i = threadIdx.x; i < n; i += blockDim.x)
+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);
+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;
+ }
+ if (lane == 0) bv[wp] = best, bi[wp] = bix;
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ float b = -FLT_MAX;
+ int ix = 0x7fffffff;
+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)
+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];
+ pick[r] = ix;
+ bs[ix] = -FLT_MAX;
+ }
+ __syncthreads();
+ }
+ if (threadIdx.x == 0) {
+ float s = 0.0f;
+ for (int r = 0; r < k; ++r) s += sc[pick[r]];
+ for (int r = 0; r < k; ++r) {
+ idx[(size_t) t * kPicks + r] = pick[r];
+ w[(size_t) t * kPicks + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;
+ }
+ }
+}
+
+__global__ void swiglu_kernel(const float* __restrict__ g, const float* __restrict__ u, int n, float lim,
+ float* __restrict__ h) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ float gv = g[i], uv = u[i];
+ if (lim > 0.0f) {
+ gv = fminf(gv, lim);
+ uv = fminf(fmaxf(uv, -lim), lim);
+ }
+ h[i] = gv / (1.0f + expf(-gv)) * uv;
+}
+
+// ---------------------------------------------------------------- the compressor (ratio > 1), a window in order
+__global__ void __launch_bounds__(512) compress_kernel(const float* __restrict__ kv, const float* __restrict__ score,
+ float* kv_state, float* score_state, int ratio, int p0, int T,
+ int d, const float* __restrict__ w, float eps,
+ float* __restrict__ latent) {
+ int g = 0;
+ for (int t = 0; t < T; ++t) {
+ const int slot = (p0 + t) % ratio;
+ const bool pool = (p0 + t + 1) % ratio == 0;
+ for (int e = threadIdx.x; e < d; e += blockDim.x) {
+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];
+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];
+ }
+ __syncthreads();
+ if (!pool) continue;
+ // (kv * softmax(score, over the slots)).sum: each slot's probability first, then the sum, as torch does
+ float val[2] = {0.0f, 0.0f}; // d <= 2 * blockDim
+ float ss = 0.0f;
+ int n = 0;
+ for (int e = threadIdx.x; e < d; e += blockDim.x, ++n) {
+ float m = -FLT_MAX;
+ for (int s = 0; s < ratio; ++s) m = fmaxf(m, score_state[(size_t) s * d + e]);
+ float den = 0.0f;
+ for (int s = 0; s < ratio; ++s) den += expf(score_state[(size_t) s * d + e] - m);
+ float acc = 0.0f;
+ for (int s = 0; s < ratio; ++s)
+ acc += kv_state[(size_t) s * d + e] * (expf(score_state[(size_t) s * d + e] - m) / den);
+ val[n] = acc;
+ ss += acc * acc;
+ }
+ const float inv = rsqrtf(block_sum(ss) / (float) d + eps);
+ n = 0;
+ for (int e = threadIdx.x; e < d; e += blockDim.x, ++n) latent[(size_t) g * d + e] = val[n] * inv * w[e];
+ ++g;
+ __syncthreads();
+ }
+}
+
+// ---------------------------------------------------------------- per-query sizes
+__device__ __forceinline__ void rows_at(const Rows& r, int t, int& n, int& k) {
+ if (r.gen == 0) {
+ n = r.n[t];
+ k = r.k[t];
+ return;
+ }
+ const int nn = (int) ((r.p0 + t + 1) / r.R);
+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);
+ switch (r.gen) {
+ case 1: n = nn, k = min(r.kcap, nn); break;
+ case 2: n = nn, k = cb; break;
+ case 3: n = cb * r.blk, k = min(r.kcap, nn); break;
+ default: n = (nn + r.blk - 1) / r.blk, k = r.kcap; break;
+ }
+}
+
+// ---------------------------------------------------------------- the indexer
+// a thread a position: its key row (KF: kKvFp4p packed, or BF16) against every head's query (shared memory), relu,
+// weighted sum
+template <int HD, int KF>
+__global__ void __launch_bounds__(kThreads) index_score_kernel(const float* __restrict__ q, const float* __restrict__ w,
+ const uint8_t* __restrict__ kc, int heads, int n,
+ const int* __restrict__ cand, int cand_stride, int blk,
+ Rows lim, float* __restrict__ out,
+ long long out_stride) {
+ extern __shared__ float qs[]; // heads * HD, then heads weights
+ const int t = blockIdx.y;
+ for (int i = threadIdx.x; i < heads * HD; i += blockDim.x) qs[i] = q[(size_t) t * heads * HD + i];
+ for (int i = threadIdx.x; i < heads; i += blockDim.x) qs[heads * HD + i] = w[(size_t) t * 64 + i];
+ __syncthreads();
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ float* o = out + (long long) t * out_stride;
+ int ln, lk;
+ rows_at(lim, t, ln, lk);
+ int p = i;
+ if (cand != nullptr) {
+ if (i >= lk * blk) {
+ o[i] = -INFINITY;
+ return;
+ }
+ p = cand[(size_t) t * cand_stride + i / blk] * blk + i % blk;
+ }
+ if (p >= ln) {
+ o[i] = -INFINITY;
+ return;
+ }
+ const uint8_t* kr = kc + (size_t) p * kv_row_bytes(KF, HD);
+ float acc[32];
+#pragma unroll
+ for (int h = 0; h < 32; ++h) acc[h] = 0.0f;
+#pragma unroll 2
+ for (int c = 0; c < HD; c += 8) {
+ float kv[8];
+ if (KF == kKvFp4p) { // 8 codes in 4 bytes, the group's power-of-two scale: the values the finish rounded to
+ const uint32_t cw = *(const uint32_t*) (kr + c / 2);
+ const float ks = ue8m0_val(kr[HD / 2 + c / 32]);
+#pragma unroll
+ for (int x = 0; x < 8; ++x) kv[x] = e2m1_val14(cw >> (4 * x)) * 16384.0f * ks; // (any 2^k: exact)
+ } else {
+ const uint4 raw = *(const uint4*) (kr + c * 2);
+ const uint16_t* b = (const uint16_t*) &raw;
+#pragma unroll
+ for (int x = 0; x < 8; ++x) kv[x] = bf16f(b[x]);
+ }
+#pragma unroll
+ for (int h = 0; h < 32; ++h) {
+ if (h >= heads) break;
+ const float* qh = qs + h * HD + c;
+#pragma unroll
+ for (int x = 0; x < 8; ++x) acc[h] += qh[x] * kv[x];
+ }
+ }
+ float s = 0.0f;
+#pragma unroll
+ for (int h = 0; h < 32; ++h) {
+ if (h >= heads) break;
+ s += fmaxf(acc[h], 0.0f) * qs[heads * HD + h];
+ }
+ o[i] = s;
+}
+
+__device__ __forceinline__ uint32_t ord_key(float f) {
+ const uint32_t b = __float_as_uint(f);
+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
+}
+
+// a block a row: the k-th largest key by four 8-bit radix passes, then everything above it and enough of its ties
+__global__ void __launch_bounds__(1024) topk_kernel(const float* __restrict__ vals, long long stride, Rows r,
+ const int* __restrict__ cand, int cand_stride, int blk,
+ int* __restrict__ out, int out_stride) {
+ __shared__ unsigned hist[256];
+ __shared__ uint32_t prefix_s, mask_s;
+ __shared__ int remaining_s;
+ __shared__ int cnt_gt, cnt_eq;
+ const int t = blockIdx.x;
+ int n, k;
+ rows_at(r, t, n, k);
+ vals += (long long) t * stride;
+ out += (size_t) t * out_stride;
+ const int* cd = cand != nullptr ? cand + (size_t) t * cand_stride : nullptr;
+ if (n <= 0) return;
+ if (k >= n) {
+ for (int i = threadIdx.x; i < n; i += blockDim.x) out[i] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;
+ return;
+ }
+ if (threadIdx.x == 0) {
+ prefix_s = 0u;
+ mask_s = 0u;
+ remaining_s = k;
+ cnt_gt = 0;
+ cnt_eq = 0;
+ }
+ __syncthreads();
+ for (int shift = 24; shift >= 0; shift -= 8) {
+ for (int i = threadIdx.x; i < 256; i += blockDim.x) hist[i] = 0u;
+ __syncthreads();
+ const uint32_t prefix = prefix_s, mask = mask_s;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
+ const uint32_t u = ord_key(vals[i]);
+ if ((u & mask) == prefix) atomicAdd(&hist[(u >> shift) & 0xffu], 1u);
+ }
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ const int rem = remaining_s;
+ int cum = 0;
+ for (int d = 255; d >= 0; --d) {
+ const int h = (int) hist[d];
+ if (cum + h >= rem) {
+ prefix_s = prefix | ((uint32_t) d << shift);
+ mask_s = mask | (0xffu << shift);
+ remaining_s = rem - cum;
+ break;
+ }
+ cum += h;
+ }
+ }
+ __syncthreads();
+ }
+ const uint32_t thr = prefix_s;
+ const int remaining = remaining_s;
+ const int n_gt = k - remaining;
+ // the picks in index order (a scan a pass of 1024 values), so the attention sums them in the same order every run
+ // and the ties kept are the lowest positions
+ __shared__ int wgt[32], weq[32];
+ const int lane = threadIdx.x & 31, wid = threadIdx.x >> 5, nw = blockDim.x >> 5;
+ const unsigned below = (1u << lane) - 1u;
+ for (int i0 = 0; i0 < n; i0 += blockDim.x) {
+ const int i = i0 + threadIdx.x;
+ const uint32_t u = i < n ? ord_key(vals[i]) : 0u;
+ const bool gt = i < n && u > thr, eq = i < n && u == thr;
+ const unsigned bg = __ballot_sync(0xffffffffu, gt), be = __ballot_sync(0xffffffffu, eq);
+ if (lane == 0) {
+ wgt[wid] = __popc(bg);
+ weq[wid] = __popc(be);
+ }
+ __syncthreads();
+ if (wid == 0) {
+ const int g0 = lane < nw ? wgt[lane] : 0, e0 = lane < nw ? weq[lane] : 0;
+ int gs = g0, es = e0;
+ for (int o = 1; o < 32; o <<= 1) {
+ const int a = __shfl_up_sync(0xffffffffu, gs, o), b = __shfl_up_sync(0xffffffffu, es, o);
+ if (lane >= o) {
+ gs += a;
+ es += b;
+ }
+ }
+ if (lane < nw) {
+ wgt[lane] = cnt_gt + gs - g0;
+ weq[lane] = cnt_eq + es - e0;
+ }
+ __syncwarp();
+ if (lane == 31) {
+ cnt_gt += gs;
+ cnt_eq += es;
+ }
+ }
+ __syncthreads();
+ int slot = -1;
+ if (gt) slot = wgt[wid] + __popc(bg & below);
+ else if (eq) {
+ const int e = weq[wid] + __popc(be & below);
+ if (e < remaining) slot = n_gt + e;
+ }
+ if (slot >= 0) out[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;
+ __syncthreads();
+ }
+}
+
+__global__ void block_max_kernel(const float* __restrict__ score, long long stride, Rows r, int blk,
+ float* __restrict__ bm, long long bm_stride) {
+ const int b = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ int n, k_;
+ rows_at(r, t, n, k_);
+ const int nb = (n + blk - 1) / blk;
+ if (b >= nb) return;
+ float m = -INFINITY;
+ for (int j = 0; j < blk; ++j) {
+ const int p = b * blk + j;
+ if (p < n) m = fmaxf(m, score[(long long) t * stride + p]);
+ }
+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in
+ bm[(long long) t * bm_stride + b] = m;
+}
+
+// ---------------------------------------------------------------- sparse attention, a block a (head, query)
+constexpr int kAttTile = 64;
+
+// kContig: query t's selected rows are ckv's rows t * sel_stride .. (the scratch of rows decoded once a query) - read
+// without sel, a warp's rows of a tile loaded kPre at a time before their dots (0: as the BF16 kernel); the
+// arithmetic the same
+template <bool kContig, int kPre = 0>
+__global__ void __launch_bounds__(kThreads) sparse_attn_kernel(const float* __restrict__ q,
+ const uint16_t* __restrict__ win,
+ const uint16_t* __restrict__ ckv,
+ const int* __restrict__ sel, int sel_stride,
+ AttnRows ar, const float* __restrict__ sink, int heads,
+ int d, float scale, float* __restrict__ out) {
+ // d == 512: a thread owns 2 output values, a lane 16 of a row's dot
+ const int h = blockIdx.x, t = blockIdx.y;
+ int n_win, n_sel, w0;
+ if (ar.gen) {
+ const long long ta = ar.t0 + t, pa = ar.p0 + ta;
+ const long long first = max((long long) ar.win - ar.p0, ta + 1); // the window buffer row of max(0, pa - win + 1)
+ w0 = (int) first;
+ n_win = (int) (ar.win + ta - first + 1);
+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;
+ } else {
+ n_win = ar.nwin[t], n_sel = ar.nsel[t], w0 = ar.win0[t];
+ }
+ const uint16_t* wbase = win + (size_t) w0 * d;
+ const int* sl = sel + (size_t) t * sel_stride;
+ __shared__ float qs[512];
+ __shared__ float sc[kAttTile];
+ __shared__ const uint16_t* rows[kAttTile];
+ __shared__ float m_run_s, tile_scale_s, den_s;
+ for (int i = threadIdx.x; i < d; i += blockDim.x) qs[i] = q[((size_t) t * heads + h) * d + i];
+ if (threadIdx.x == 0) {
+ m_run_s = -1e30f;
+ den_s = 0.0f;
+ }
+ __syncthreads();
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ float o0 = 0.0f, o1 = 0.0f;
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += kAttTile) {
+ const int nt = min(kAttTile, total - t0);
+ if (threadIdx.x < nt) {
+ const int j = t0 + threadIdx.x;
+ rows[threadIdx.x] = j < n_win ? wbase + (size_t) j * d
+ : ckv + (kContig ? (size_t) t * sel_stride + (j - n_win) : (size_t) sl[j - n_win]) * d;
+ }
+ __syncthreads();
+ if (kContig && kPre > 0) { // the warp's rows of the tile (wp, wp + 8, ..), kPre loads at a time, then dots
+#pragma unroll
+ for (int k0 = 0; k0 < kAttTile / 8; k0 += (kPre > 0 ? kPre : 1)) {
+ uint4 ra[kPre > 0 ? kPre : 1], rb[kPre > 0 ? kPre : 1];
+#pragma unroll
+ for (int k = 0; k < kPre; ++k) {
+ const int jj = wp + 8 * (k0 + k);
+ if (jj < nt) {
+ ra[k] = *(const uint4*) (rows[jj] + lane * 16);
+ rb[k] = *(const uint4*) (rows[jj] + lane * 16 + 8);
+ }
+ }
+#pragma unroll
+ for (int k = 0; k < kPre; ++k) {
+ const int jj = wp + 8 * (k0 + k);
+ if (jj < nt) {
+ const uint16_t* pa = (const uint16_t*) &ra[k];
+ const uint16_t* pb = (const uint16_t*) &rb[k];
+ float s = 0.0f;
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + x] * bf16f(pa[x]);
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + 8 + x] * bf16f(pb[x]);
+ s = warp_sum(s);
+ if (lane == 0) sc[jj] = s * scale;
+ }
+ }
+ }
+ } else
+ for (int jj = wp; jj < nt; jj += kThreads / 32) {
+ const uint16_t* rr = rows[jj];
+ const uint4 a = *(const uint4*) (rr + lane * 16);
+ const uint4 b = *(const uint4*) (rr + lane * 16 + 8);
+ const uint16_t* pa = (const uint16_t*) &a;
+ const uint16_t* pb = (const uint16_t*) &b;
+ float s = 0.0f;
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + x] * bf16f(pa[x]);
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + 8 + x] * bf16f(pb[x]);
+ s = warp_sum(s);
+ if (lane == 0) sc[jj] = s * scale;
+ }
+ __syncthreads();
+ if (wp == 0) {
+ float m = -INFINITY;
+ for (int j = lane; j < nt; j += 32) m = fmaxf(m, sc[j]);
+ m = warp_max(m);
+ const float m_old = m_run_s;
+ const float m_new = fmaxf(m_old, m);
+ float s = 0.0f;
+ for (int j = lane; j < nt; j += 32) {
+ const float p = expf(sc[j] - m_new);
+ sc[j] = p;
+ s += p;
+ }
+ s = warp_sum(s);
+ __syncwarp();
+ if (lane == 0) {
+ tile_scale_s = expf(m_old - m_new);
+ den_s = den_s * tile_scale_s + s;
+ m_run_s = m_new;
+ }
+ }
+ __syncthreads();
+ const float cs = tile_scale_s;
+ o0 *= cs;
+ o1 *= cs;
+ const int c = threadIdx.x * 2;
+ if (kContig && kPre > 0) { // (eight rows' loads at once)
+#pragma unroll 8
+ for (int j = 0; j < nt; ++j) {
+ const uint32_t pr = *(const uint32_t*) (rows[j] + c);
+ o0 += sc[j] * bf16f((uint16_t) (pr & 0xffffu));
+ o1 += sc[j] * bf16f((uint16_t) (pr >> 16));
+ }
+ } else
+ for (int j = 0; j < nt; ++j) {
+ const uint32_t pr = *(const uint32_t*) (rows[j] + c);
+ o0 += sc[j] * bf16f((uint16_t) (pr & 0xffffu));
+ o1 += sc[j] * bf16f((uint16_t) (pr >> 16));
+ }
+ __syncthreads();
+ }
+ const float den = den_s + expf(sink[h] - m_run_s);
+ const int c = threadIdx.x * 2;
+ float* ov = out + ((size_t) t * heads + h) * d;
+ ov[c] = o0 / den;
+ ov[c + 1] = o1 / den;
+}
+
+// ---------------------------------------------------------------- sparse attention over the packed compressed rows
+// G heads of one query a block (8 / G warps a head): each pass stages 16 of a tile's rows in shared memory as BF16
+// (the window's rows as they are; the packed rows decoded - exact in BF16), once for all G heads, through one 16-byte
+// load a thread. Each head's arithmetic is sparse_attn_kernel's, operation for operation: its tiles of 64 rows, its
+// dots by a warp's lanes (16 values each, in order) and the same butterfly, its softmax by one warp the same way, each
+// output value's weighted sum by the same thread in row order - the same bits for any G.
+constexpr int kStage = 16; // rows a pass stages (a thread: 32 values of one)
+
+// values part * 32 .. + 32 of a packed compressed row (kKvFp4, 512 values) as BF16 into dst[0 .. 4) (exact: the
+// values are E2M1 times an E4M3 scale)
+__device__ __forceinline__ void unpack32(const uint8_t* rr, int part, uint4* dst) {
+ const uint4 cw = *(const uint4*) (rr + part * 16); // 32 codes, two scale groups
+ const float k0 = e4m3_value(rr[256 + part * 2]) * 16384.0f, k1 = e4m3_value(rr[256 + part * 2 + 1]) * 16384.0f;
+ const uint32_t wv[4] = {cw.x, cw.y, cw.z, cw.w};
+ uint32_t o[16];
+#pragma unroll
+ for (int i = 0; i < 16; ++i) { // values 2i, 2i + 1 (the low and the high nibble of byte i)
+ const uint32_t b = wv[i >> 2] >> (8 * (i & 3));
+ const float ks = i < 8 ? k0 : k1;
+ const float lo = e2m1_val14(b) * ks, hi = e2m1_val14(b >> 4) * ks; // (exact: their BF16 bits)
+ o[i] = (__float_as_uint(lo) >> 16) | (__float_as_uint(hi) & 0xffff0000u);
+ }
+ dst[0] = make_uint4(o[0], o[1], o[2], o[3]);
+ dst[1] = make_uint4(o[4], o[5], o[6], o[7]);
+ dst[2] = make_uint4(o[8], o[9], o[10], o[11]);
+ dst[3] = make_uint4(o[12], o[13], o[14], o[15]);
+}
+
+// a few queries' attention (a decode step, a short window): the rows each query selects decoded once to BF16 rows -
+// out[t * cap + i] = ckv[sel[t][i]] for i < the query's n_sel - for the BF16 kernel reading them in place (once a
+// query, not once a head)
+__global__ void __launch_bounds__(kThreads) unpack_sel_kernel(const uint8_t* __restrict__ ckv,
+ const int* __restrict__ sel, int sel_stride, AttnRows ar,
+ int cap, uint16_t* __restrict__ out) {
+ const int t = blockIdx.y, i = blockIdx.x * kStage + (int) (threadIdx.x >> 4), part = threadIdx.x & 15;
+ int n_sel;
+ if (ar.gen) {
+ const long long pa = ar.p0 + ar.t0 + t;
+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;
+ } else {
+ n_sel = ar.nsel[t];
+ }
+ if (i >= n_sel) return;
+ const size_t r = (size_t) t * cap + i;
+ unpack32(ckv + (size_t) sel[(size_t) t * sel_stride + i] * kv_row_bytes(kKvFp4, 512), part, (uint4*) (out + r * 512 + part * 32));
+}
+
+template <int G>
+__global__ void __launch_bounds__(kThreads) sparse_attn_packed_kernel(const float* __restrict__ q,
+ const uint16_t* __restrict__ win,
+ const uint8_t* __restrict__ ckv,
+ const int* __restrict__ sel, int sel_stride,
+ AttnRows ar, const float* __restrict__ sink,
+ int heads, float scale, float* __restrict__ out) {
+ constexpr int d = 512, W = 8 / G; // (kThreads / 32 warps)
+ const int hb = blockIdx.x * G, t = blockIdx.y;
+ int n_win, n_sel, w0;
+ if (ar.gen) {
+ const long long ta = ar.t0 + t, pa = ar.p0 + ta;
+ const long long first = max((long long) ar.win - ar.p0, ta + 1);
+ w0 = (int) first;
+ n_win = (int) (ar.win + ta - first + 1);
+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;
+ } else {
+ n_win = ar.nwin[t], n_sel = ar.nsel[t], w0 = ar.win0[t];
+ }
+ const uint16_t* wbase = win + (size_t) w0 * d;
+ const int* sl = sel + (size_t) t * sel_stride;
+ const size_t crow = kv_row_bytes(kKvFp4, d);
+ __shared__ __align__(16) uint16_t tile[kStage * d];
+ __shared__ float sc[G][kAttTile];
+ __shared__ const uint8_t* rows[kAttTile];
+ __shared__ float m_run_s[G], tile_scale_s[G], den_s[G];
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5, gw = wp / W, wr = wp % W;
+ float qr[16]; // this lane's 16 values of its warp's head
+#pragma unroll
+ for (int x = 0; x < 16; ++x) qr[x] = q[((size_t) t * heads + hb + gw) * d + lane * 16 + x];
+ if (threadIdx.x < G) {
+ m_run_s[threadIdx.x] = -1e30f;
+ den_s[threadIdx.x] = 0.0f;
+ }
+ float o0[G], o1[G];
+#pragma unroll
+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;
+ const int c = threadIdx.x * 2;
+ const int sr = threadIdx.x >> 4, part = threadIdx.x & 15; // the staging: a thread's row and 32 values
+ // rows [s0, s0 + n) of the tile at t0 into tile[] as BF16
+ auto stage = [&](int t0, int s0, int n) {
+ if (sr >= n) return;
+ const int j = s0 + sr;
+ uint4* dst = (uint4*) (tile + sr * d + part * 32);
+ if (t0 + j < n_win) { // a window row: BF16 already
+ const uint4* src = (const uint4*) (rows[j] + part * 64);
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
+ dst[3] = src[3];
+ return;
+ }
+ unpack32(rows[j], part, dst);
+ };
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += kAttTile) {
+ const int nt = min(kAttTile, total - t0);
+ if (threadIdx.x < nt) {
+ const int j = t0 + threadIdx.x;
+ rows[threadIdx.x] = j < n_win ? (const uint8_t*) (wbase + (size_t) j * d) : ckv + (size_t) sl[j - n_win] * crow;
+ }
+ __syncthreads();
+ for (int s0 = 0; s0 < nt; s0 += kStage) { // the dots
+ const int n = min(kStage, nt - s0);
+ stage(t0, s0, n);
+ __syncthreads();
+#if !defined(STRATA_USE_HIP) // (HIP leaves a loop with shuffles and a remainder rolled anyway)
+#pragma unroll 2
+#endif
+ for (int r = wr; r < n; r += W) { // (two rows' chains at once)
+ const uint16_t* vr = tile + r * d + lane * 16;
+ const uint4 a = *(const uint4*) vr;
+ const uint4 b = *(const uint4*) (vr + 8);
+ const uint16_t* pa = (const uint16_t*) &a;
+ const uint16_t* pb = (const uint16_t*) &b;
+ float s = 0.0f;
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qr[x] * bf16f(pa[x]);
+#pragma unroll
+ for (int x = 0; x < 8; ++x) s += qr[8 + x] * bf16f(pb[x]);
+ s = warp_sum(s);
+ if (lane == 0) sc[gw][s0 + r] = s * scale;
+ }
+ __syncthreads();
+ }
+ if (wp < G) { // head wp's softmax over the tile
+ float* scg = sc[wp];
+ float m = -INFINITY;
+ for (int j = lane; j < nt; j += 32) m = fmaxf(m, scg[j]);
+ m = warp_max(m);
+ const float m_old = m_run_s[wp];
+ const float m_new = fmaxf(m_old, m);
+ float s = 0.0f;
+ for (int j = lane; j < nt; j += 32) {
+ const float p = expf(scg[j] - m_new);
+ scg[j] = p;
+ s += p;
+ }
+ s = warp_sum(s);
+ __syncwarp();
+ if (lane == 0) {
+ tile_scale_s[wp] = expf(m_old - m_new);
+ den_s[wp] = den_s[wp] * tile_scale_s[wp] + s;
+ m_run_s[wp] = m_new;
+ }
+ }
+ __syncthreads();
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ const float cs = tile_scale_s[g];
+ o0[g] *= cs;
+ o1[g] *= cs;
+ }
+ for (int s0 = 0; s0 < nt; s0 += kStage) { // the weighted sums
+ const int n = min(kStage, nt - s0);
+ stage(t0, s0, n);
+ __syncthreads();
+ for (int r = 0; r < n; ++r) {
+ const uint32_t pr = *(const uint32_t*) (tile + r * d + c);
+ const float v0 = bf16f((uint16_t) (pr & 0xffffu)), v1 = bf16f((uint16_t) (pr >> 16));
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ o0[g] += sc[g][s0 + r] * v0;
+ o1[g] += sc[g][s0 + r] * v1;
+ }
+ }
+ __syncthreads();
+ }
+ }
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ const float den = den_s[g] + expf(sink[hb + g] - m_run_s[g]);
+ float* ov = out + ((size_t) t * heads + hb + g) * d;
+ ov[c] = o0[g] / den;
+ ov[c + 1] = o1[g] / den;
+ }
+}
+
+// the ring's rows are kKvFp8: E4M3 codes, a UE8M0 scale per 32 (decoded: exactly the BF16 values they were made from)
+__global__ void ring_to_ext_kernel(const uint8_t* __restrict__ ring, uint16_t* __restrict__ ext, long long p0, int n,
+ int d) {
+ const int r = blockIdx.x;
+ const long long p = p0 - n + r;
+ const uint8_t* row = ring + (size_t) (p >= 0 ? p % n : 0) * kv_row_bytes(kKvFp8, d);
+ for (int e = threadIdx.x; e < d; e += blockDim.x)
+ ext[(size_t) r * d + e] = p >= 0 ? f2bf16(e4m3_value(row[e]) * ue8m0_val(row[d + e / 32])) : (uint16_t) 0;
+}
+
+// a window row's BF16 values (FP8 roundings: each an E4M3 value times its group's power of two) as E4M3 codes at the
+// smallest power-of-two scale 2^k their group fits under (max |v| <= 448 * 2^k = 1.75 * 2^(8 + k)): never above the
+// rounding's own scale, so each v / 2^k is that E4M3 value times a power of two >= 1, at most 448 - an E4M3 value again
+__global__ void __launch_bounds__(kThreads) ext_to_ring_kernel(const uint16_t* __restrict__ ext,
+ uint8_t* __restrict__ ring, long long p0, int n, int T,
+ int d, uint8_t* __restrict__ backup) {
+ const int t = blockIdx.x;
+ if (T - t > n) return; // overwritten by a later row of the window anyway
+ const size_t rb = kv_row_bytes(kKvFp8, d);
+ uint8_t* row = ring + (size_t) ((p0 + t) % n) * rb;
+ if (backup != nullptr) {
+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) backup[(size_t) t * rb + i] = row[i];
+ __syncthreads();
+ }
+ const uint16_t* src = ext + (size_t) (n + t) * d;
+ for (int e0 = 0; e0 < d; e0 += blockDim.x) { // (d a multiple of 32: a warp's 32 values one group)
+ const int e = e0 + (int) threadIdx.x;
+ const float v = e < d ? bf16f(src[e]) : 0.0f;
+ uint32_t a = __float_as_uint(v) & 0x7fffffffu; // (the magnitudes' order as bits)
+ for (int o = 16; o > 0; o >>= 1) a = max(a, (uint32_t) __shfl_xor_sync(0xffffffffu, a, o));
+ const int k = a == 0u ? 0 : (int) (a >> 23) - 135 + ((a & 0x7fffffu) > 0x600000u ? 1 : 0);
+ if (e >= d) continue;
+ row[e] = (uint8_t) e4m3_code(v, k);
+ if ((e & 31) == 0) row[d + e / 32] = (uint8_t) (k + 127);
+ }
+}
+
+__global__ void ring_restore_kernel(uint8_t* __restrict__ ring, const uint8_t* __restrict__ backup, long long p0, int n,
+ int keep, size_t rb) {
+ const int t = keep + blockIdx.x;
+ uint8_t* row = ring + (size_t) ((p0 + t) % n) * rb;
+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) row[i] = backup[(size_t) t * rb + i];
+}
+__global__ void ring_restore_batch_kernel(const RingBatch b, long long p0, int n, int keep, size_t rb) {
+ const int t = keep + blockIdx.x, li = blockIdx.y;
+ uint8_t* row = b.ring[li] + (size_t) ((p0 + t) % n) * rb;
+ const uint8_t* src = b.bak[li] + (size_t) t * rb;
+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) row[i] = src[i];
+}
+
+__global__ void hc_mean_kernel(const float* __restrict__ h, int hc, int dim, float* __restrict__ out,
+ long long out_stride) {
+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * hc + s) * dim + e];
+ out[(long long) t * out_stride + e] = a / (float) hc;
+}
+
+__global__ void iota_rows_kernel(int* out, int n, int stride) {
+ const int t = blockIdx.x;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) out[(size_t) t * stride + i] = i;
+}
+
+// a warp a vocabulary row: E[*tok] (shared memory) . H[v]
+__global__ void __launch_bounds__(kThreads) markov_kernel(float* __restrict__ logits, const uint16_t* __restrict__ E,
+ const uint16_t* __restrict__ H, const int* __restrict__ tok,
+ int V, int R) {
+ extern __shared__ float er[];
+ const uint16_t* erow = E + (size_t) (*tok) * R;
+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);
+ __syncthreads();
+ const int lane = threadIdx.x & 31;
+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ if (v >= V) return;
+ const uint16_t* hr = H + (size_t) v * R;
+ float a = 0.0f;
+ for (int c = lane * 8; c < R; c += 32 * 8) {
+ const uint4 q = *(const uint4*) (hr + c);
+ const __half2* hp = (const __half2*) &q;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = __half22float2(hp[i]);
+ a += f.x * er[c + 2 * i] + f.y * er[c + 2 * i + 1];
+ }
+ }
+ a = warp_sum(a);
+ if (lane == 0) logits[v] += a;
+}
+
+// ---------------------------------------------------------------- the markov bias over a head group (markov_share)
+// a member's rows [v0, v0 + vn) of the bias (markov_kernel's sums, the prev token's E row already on the device, BF16)
+__global__ void __launch_bounds__(kThreads) markov_share_kernel(float* __restrict__ logits,
+ const uint16_t* __restrict__ H,
+ const uint16_t* __restrict__ erow, int vn, int R) {
+ extern __shared__ float er[];
+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);
+ __syncthreads();
+ const int lane = threadIdx.x & 31;
+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);
+ if (v >= vn) return;
+ const uint16_t* hr = H + (size_t) v * R;
+ float a = 0.0f;
+ for (int c = lane * 8; c < R; c += 32 * 8) {
+ const uint4 q = *(const uint4*) (hr + c);
+ const __half2* hp = (const __half2*) &q;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = __half22float2(hp[i]);
+ a += f.x * er[c + 2 * i] + f.y * er[c + 2 * i + 1];
+ }
+ }
+ a = warp_sum(a);
+ if (lane == 0) logits[v] += a;
+}
+// the share's argmax (the greedy sampler's rule: the larger value, on a tie the lower index) -> slot {value, v0 + i,
+// seq}, the seq last
+__global__ void __launch_bounds__(1024) argmax_share_kernel(const float* __restrict__ row, int vn, int v0,
+ unsigned* slot, unsigned seq) {
+ __shared__ float sv[32];
+ __shared__ int si[32];
+ float bv = __int_as_float(0xff800000);
+ int bi = 0x7fffffff;
+ for (int i = threadIdx.x; i < vn; i += blockDim.x) {
+ const float x = row[i];
+ if (x > bv) bv = x, bi = i; // (ascending: the lowest index of a tie stays)
+ }
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);
+ if (ov > bv || (ov == bv && oi < bi)) bv = ov, bi = oi;
+ }
+ if ((threadIdx.x & 31) == 0) sv[threadIdx.x >> 5] = bv, si[threadIdx.x >> 5] = bi;
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ for (int w = 1; w < (int) (blockDim.x >> 5); ++w)
+ if (sv[w] > bv || (sv[w] == bv && si[w] < bi)) bv = sv[w], bi = si[w];
+ slot[0] = __float_as_uint(bv);
+ slot[1] = (unsigned) (v0 + (bi == 0x7fffffff ? 0 : bi));
+ __threadfence_system();
+ *(volatile unsigned*) (slot + 2) = seq;
+ }
+}
+// every member's share argmax of this step (slots nm apart) -> the global one into *tok, and its E row (host-mapped
+// BF16 [V][R]) into erow for the next step's bias
+__global__ void markov_pick_kernel(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E,
+ uint16_t* erow, int R) {
+ __shared__ int pick;
+ if ((int) threadIdx.x < nm)
+ while ((int) (*(volatile const unsigned*) (slots + (size_t) threadIdx.x * 4 + 2) - seq) < 0) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+ __nanosleep(32);
+#endif
+ }
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ float bv = __int_as_float(0xff800000);
+ int bi = 0x7fffffff;
+ for (int m = 0; m < nm; ++m) { // (members in vocabulary order: a tie keeps the lower index)
+ const float v = __uint_as_float(*(volatile const unsigned*) (slots + (size_t) m * 4));
+ const int i = (int) *(volatile const unsigned*) (slots + (size_t) m * 4 + 1);
+ if (v > bv || (v == bv && i < bi)) bv = v, bi = i;
+ }
+ pick = bi;
+ *tok = bi;
+ }
+ __syncthreads();
+#if defined(STRATA_USE_HIP) // (E is fixed once loaded: a plain load is as fresh)
+ for (int i = threadIdx.x; i < R; i += blockDim.x) erow[i] = E[(size_t) pick * R + i];
+#else
+ for (int i = threadIdx.x; i < R; i += blockDim.x) erow[i] = __ldcv(E + (size_t) pick * R + i);
+#endif
+}
+
+__global__ void __launch_bounds__(1024) confidence_kernel(const float* __restrict__ x, const uint16_t* __restrict__ E,
+ const int* __restrict__ tok, const float* __restrict__ w,
+ int dim, int R, float* __restrict__ out) {
+ const int t = blockIdx.x;
+ const float* xt = x + (size_t) t * dim;
+ const uint16_t* er = E + (size_t) tok[t] * R;
+ float a = 0.0f;
+ for (int i = threadIdx.x; i < dim + R; i += blockDim.x) a += w[i] * (i < dim ? xt[i] : bf16f(er[i - dim]));
+ a = block_sum(a);
+ if (threadIdx.x == 0) out[t] = 1.0f / (1.0f + expf(-a));
+}
+
+// ---------------------------------------------------------------- engram
+__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,
+ float* __restrict__ out, bool q6k) {
+ const int r = blockIdx.x;
+ if (q6k) { // (ggml Q6_K superblocks at the row's start)
+ for (int e = threadIdx.x; e < hd; e += blockDim.x)
+ out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);
+ return;
+ }
+ for (int e = threadIdx.x; e < hd; e += blockDim.x) {
+ const float hv = e4m3_value(rows[(size_t) r * hd + e]);
+ const uint8_t sb = scales[(size_t) r * (hd / 32) + e / 32];
+ const float s = sb == 0xffu ? __int_as_float(0x7fc00000) : (sb == 0 ? 5.877471754111438e-39f
+ : __uint_as_float((uint32_t) sb << 23));
+ out[(size_t) r * hd + e] = hv * s;
+ }
+}
+
+__global__ void __launch_bounds__(1024) engram_gate_kernel(float* __restrict__ h, const float* __restrict__ kv,
+ const float* __restrict__ w, int hc, int dim, float eps) {
+ const int k = blockIdx.x, t = blockIdx.y;
+ float* hk = h + ((size_t) t * hc + k) * dim;
+ const float* kvt = kv + (size_t) t * (hc + 1) * dim;
+ const float* key = kvt + (size_t) k * dim;
+ const float* val = kvt + (size_t) hc * dim;
+ const float* wk = w + (size_t) k * dim;
+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;
+ for (int e = threadIdx.x; e < dim; e += blockDim.x) {
+ const float hv = hk[e], kv_ = key[e];
+ sh += hv * hv;
+ sk += kv_ * kv_;
+ sd += hv * wk[e] * kv_;
+ }
+ sh = block_sum(sh);
+ sk = block_sum(sk);
+ sd = block_sum(sd);
+ const float rstd = rsqrtf(sh / (float) dim + eps) * rsqrtf(sk / (float) dim + eps);
+ const float dot = sd * rstd * rsqrtf((float) dim);
+ const float sq = copysignf(sqrtf(fmaxf(fabsf(dot), 1e-6f)), dot);
+ const float gate = 1.0f / (1.0f + expf(-sq));
+ for (int e = threadIdx.x; e < dim; e += blockDim.x) hk[e] = hk[e] + gate * val[e];
+}
+
+__global__ void gather_kernel(const float* __restrict__ x, Gather g, int cols, float* __restrict__ out) {
+ const int r = blockIdx.y;
+ const int c = blockIdx.x * blockDim.x + threadIdx.x;
+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];
+}
+
+__global__ void combine_kernel(const float* __restrict__ ed, const float* __restrict__ cpu,
+ const float* __restrict__ sh, Combine cm, int dim, float* __restrict__ out) {
+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int j = 0; j < cm.n[t]; ++j) a += cm.w[t][j] * ed[(size_t) cm.row[t][j] * dim + e];
+ if (cpu != nullptr && cm.cpu[t]) a += cpu[(size_t) t * dim + e];
+ if (sh != nullptr) a += sh[(size_t) t * dim + e];
+ out[(size_t) t * dim + e] = a;
+}
+
+__global__ void __launch_bounds__(512) route_publish_kernel(const float* __restrict__ logits,
+ const float* __restrict__ bias, int n, int k,
+ float route_scale, const unsigned long long* __restrict__ vram,
+ const float* __restrict__ x, int dim, int* idx, float* w,
+ unsigned long long* plan_ptr, Req* req,
+ const float* __restrict__ bias_vl, unsigned vl) {
+ __shared__ float sc[1024], bs[1024];
+ __shared__ float bv[32];
+ __shared__ int bi[32];
+ __shared__ int pick[16];
+ __shared__ int on_cpu;
+ const int t = blockIdx.x;
+ logits += (size_t) t * 1024;
+ if (bias_vl != nullptr && ((vl >> t) & 1u)) bias = bias_vl;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
+ const float l = logits[i];
+ const float sp = l > 20.0f ? l : log1pf(expf(l));
+ sc[i] = sqrtf(sp);
+ bs[i] = sc[i] + bias[i];
+ }
+ __syncthreads();
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ for (int r = 0; r < k; ++r) {
+ float best = -FLT_MAX;
+ int bix = 0x7fffffff;
+ for (int i = threadIdx.x; i < n; i += blockDim.x)
+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);
+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;
+ }
+ if (lane == 0) bv[wp] = best, bi[wp] = bix;
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ float b = -FLT_MAX;
+ int ix = 0x7fffffff;
+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)
+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];
+ pick[r] = ix;
+ bs[ix] = -FLT_MAX;
+ }
+ __syncthreads();
+ }
+ if (threadIdx.x == 0) {
+ float s = 0.0f;
+ for (int r = 0; r < k; ++r) s += sc[pick[r]];
+ int cpu = 0;
+ for (int r = 0; r < k; ++r) {
+ const float wv = sc[pick[r]] / (s + 1e-20f) * route_scale;
+ idx[(size_t) t * kPicks + r] = pick[r];
+ w[(size_t) t * kPicks + r] = wv;
+ plan_ptr[(size_t) t * kPicks + r] = vram[pick[r]];
+ cpu |= vram[pick[r]] == 0ull;
+ if (req != nullptr) {
+ req->ids[t][r] = pick[r];
+ req->w[t][r] = wv;
+ }
+ }
+ on_cpu = cpu;
+ }
+ // the token's FFN input to the host only where the CPU lane reads it (a pick of its on the CPU lane): each row
+ // 20 KB over PCIe, where most windows have none
+ __syncthreads();
+ if (req != nullptr && on_cpu)
+ for (int i = threadIdx.x; i < dim; i += blockDim.x) req->x[t][i] = x[(size_t) t * dim + i];
+}
+
+// a later layer's m likely picks (route_kernel's choice from its logits) into req->ahead[t][off ..)
+__global__ void __launch_bounds__(512) route_ahead_kernel(const float* __restrict__ logits,
+ const float* __restrict__ bias, int n, int m, Req* req,
+ int off) {
+ __shared__ float bs[1024];
+ __shared__ float bv[32];
+ __shared__ int bi[32];
+ const int t = blockIdx.x;
+ logits += (size_t) t * 1024;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
+ const float l = logits[i];
+ const float sp = l > 20.0f ? l : log1pf(expf(l));
+ bs[i] = sqrtf(sp) + bias[i];
+ }
+ __syncthreads();
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ for (int r = 0; r < m; ++r) {
+ float best = -FLT_MAX;
+ int bix = 0x7fffffff;
+ for (int i = threadIdx.x; i < n; i += blockDim.x)
+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;
+ for (int o = 16; o > 0; o >>= 1) {
+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);
+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);
+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;
+ }
+ if (lane == 0) bv[wp] = best, bi[wp] = bix;
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ float b = -FLT_MAX;
+ int ix = 0x7fffffff;
+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)
+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];
+ req->ahead[t][off + r] = ix < n ? ix : -1;
+ if (ix < n) bs[ix] = -FLT_MAX;
+ }
+ __syncthreads();
+ }
+}
+
+__global__ void publish_kernel(Req* req, int T, unsigned seq, int na0, int na1) {
+ __threadfence_system();
+ req->na[0] = na0;
+ req->na[1] = na1;
+ req->T = T;
+ __threadfence_system();
+ *(volatile unsigned*) &req->seq = seq;
+}
+
+__global__ void wait_seq_kernel(const unsigned* seq_ptr, unsigned seq) {
+ if (threadIdx.x != 0) return;
+ while (*(volatile const unsigned*) seq_ptr != seq) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+ __nanosleep(200);
+#endif
+ }
+ __threadfence_system();
+}
+
+__global__ void init_pre_kernel(float* pre, int T) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;
+}
+
+
+// the hand-offs through host-mapped memory: a grid-stride copy of the segments (16 bytes a thread), then (put, flag
+// given) the last block to finish sets the flag after a system fence; a get first waits for the flag
+__device__ __forceinline__ void xfer_copy(const Xfer& x, bool from_host) {
+ size_t total = 0;
+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
+ for (size_t q = (size_t) blockIdx.x * blockDim.x + threadIdx.x; q < total; q += (size_t) gridDim.x * blockDim.x) {
+ size_t r = q;
+ int i = 0;
+ while (r >= x.bytes[i] / 16) r -= x.bytes[i++] / 16;
+ const uint4* src = (const uint4*) x.src[i] + r;
+ ((uint4*) x.dst[i])[r] = from_host ? ld_fresh(src) : *src;
+ }
+}
+
+__global__ void __launch_bounds__(kThreads) xfer_put_kernel(const Xfer x, unsigned* flag, unsigned seq, unsigned* cnt) {
+ xfer_copy(x, false);
+ if (flag == nullptr) return;
+ __threadfence_system();
+ __syncthreads();
+ if (threadIdx.x == 0 && atomicAdd(cnt, 1u) == gridDim.x - 1) {
+ *cnt = 0u;
+ __threadfence_system();
+ *(volatile unsigned*) flag = seq;
+ }
+}
+
+__device__ __forceinline__ void wait_at_least(const unsigned* flag, unsigned seq) {
+ while ((int) (*(volatile const unsigned*) flag - seq) < 0) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+ __nanosleep(100);
+#endif
+ }
+}
+
+__global__ void __launch_bounds__(kThreads) xfer_get_kernel(const Xfer x, const unsigned* flag, unsigned seq) {
+ if (threadIdx.x == 0) wait_at_least(flag, seq);
+ __syncthreads();
+ xfer_copy(x, true);
+}
+
+__global__ void wait_flags_kernel(const unsigned* flags, int n, int stride, unsigned seq) {
+ if (threadIdx.x != 0) return;
+ for (int i = 0; i < n; ++i) wait_at_least(flags + (size_t) i * stride, seq);
+ __threadfence_system();
+}
+__global__ void set_flag_kernel(unsigned* flag, unsigned v) {
+ __threadfence_system();
+ *(volatile unsigned*) flag = v;
+}
+
+// three members or more: a reduce-scatter and an all-gather in one launch, a block a contiguous range of the row cut
+// in nm chunks. The block puts its range of this member's partial out and flags it; once every member's flag of the
+// range is up it sums chunk mi of the range in member order (ar_sum_kernel's sums, bit for bit), puts that out and
+// flags it; once every member's sum flag is up it reads the other chunks' sums. Each member reads about two rows over
+// PCIe whatever nm is, where every member reading every other row grows with nm
+__global__ void __launch_bounds__(kThreads) ar_rs_kernel(float* x, int n4, float* rows, float* sums, size_t stride,
+ unsigned* bflags, int nm, int mi, int slot, int slots,
+ unsigned seq) {
+ float4* x4 = (float4*) x;
+ const int per = (n4 + (int) gridDim.x - 1) / (int) gridDim.x;
+ const int lo = blockIdx.x * per, hi = min(n4, lo + per), cs = (hi - lo + nm - 1) / nm;
+ auto rflag = [&](int m) { return bflags + (((size_t) m * slots + slot) * kArBlocks + blockIdx.x) * 8; };
+ auto sflag = [&](int m) {
+ return bflags + ((((size_t) nm + m) * slots + slot) * kArBlocks + blockIdx.x) * 8;
+ };
+ auto row = [&](int m) { return (float4*) (rows + ((size_t) m * slots + slot) * stride); };
+ float4* sum = (float4*) (sums + (size_t) slot * stride);
+ if (lo < hi) {
+ float4* mine = row(mi);
+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x) mine[q] = x4[q];
+ }
+ __threadfence_system();
+ __syncthreads();
+ if (threadIdx.x == 0) *(volatile unsigned*) rflag(mi) = seq;
+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(rflag(threadIdx.x), seq);
+ __syncthreads();
+ const int c0 = min(hi, lo + mi * cs), c1 = min(hi, c0 + cs);
+ for (int q = c0 + threadIdx.x; q < c1; q += blockDim.x) {
+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ for (int m = 0; m < nm; ++m) {
+ const float4 p = m == mi ? x4[q] : ld_fresh(row(m) + q);
+ if (m == 0) v = p;
+ else {
+ v.x = __fadd_rn(v.x, p.x);
+ v.y = __fadd_rn(v.y, p.y);
+ v.z = __fadd_rn(v.z, p.z);
+ v.w = __fadd_rn(v.w, p.w);
+ }
+ }
+ sum[q] = v;
+ x4[q] = v;
+ }
+ __threadfence_system();
+ __syncthreads();
+ if (threadIdx.x == 0) *(volatile unsigned*) sflag(mi) = seq;
+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(sflag(threadIdx.x), seq);
+ __syncthreads();
+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x)
+ if (q < c0 || q >= c1) x4[q] = ld_fresh(sum + q);
+}
+
+// ar_rs_kernel with the rows and the sums in FP16 (DS4_AR_F16): half the bytes over PCIe. Every member's partial is
+// rounded to FP16 (the owner's too), summed in F32 in member order, the sum rounded to FP16 - and every member, the
+// owner too, takes that rounded sum, so the members' copies of the stream stay the same
+__device__ __forceinline__ uint2 to_h4(float4 v) {
+ const __half2 a = __floats2half2_rn(v.x, v.y), b = __floats2half2_rn(v.z, v.w);
+ return make_uint2(*(const unsigned*) &a, *(const unsigned*) &b);
+}
+__device__ __forceinline__ float4 from_h4(uint2 u) {
+ const float2 a = __half22float2(*(const __half2*) &u.x), b = __half22float2(*(const __half2*) &u.y);
+ return make_float4(a.x, a.y, b.x, b.y);
+}
+__device__ __forceinline__ float4 sat4(float4 v) { // (FP16's range: a partial past it saturates instead of inf)
+ const float m = 65504.0f;
+ return make_float4(fminf(fmaxf(v.x, -m), m), fminf(fmaxf(v.y, -m), m), fminf(fmaxf(v.z, -m), m), fminf(fmaxf(v.w, -m), m));
+}
+__global__ void __launch_bounds__(kThreads) ar_rs16_kernel(float* x, int n4, float* rows, float* sums, size_t stride,
+ unsigned* bflags, int nm, int mi, int slot, int slots,
+ unsigned seq) {
+ float4* x4 = (float4*) x;
+ const int per = (n4 + (int) gridDim.x - 1) / (int) gridDim.x;
+ const int lo = blockIdx.x * per, hi = min(n4, lo + per), cs = (hi - lo + nm - 1) / nm;
+ auto rflag = [&](int m) { return bflags + (((size_t) m * slots + slot) * kArBlocks + blockIdx.x) * 8; };
+ auto sflag = [&](int m) {
+ return bflags + ((((size_t) nm + m) * slots + slot) * kArBlocks + blockIdx.x) * 8;
+ };
+ auto row = [&](int m) { return (uint2*) (rows + ((size_t) m * slots + slot) * stride); }; // (FP16: half a row)
+ uint2* sum = (uint2*) (sums + (size_t) slot * stride);
+ if (lo < hi) {
+ uint2* mine = row(mi);
+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x) mine[q] = to_h4(sat4(x4[q]));
+ }
+ __threadfence_system();
+ __syncthreads();
+ if (threadIdx.x == 0) *(volatile unsigned*) rflag(mi) = seq;
+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(rflag(threadIdx.x), seq);
+ __syncthreads();
+ const int c0 = min(hi, lo + mi * cs), c1 = min(hi, c0 + cs);
+ for (int q = c0 + threadIdx.x; q < c1; q += blockDim.x) {
+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ for (int m = 0; m < nm; ++m) {
+ const uint2 u = m == mi ? to_h4(sat4(x4[q])) : ld_fresh(row(m) + q);
+ const float4 p = from_h4(u);
+ if (m == 0) v = p;
+ else {
+ v.x = __fadd_rn(v.x, p.x);
+ v.y = __fadd_rn(v.y, p.y);
+ v.z = __fadd_rn(v.z, p.z);
+ v.w = __fadd_rn(v.w, p.w);
+ }
+ }
+ const uint2 s16 = to_h4(sat4(v));
+ sum[q] = s16;
+ x4[q] = from_h4(s16);
+ }
+ __threadfence_system();
+ __syncthreads();
+ if (threadIdx.x == 0) *(volatile unsigned*) sflag(mi) = seq;
+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(sflag(threadIdx.x), seq);
+ __syncthreads();
+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x)
+ if (q < c0 || q >= c1) x4[q] = from_h4(ld_fresh(sum + q));
+}
+
+// x[i] = the members' partials summed in member order (this member's from x, the others' from their host rows)
+__global__ void __launch_bounds__(kThreads) ar_sum_kernel(float* x, int n4, const float* peers, size_t stride,
+ const unsigned* flags, int nm, int mi, int slot, int slots,
+ unsigned seq) {
+ if (threadIdx.x == 0)
+ for (int m = 0; m < nm; ++m)
+ if (m != mi) wait_at_least(flags + ((size_t) m * slots + slot) * 32, seq);
+ __syncthreads();
+ for (int q = blockIdx.x * blockDim.x + threadIdx.x; q < n4; q += gridDim.x * blockDim.x) {
+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ for (int m = 0; m < nm; ++m) {
+ const float4 p = m == mi ? ((const float4*) x)[q]
+ : ld_fresh((const float4*) (peers + ((size_t) m * slots + slot) * stride) + q);
+ if (m == 0) v = p;
+ else {
+ v.x = __fadd_rn(v.x, p.x);
+ v.y = __fadd_rn(v.y, p.y);
+ v.z = __fadd_rn(v.z, p.z);
+ v.w = __fadd_rn(v.w, p.w);
+ }
+ }
+ ((float4*) x)[q] = v;
+ }
+}
+
+
+// a thread a (token, value): every stream's value read before any is written
+__global__ void hc_post_inplace_kernel(const float* __restrict__ y, float* h, const float* __restrict__ post,
+ const float* __restrict__ comb, int dim, int hc) {
+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (e >= dim) return;
+ float* r = h + (size_t) t * hc * dim;
+ float rv[8];
+ for (int j = 0; j < hc; ++j) rv[j] = r[(size_t) j * dim + e];
+ const float yv = y[(size_t) t * dim + e];
+ for (int k = 0; k < hc; ++k) {
+ float a = post[(size_t) t * 8 + k] * yv;
+ for (int j = 0; j < hc; ++j) a += comb[(size_t) t * 64 + j * hc + k] * rv[j];
+ r[(size_t) k * dim + e] = a;
+ }
+}
+
+__global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __restrict__ h, int hcdim, int n_mix,
+ float eps, float* mixes) {
+ const int t = blockIdx.x;
+ const float* x = h + (size_t) t * hcdim;
+ float ss = 0.0f;
+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {
+ const float4 hv = *(const float4*) (x + k);
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ ss = block_sum(ss);
+ const float inv = rsqrtf(ss / (float) hcdim + eps);
+ if (threadIdx.x < n_mix) mixes[(size_t) t * 32 + threadIdx.x] *= inv;
+}
+
+__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,
+ float* __restrict__ out, bool q6k) {
+ const int r = blockIdx.x, t = r / cols, c = r - t * cols;
+ const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;
+ const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);
+ if (q6k) { // (ggml Q6_K superblocks at the row's start)
+ for (int e = threadIdx.x; e < hd; e += blockDim.x)
+ out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);
+ return;
+ }
+ for (int e = threadIdx.x; e < hd; e += blockDim.x) {
+ const float hv = e4m3_value(rows[e]);
+ const uint8_t sb = scales[e / 32];
+ const float s = sb == 0xffu ? __int_as_float(0x7fc00000) : (sb == 0 ? 5.877471754111438e-39f
+ : __uint_as_float((uint32_t) sb << 23));
+ out[(size_t) r * hd + e] = hv * s;
+ }
+}
+
+// a block a virtual slot: its picks counted (sort pass 1), or (fill) placed in token order by a block-wide scan
+template <bool FILL>
+__global__ void __launch_bounds__(kThreads) moe_sort_kernel(const int* __restrict__ idx, int T, int k,
+ const int* __restrict__ vs, int* bounds, int* row_tok,
+ int* pos) {
+ const int slot = blockIdx.x;
+ __shared__ int warp_n[kThreads / 32];
+ __shared__ int base_s;
+ if (threadIdx.x == 0) base_s = FILL ? bounds[slot] : 0;
+ __syncthreads();
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ const int n = T * k;
+ for (int q0 = 0; q0 < n; q0 += kThreads) {
+ const int q = q0 + threadIdx.x;
+ const int t = q / k, i = q - t * k;
+ const bool hit = q < n && vs[idx[(size_t) t * kPicks + i]] == slot;
+ const unsigned m = __ballot_sync(0xffffffffu, hit);
+ if (lane == 0) warp_n[wp] = __popc(m);
+ __syncthreads();
+ int before = 0, total = 0;
+ for (int w = 0; w < kThreads / 32; ++w) {
+ if (w < wp) before += warp_n[w];
+ total += warp_n[w];
+ }
+ if (FILL && hit) {
+ const int r = base_s + before + __popc(m & ((1u << lane) - 1u));
+ row_tok[r] = t;
+ pos[(size_t) t * kPicks + i] = r;
+ }
+ __syncthreads();
+ if (threadIdx.x == 0) base_s += total;
+ __syncthreads();
+ }
+ if (!FILL && threadIdx.x == 0) bounds[slot + 1] = base_s; // (the counts; the scan makes them bounds)
+}
+
+__global__ void bounds_scan_kernel(int* bounds, int n_vs) { // one thread: bounds[0] = 0, bounds[s + 1] += bounds[s]
+ if (threadIdx.x != 0) return;
+ bounds[0] = 0;
+ for (int s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];
+}
+
+__global__ void moe_combine_kernel(float* acc, const float* __restrict__ y, const int* __restrict__ pos,
+ const float* __restrict__ w, int k, int r0, int r1, int dim) {
+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;
+ if (e >= dim) return;
+ float a = acc[(size_t) t * dim + e];
+ bool any = false;
+ for (int i = 0; i < k; ++i) {
+ const int r = pos[(size_t) t * kPicks + i];
+ if (r < r0 || r >= r1) continue;
+ a = __fmaf_rn(w[(size_t) t * kPicks + i], y[(size_t) (r - r0) * dim + e], a);
+ any = true;
+ }
+ if (any) acc[(size_t) t * dim + e] = a;
+}
+
+int cdiv(int a, int b) { return (a + b - 1) / b; }
+
+} // namespace
+
+void route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale,
+ const unsigned long long* vram, const float* x, int dim, int* idx, float* w,
+ unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl,
+ unsigned vl, int na0, int na1) {
+ route_publish_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,
+ vl != 0 ? bias_vl : nullptr, vl);
+ if (req != nullptr) publish_kernel<<<1, 1, 0, s>>>(req, T, seq, na0, na1);
+}
+void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s) {
+ if (req != nullptr && m > 0 && T > 0 && n <= 1024 && off + m <= kAhead)
+ route_ahead_kernel<<<T, 512, 0, s>>>(logits, bias, n, m, req, off);
+}
+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { wait_seq_kernel<<<1, 32, 0, s>>>(seq_ptr, seq); }
+void init_pre(float* pre, int T, cudaStream_t s) { init_pre_kernel<<<cdiv(T * 8, 128), 128, 0, s>>>(pre, T); }
+
+void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {
+ embed_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(rows, h, dim, hc);
+}
+
+void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s) {
+ if (len > kFinMax * kThreads) {
+ std::fprintf(stderr, "ds4 finish: a vector of %d values is more than %d\n", len, kFinMax * kThreads);
+ return;
+ }
+ if (n_vec <= 0) return;
+ if (f.pdst != nullptr) {
+ if (f.quant < kKvFp8 || f.quant > kKvFp4p || len % 32 != 0) {
+ std::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);
+ return;
+ }
+ finish_kernel<true><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);
+ } else
+ finish_kernel<false><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);
+}
+
+void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
+ hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);
+}
+void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
+ float* pre, float* post, float* comb, cudaStream_t s) {
+ hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);
+}
+void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
+ cudaStream_t s) {
+ hc_pre_norm_kernel<<<T, 1024, 0, s>>>(h, mix, w, dim, hc, eps, x);
+}
+void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,
+ float* out, cudaStream_t s) {
+ hc_post_kernel<<<dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s>>>(y, res, post, comb, dim, hc, out);
+}
+
+void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,
+ cudaStream_t s) {
+ hc_post_inplace_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(y, h, post, comb, dim, hc);
+}
+void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
+ hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);
+}
+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {
+ if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);
+}
+void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
+ cudaStream_t s) {
+ moe_sort_kernel<false><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);
+ bounds_scan_kernel<<<1, 32, 0, s>>>(bounds, n_vs);
+ moe_sort_kernel<true><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);
+}
+void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,
+ cudaStream_t s) {
+ if (r1 > r0 && T > 0) moe_combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(acc, y, pos, w, k, r0, r1, dim);
+}
+
+void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s) {
+ gemv_f16_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, alpha, y);
+}
+void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,
+ cudaStream_t s) {
+ if (T > 0)
+ gemv_f16_rows_kernel<<<dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s>>>(W, x, ldx, n, k, T, alpha,
+ y, ldy);
+}
+
+void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s) {
+ if (T > 0) gemv_f32_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, y, ldy);
+}
+
+void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
+ cudaStream_t s, const float* bias_vl, const VlBits* vl) {
+ static const VlBits none = {};
+ route_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,
+ vl != nullptr ? *vl : none);
+}
+
+void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s) {
+ if (n > 0) swiglu_kernel<<<cdiv(n, kThreads), kThreads, 0, s>>>(g, u, n, lim, h);
+}
+
+void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
+ int d, const float* w, float eps, float* latent, cudaStream_t s) {
+ compress_kernel<<<1, 512, 0, s>>>(kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);
+}
+
+void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,
+ int cand_stride, int blk, const Rows& lim, float* out, long long out_stride, cudaStream_t s,
+ int kfmt) {
+ if (hd != 128 || heads > 32 || (kfmt != kKvFp4p && kfmt != kKvBf16)) {
+ std::fprintf(stderr, "ds4 index_score: %d heads of %d (keys %d) not covered\n", heads, hd, kfmt);
+ return;
+ }
+ if (n <= 0) return;
+ const size_t smem = (size_t) (heads * hd + heads) * sizeof(float);
+ const dim3 grid(cdiv(n, kThreads), T);
+ if (kfmt == kKvFp4p)
+ index_score_kernel<128, kKvFp4p><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,
+ cand_stride, blk, lim, out, out_stride);
+ else
+ index_score_kernel<128, kKvBf16><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,
+ cand_stride, blk, lim, out, out_stride);
+}
+
+void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,
+ int* out, int out_stride, cudaStream_t s) {
+ topk_kernel<<<T, 1024, 0, s>>>(vals, stride, r, cand, cand_stride, blk, out, out_stride);
+}
+
+void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,
+ int* out, int out_stride, cudaStream_t s) {
+ int nb_max = 1;
+ Rows rb = r;
+ if (r.gen != 0) { // (a chunk: the widest query's blocks, made on the device per query)
+ rb.gen = 4;
+ rb.blk = blk;
+ nb_max = cdiv((int) ((r.p0 + T) / r.R), blk);
+ } else
+ for (int t = 0; t < T; ++t) {
+ rb.n[t] = cdiv(r.n[t], blk);
+ rb.k[t] = r.k[t];
+ nb_max = rb.n[t] > nb_max ? rb.n[t] : nb_max;
+ }
+ block_max_kernel<<<dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s>>>(score, stride, r, blk, bm, bm_stride);
+ topk(bm, bm_stride, T, rb, nullptr, 0, 1, out, out_stride, s);
+}
+
+// the multiprocessors of the current device (cached a device)
+static int sm_count() {
+ static std::atomic<int> cached[64];
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) dev = 0;
+ int v = cached[dev].load(std::memory_order_relaxed);
+ if (v <= 0) {
+ if (cudaDeviceGetAttribute(&v, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess || v <= 0) v = 1;
+ cached[dev].store(v, std::memory_order_relaxed);
+ }
+ return v;
+}
+
+void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride,
+ const AttnRows& a, const float* sink, int heads, int d, int T, float scale, float* out, cudaStream_t s,
+ int ckv_fmt, int heads_per_block, const AttnScratch* scr) {
+ if (d != 512 || (ckv_fmt != kKvFp4 && ckv_fmt != kKvBf16)) {
+ std::fprintf(stderr, "ds4 sparse_attn: head dim %d (rows %d) not covered\n", d, ckv_fmt);
+ return;
+ }
+ if (ckv_fmt == kKvBf16) {
+ sparse_attn_kernel<false><<<dim3(heads, T), kThreads, 0, s>>>(q, win, (const uint16_t*) ckv, sel, sel_stride, a,
+ sink, heads, d, scale, out);
+ return;
+ }
+ // a few queries (a decode step, a short window): their selected rows decoded once a query into the scratch, then
+ // the BF16 rows' kernel itself (heads_per_block -1: this way at any T)
+ const bool few = heads_per_block == -1 || (heads_per_block == 0 && T <= kAttnScratchT);
+ if (few && scr != nullptr && scr->rows != nullptr && T <= scr->max_t && sel_stride <= scr->cap) {
+ unpack_sel_kernel<<<dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s>>>((const uint8_t*) ckv, sel, sel_stride, a,
+ scr->cap, scr->rows);
+#if !defined(STRATA_USE_HIP)
+ if (T == 1) // (V100: the loads ahead pay for the decode's kernel at one query, not at more)
+ sparse_attn_kernel<true, 8><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,
+ heads, d, scale, out);
+ else
+#endif
+ sparse_attn_kernel<true, 0><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,
+ heads, d, scale, out);
+ return;
+ }
+ // packed rows: as many heads a block as still leaves two blocks a multiprocessor (a decoded row serves them all)
+ int G = heads_per_block;
+ if (G != 1 && G != 2 && G != 4 && G != 8) {
+ G = 8;
+ while (G > 1 && (heads % G != 0 || (heads / G) * T < 2 * sm_count())) G /= 2;
+ }
+ if (heads % G != 0) G = 1;
+ const dim3 grid(heads / G, T);
+ const uint8_t* c = (const uint8_t*) ckv;
+ if (G == 8) sparse_attn_packed_kernel<8><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);
+ else if (G == 4) sparse_attn_packed_kernel<4><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);
+ else if (G == 2) sparse_attn_packed_kernel<2><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);
+ else sparse_attn_packed_kernel<1><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);
+}
+
+void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s) {
+ ring_to_ext_kernel<<<n, kThreads, 0, s>>>(ring, ext, (long long) p0, n, d);
+}
+void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s, uint8_t* backup) {
+ if (d % 32 != 0) {
+ std::fprintf(stderr, "ds4 ext_to_ring: row of %d not covered\n", d);
+ return;
+ }
+ ext_to_ring_kernel<<<T, kThreads, 0, s>>>(ext, ring, (long long) p0, n, T, d, backup);
+}
+void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {
+ if (T > keep)
+ ring_restore_kernel<<<T - keep, kThreads, 0, s>>>(ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));
+}
+void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {
+ if (T > keep && b.n > 0)
+ ring_restore_batch_kernel<<<dim3(T - keep, b.n), kThreads, 0, s>>>(b, (long long) p0, n, keep,
+ kv_row_bytes(kKvFp8, d));
+}
+void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {
+ hc_mean_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(h, hc, dim, out, out_stride);
+}
+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kernel<<<T, 64, 0, s>>>(out, n, stride); }
+void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {
+ markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);
+}
+void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {
+ markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);
+}
+void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s) {
+ argmax_share_kernel<<<1, 1024, 0, s>>>(row, vn, v0, slot, seq);
+}
+void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,
+ cudaStream_t s) {
+ markov_pick_kernel<<<1, 256, 0, s>>>(slots, nm, seq, tok, E, erow, R);
+}
+void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,
+ cudaStream_t s) {
+ confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);
+}
+
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {
+ engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);
+}
+
+void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {
+ engram_gate_kernel<<<dim3(hc, T), 1024, 0, s>>>(h, kv, w, hc, dim, eps);
+}
+
+void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s) {
+ if (n > 0) gather_kernel<<<dim3(cdiv(cols, kThreads), n), kThreads, 0, s>>>(x, g, cols, out);
+}
+
+void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,
+ cudaStream_t s) {
+ combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(ed, cpu, sh, c, dim, out);
+}
+
+void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s) {
+ size_t total = 0;
+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
+ const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));
+ xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);
+}
+void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {
+ size_t total = 0;
+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
+ const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));
+ xfer_get_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq);
+}
+void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {
+ if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);
+}
+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { set_flag_kernel<<<1, 1, 0, s>>>(flag, v); }
+void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {
+ const int slot = (int) (seq % (unsigned) a.slots);
+ if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)
+ static const bool f16 = [] {
+ const char* e = std::getenv("DS4_AR_F16");
+ return e == nullptr || std::atoi(e) != 0; // (on unless DS4_AR_F16=0)
+ }();
+ const int n4 = n / 4;
+ const int blocks = std::max(1, std::min(kArBlocks, cdiv(n4, kThreads)));
+ if (f16)
+ ar_rs16_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);
+ else
+ ar_rs_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);
+ return;
+ }
+ Xfer xp;
+ xp.n = 1;
+ xp.src[0] = x;
+ xp.dst[0] = a.data + ((size_t) mi * a.slots + slot) * a.stride;
+ xp.bytes[0] = (size_t) n * 4;
+ xfer_put(xp, a.flags + ((size_t) mi * a.slots + slot) * 32, seq, cnt, s);
+ const int n4 = n / 4;
+ const int blocks = std::max(1, std::min(64, cdiv(n4, kThreads)));
+ ar_sum_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.stride, a.flags, a.nm, mi, slot, a.slots, seq);
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuh
new file mode 100644
index 000000000..f1903c679
--- /dev/null
+++ b/src/dsv41/ds4_kernels.cuh
@@ -0,0 +1,296 @@
+// src/ds4/ds4_kernels.cuh - DeepSeek-V4.1's own device kernels for a window of T tokens (1 <= T <= kWin) at positions
+// p0 .. p0 + T - 1 (the EXL3 matrices go through strata::kernels::exl3). The math is DeepSeek's inference/model.py
+// (MIT); tools/ds4/ref.py is the oracle the engine is checked against. Every pointer is device memory, every activation
+// F32, rows of a window back to back; the KV caches keep the reference's FP8 / FP4 roundings packed as DeepSeek stores
+// them (codes and scales, kv_row_bytes), the window attention's working rows BF16 (those values are exact in BF16).
+// What differs from query to query (its window, how much of the compressed KV it sees, how many rows it selects) is
+// known on the host and passed by value.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+namespace strata::ds4 {
+
+constexpr int kWin = 16; // tokens a window takes at most (the EXL3 GEMV's activations per job)
+constexpr int kPicks = 8; // routed experts a token can take at most
+
+/// The caches' rows: DeepSeek's own roundings kept as codes and scales - lossless, the values are on those grids already
+/// (the numbers are Finish::quant's; a finish with that quant and a pdst writes the format), a row of d values in
+/// kv_row_bytes(fmt, d) bytes, padded to 16 (the hand-offs copy 16-byte units):
+/// kKvFp8 the window ring: d E4M3 codes, then d / 32 power-of-two scales (UE8M0: the F32 exponent byte)
+/// kKvFp4 the compressed KV: d / 2 bytes of E2M1 codes (value 2i in the low nibble), then d / 16 E4M3 scales
+/// kKvFp4p the index keys: d / 2 bytes of E2M1 codes, then d / 32 UE8M0 scales
+/// kKvBf16: plain BF16 rows (a DSpark block's own rows, and the reference the tests check the packed rows against).
+constexpr int kKvBf16 = 0, kKvFp8 = 1, kKvFp4 = 2, kKvFp4p = 3;
+__host__ __device__ constexpr size_t kv_row_bytes(int fmt, int d) {
+ return fmt == kKvBf16 ? (size_t) d * 2
+ : fmt == kKvFp8 ? ((size_t) d + (size_t) d / 32 + 15) / 16 * 16
+ : fmt == kKvFp4 ? ((size_t) d / 2 + (size_t) d / 16 + 15) / 16 * 16
+ : ((size_t) d / 2 + (size_t) d / 32 + 15) / 16 * 16;
+}
+
+/// h[t][s][e] = BF16 rows[t][e] for each of the hc streams.
+void embed(const uint16_t* rows_bf16, int T, float* h, int dim, int hc, cudaStream_t s);
+
+/// The per-vector finish: n_vec vectors of len values at x + v * stride, each in order
+/// norm (w != null): x = x * rsqrt(mean(x^2) + eps) * w
+/// rope (rd > 0): the last rd values as adjacent complex pairs rotated by pos_v * freqs[i] (inverse: the conjugate),
+/// pos_v = pos + (v / vpp) * pos_step
+/// quant 0 none, 1 FP8 E4M3 with a power-of-two scale per 32, 2 FP4 E2M1 with an E4M3 scale per 16, 3 FP4 with a
+/// power-of-two scale per 32 (the reference kernels' act_quant / fp4_act_quant, inplace)
+/// and written back to x, or as BF16 to dst + v * dst_stride when dst is given, or packed (quant > 0: the cache format of
+/// that number, kv_row_bytes) to pdst + v * pdst_stride bytes when pdst is given (x then left as it was).
+struct Finish {
+ const float* w = nullptr;
+ float eps = 1e-20f;
+ int rd = 0, pos = 0, inverse = 0, vpp = 1, pos_step = 1;
+ const float* freqs = nullptr; // rd / 2 values
+ int quant = 0;
+ uint16_t* dst = nullptr;
+ long long dst_stride = 0;
+ uint8_t* pdst = nullptr;
+ long long pdst_stride = 0;
+};
+void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s);
+
+/// Hyper-Connections, row t of each [T][..] array: mixes[t][m] = (h_flat . fn[m]) * rsqrt(mean(h_flat^2) + eps), then
+/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes
+/// 32, pre / post 8, comb 64 floats a row.
+void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
+ float* pre, float* post, float* comb, cudaStream_t s);
+/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).
+void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
+ cudaStream_t s);
+/// out[t][k][e] = post[t][k] y[t][e] + sum_j comb[t][j][k] res[t][j][e] (out must not alias res).
+void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,
+ float* out, cudaStream_t s);
+/// hc_post with out = res (h), the same arithmetic: a chunk keeps one copy of its streams.
+void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,
+ cudaStream_t s);
+/// A chunk's hc_mix: the dots of each row with the n_mix weights already in mixes (a GEMM's, rows 32 apart), times
+/// rsqrt(mean(h_row^2) + eps) - hc_mix's value up to the dots' summation order.
+void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+
+/// y[t][r] = alpha * sum_k W[r][k] x[t][k], W F16 bits [n][k] (the indexer's weights_proj).
+void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s);
+/// The same for T activations x[t] = x + t ldx into y + t ldy, each weight read once for all of them; each output is
+/// gemv_f16's bit for bit.
+void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,
+ cudaStream_t s);
+/// y[t ldy + r] = sum_k W[r][k] x[t][k] in F32 (W [n][k], k % 128 == 0): the router's logits, as DeepSeek's
+/// inference computes them (linear(x.float(), weight.float())) - a warp a row, the same summing order for any T, so
+/// the token, window and chunk paths get the same bits.
+void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s);
+
+/// The router, row t: scores = sqrt(softplus(logits[t])) (n experts, rows 1024 apart), the top `k` of scores + bias;
+/// weights from the unbiased scores, normalized to sum 1 (+1e-20) and times route_scale -> idx[t][i], w[t][i] (rows
+/// kPicks apart). Rows of image tokens (bit t of vl) take bias_vl instead (training's noaux_tc_for_vl).
+constexpr int kVlRows = 8192;
+struct VlBits {
+ uint32_t w[kVlRows / 32];
+};
+void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
+ cudaStream_t s, const float* bias_vl = nullptr, const VlBits* vl = nullptr);
+
+/// h = silu(min(g, lim)) * clamp(u, -lim, lim) (lim <= 0: no clamp); n values.
+void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s);
+
+/// The ratio-r compressor over a window: token t (position p0 + t) takes slot (p0 + t) % r of the state; each token that
+/// completes a group writes the group's pooled latent (softmax over the slots, per value, then rmsnorm(w)) to the next
+/// row of latent.
+void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
+ int d, const float* w, float eps, float* latent, cudaStream_t s);
+
+/// Per-query sizes, by value - or (gen > 0, a chunk of any length) made on the device from the query's position: query
+/// t at position p0 + t has nn = (p0 + t + 1) / R compressed positions and cb = min(ccap, ceil(nn / blk)) candidate
+/// blocks; gen 1: n = nn, k = min(kcap, nn) (the indexer's scores and their top-k)
+/// gen 2: n = nn, k = cb (index_score over the candidates)
+/// gen 3: n = cb * blk, k = min(kcap, nn) (the top-k of the candidates' scores)
+/// gen 4: n = ceil(nn / blk), k = kcap (the candidate blocks' top-k)
+struct Rows {
+ int n[kWin];
+ int k[kWin];
+ int gen = 0, R = 1, kcap = 0, ccap = 0, blk = 1;
+ long long p0 = 0;
+};
+
+/// The indexer's scores, row t: out[t][i] = sum_h relu(q[t][h] . k[p]) w[t][h] for i < n (the widest query's), p = i, or
+/// with candidates p = cand[t][i / blk] * blk + i % blk for i < lim.k[t] * blk; positions >= lim.n[t] (or past the
+/// query's candidates) -> -inf. q rows heads * hd apart, w rows 64, cand rows cand_stride, out rows out_stride; the keys
+/// k rows of format kfmt (the cache's kKvFp4p, or kKvBf16).
+void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,
+ int cand_stride, int blk, const Rows& lim, float* out, long long out_stride, cudaStream_t s,
+ int kfmt = kKvFp4p);
+/// The top r.k[t] of vals[t][0, r.n[t]) (row t at vals + t * stride): their positions (unordered; with candidates
+/// cand[t][i / blk] * blk + i % blk) into out[t][0, k). r.k[t] >= r.n[t] takes them all.
+void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,
+ int* out, int out_stride, cudaStream_t s);
+/// Level one of the candidate pre-filter, row t: block b's score = the max over its blk positions of the r.n[t] the
+/// query sees; the block of its newest position is pinned; the top r.k[t] blocks -> out[t] (bm: scratch rows bm_stride).
+void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,
+ int* out, int out_stride, cudaStream_t s);
+
+/// Sparse attention, query t: heads x d values of q[t] against win rows win0[t] .. win0[t] + nwin[t] - 1 of the window
+/// buffer (d BF16 each) and the nsel[t] compressed rows ckv[sel[t][i]] (format ckv_fmt: the cache's kKvFp4, or kKvBf16);
+/// K = V, a learned sink logit per head. The packed rows, the same bits either way: up to kAttnScratchT queries (with
+/// a scratch, heads_per_block 0; or -1, any T) decoded once a query into scr's BF16 rows for the BF16 rows' kernel;
+/// else a kernel that decodes each staged row once for heads_per_block heads of a query (1, 2, 4, 8; 0: as many as
+/// keep two blocks a multiprocessor).
+struct AttnScratch {
+ uint16_t* rows = nullptr; // [max_t * cap][512] BF16
+ int max_t = 0, cap = 0; // queries, rows a query (at least the sel_stride)
+};
+constexpr int kAttnScratchT = 7; // (the staged kernel wins from 8 queries: V100, Sun)
+struct AttnRows {
+ int win0[kWin], nwin[kWin], nsel[kWin];
+ // gen: query t is a chunk's row t0 + t at position p0 + t0 + t (the window buffer: the ring's win rows before p0,
+ // then the chunk's), selecting min(kcap, (its position + 1) / R) compressed rows when R > 0
+ int gen = 0, t0 = 0, win = 0, R = 0, kcap = 0;
+ long long p0 = 0;
+};
+void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride,
+ const AttnRows& a, const float* sink, int heads, int d, int T, float scale, float* out, cudaStream_t s,
+ int ckv_fmt = kKvFp4, int heads_per_block = 0, const AttnScratch* scr = nullptr);
+/// The window buffer: ext[r] = ring[(p0 - n + r) % n] for the n rows before p0 (zeros before position 0); afterwards
+/// ring[(p0 + t) % n] = ext[n + t] for t < T. The ring's rows kKvFp8 (each group of 32 at the smallest power-of-two
+/// scale its BF16 values fit under: exact for values on the FP8 grid, the window's), the window buffer's BF16 (p0 = n:
+/// the ring's rows as they lie).
+void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s);
+void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s,
+ uint8_t* backup = nullptr);
+/// A speculative window's rollback: ring[(p0 + t) % n] = backup[t] for keep <= t < T (the rows ext_to_ring replaced).
+void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);
+/// The same for up to kMaxRingBatch layers' rings at once (one launch).
+constexpr int kMaxRingBatch = 64;
+struct RingBatch {
+ uint8_t* ring[kMaxRingBatch] = {};
+ const uint8_t* bak[kMaxRingBatch] = {};
+ int n = 0;
+};
+void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);
+
+/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are
+/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),
+/// decoded as ggml's dequantize_row_q6_K.
+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);
+/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;
+/// T tokens -> out F32 [T * cols][hd].
+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,
+ bool q6k = false);
+/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]
+/// (kv[t] = [key hc x dim | value dim]).
+void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);
+
+/// out[r] = x[src[r]] for r < n (rows of cols floats).
+struct Gather {
+ int src[kWin * kPicks];
+};
+void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s);
+/// The MoE's sum, token t: out[t] = sum_j w[t][j] ed[row[t][j]] (its VRAM experts, in pick order) + cpu[t] (when its
+/// cpu flag is set) + sh[t].
+struct Combine {
+ int n[kWin];
+ int row[kWin][kPicks];
+ float w[kWin][kPicks];
+ int cpu[kWin];
+};
+void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,
+ cudaStream_t s);
+
+/// The mailboxes of a small window (decode, a draft's verification, the drafter's block): host-mapped memory, a ring of
+/// kRing. The device publishes a layer's routing and its FFN inputs for the window's T tokens (seq written last, after
+/// a system fence); the host answers with the CPU lane's weighted part of each token that took a CPU-lane expert.
+constexpr int kRing = 64;
+constexpr int kMaxDim = 8192;
+constexpr int kAhead1 = 16, kAhead = kAhead1 + kPicks; // a token's guesses for the next layer, then the one after
+struct Req {
+ unsigned seq;
+ int T;
+ int na[2]; // lookahead: ahead[t][0 .. na[0]) the next layer's likely picks, ahead[t][kAhead1 ..) +na[1] the one after's
+ int ids[kWin][kPicks];
+ float w[kWin][kPicks];
+ int ahead[kWin][kAhead];
+ float x[kWin][kMaxDim];
+};
+struct Resp {
+ unsigned seq;
+ int pad[3];
+ int cpu[kWin]; // 1: part[t] is the CPU lane's weighted sum for token t
+ float part[kWin][kMaxDim];
+};
+/// route() for the T tokens of a window, plus: plan_ptr[t][i] = vram[idx[t][i]] (0: not on the device), each token's x
+/// (dim floats) and picks into the request slot, then T and seq (req null: no request, the plan alone).
+void route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale,
+ const unsigned long long* vram, const float* x, int dim, int* idx, float* w,
+ unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl = nullptr,
+ unsigned vl = 0, int na0 = 0, int na1 = 0);
+/// Lookahead: the m picks a later layer will likely make (its gate's logits on this layer's FFN input, the routing's
+/// score + bias, the m best of n) into req->ahead[t][off ..) for the T tokens - before route_publish, which publishes them.
+void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s);
+/// Waits (one thread, on the device) until *seq_ptr == seq.
+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s);
+/// A chunk's routed experts as sorted rows: the picks (t, i < k) of T tokens (idx rows kPicks apart) by the virtual
+/// slot vs[expert] (n_vs of them: the VRAM slots, then the experts streamed in), each slot's rows in token order:
+/// bounds[n_vs + 1], row_tok[r], pos[t][i] (the pick's sorted row, rows kPicks apart).
+void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
+ cudaStream_t s);
+/// acc[t] += w[t][i] * y[pos[t][i] - r0] for each pick of t (in pick order) whose sorted row is in [r0, r1).
+void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,
+ cudaStream_t s);
+/// rows[t][0] = 1, the rest 0 (the first layer's pre-mix: stream 0).
+void init_pre(float* pre, int T, cudaStream_t s);
+
+/// out[t][e] = mean over the hc streams of h[t][s][e] (the DSpark drafter's taps), rows out_stride floats apart.
+void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s);
+/// out[t][i] = i for i < n, rows `stride` apart (the drafter's block rows as every query's selection).
+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s);
+/// The DSpark markov head: logits[v] += E[*tok] . H[v] for v < V (E BF16 [V][R], H F16 [V][R]).
+void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s);
+/// The markov bias over a head group: a member's rows [v0, v0 + vn) of it added to its share of a logits row (erow:
+/// the previous token's E row, BF16, on the device) - markov_add's sums; the share's argmax (the greedy sampler's rule)
+/// into a host-mapped slot {value, index, seq}; every member's slot of the step -> the global argmax into *tok and its
+/// E row (E host-mapped) into erow.
+void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s);
+void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s);
+void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,
+ cudaStream_t s);
+/// The DSpark confidence: out[t] = sigmoid(w . [x[t] | E[tok[t]]]) (x rows dim floats, w F32 [dim + R]).
+void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,
+ cudaStream_t s);
+
+/// The pipeline's hand-offs and tensor parallelism, through host-mapped memory (these cards copy peer to peer only
+/// through the driver's staging). xfer_put copies up to 4 segments (16-byte multiples, 16-byte aligned) device ->
+/// host-mapped, then (flag given) sets *flag = seq after a system fence (cnt: a device counter of the stream, 0
+/// between uses); xfer_get waits on the device until *flag >= seq, then copies host-mapped -> device.
+struct Xfer {
+ const void* src[4] = {nullptr, nullptr, nullptr, nullptr};
+ void* dst[4] = {nullptr, nullptr, nullptr, nullptr};
+ size_t bytes[4] = {0, 0, 0, 0};
+ int n = 0;
+};
+void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s);
+void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s);
+/// Waits (one thread) until flags[i * stride] >= seq for i < n (a pipelined chunk's readers have taken a buffer);
+/// set_flag: *flag = v after a system fence (a reader's acknowledgement).
+void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s);
+void set_flag(unsigned* flag, unsigned v, cudaStream_t s);
+/// A tensor-parallel group's all-reduce: host-mapped rows data[member][slot] (stride floats), flags[member][slot]
+/// (32 apart). x (n floats, n % 4 == 0: this member's partial) -> the sum of the nm members' partials in member order,
+/// the same bits on every member; seq counts the group's all-reduces (the same on each member), slot = seq % slots.
+struct ArGroup {
+ float* data = nullptr;
+ unsigned* flags = nullptr;
+ size_t stride = 0;
+ int nm = 1, slots = 4;
+ // three members or more: a reduce-scatter + all-gather (each member sums its chunk of every block's range, in
+ // member order, then reads the others' sums) - [slots][stride] sums, [2][nm][slots][kArBlocks][8] flags
+ float* sums = nullptr;
+ unsigned* bflags = nullptr;
+};
+constexpr int kArBlocks = 64;
+void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s);
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
new file mode 100644
index 000000000..7916a0384
--- /dev/null
+++ b/src/dsv41/ds4_main.cpp
@@ -0,0 +1,771 @@
+// src/ds4/ds4_main.cpp - maya-ds4: DeepSeek-V4.1-Flash on Maya.
+//
+// maya-ds4 --pack <model>/maya-ds4 --ids 0,128803,... --n 16 greedy tokens after the ids
+// maya-ds4 --pack ... --ids ... --dump-logits out.f32 every position's logits (the oracle's check)
+// maya-ds4 --pack ... --serve Maya's engine protocol on stdin / stdout
+//
+// --serve speaks what serve/server.py's StrataEngine expects: INFO lines, then `READY <context> stop`; per request
+// `GEN <max_new> [key=value ...] <id,id,...>` answered by `T <id>` lines and `DONE <generated> <prompt> <prompt ms>
+// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that
+// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to
+// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
+// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
+#include "ds4_model.hpp"
+#include "strata/platform/env_alias.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace {
+
+double now_ms() {
+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+
+// the bytes this process had the storage read for it (Linux's /proc/self/io; -1 where there is none): the Monitor's
+// disk reads a token, the experts read in place from the model's files included
+long long io_read_bytes() {
+#ifdef __linux__
+ std::FILE* f = std::fopen("/proc/self/io", "r");
+ if (f == nullptr) return -1;
+ char line[128];
+ long long v = -1;
+ while (std::fgets(line, sizeof line, f) != nullptr)
+ if (std::sscanf(line, "read_bytes: %lld", &v) == 1) break;
+ std::fclose(f);
+ return v;
+#else
+ return -1;
+#endif
+}
+
+std::vector<int> parse_ids(const std::string& s) {
+ std::vector<int> v;
+ const char* p = s.c_str();
+ while (*p) {
+ char* e = nullptr;
+ const long x = std::strtol(p, &e, 10);
+ if (e == p) break;
+ v.push_back((int) x);
+ p = e;
+ while (*p == ',' || *p == ' ') ++p;
+ }
+ return v;
+}
+
+void usage() {
+ std::fprintf(stderr,
+ "maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"
+ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--no-expert-cache] [--verbose]\n"
+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"
+ " [--vision on|off] [--mmap-experts on|off|auto]\n"
+ " (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
+ " | --encode-image FILE.rgb --size WxH --dump-image OUT)\n");
+}
+
+// stdin lines on a thread, so a request can see STOP while it decodes
+struct Lines {
+ std::mutex mu;
+ std::condition_variable cv;
+ std::deque<std::string> q;
+ bool eof = false;
+ std::atomic<bool> stop{false};
+ void start() {
+ std::thread([this] {
+ std::string l;
+ while (std::getline(std::cin, l)) {
+ if (!l.empty() && l.back() == '\r') l.pop_back();
+ std::lock_guard<std::mutex> lk(mu);
+ if (l == "STOP" || l.rfind("STOP ", 0) == 0) {
+ stop.store(true);
+ continue;
+ }
+ if (l == "QUIT") stop.store(true);
+ q.push_back(l);
+ cv.notify_all();
+ }
+ std::lock_guard<std::mutex> lk(mu);
+ eof = true;
+ stop.store(true);
+ cv.notify_all();
+ }).detach();
+ }
+ bool next(std::string& l) {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return !q.empty() || eof; });
+ if (q.empty()) return false;
+ l = q.front();
+ q.pop_front();
+ return true;
+ }
+};
+
+// The drafts between two stats: each draft position (1st, 2nd, ..) - how often it was verified with every draft
+// before it kept, and how often it was kept then
+std::string per_position(const strata::ds4::Stats& s0, const strata::ds4::Stats& s1) {
+ std::string pp = "per position (kept / reached):";
+ for (int i = 0; i < 8; ++i) {
+ const long long r = s1.draft_reached[i] - s0.draft_reached[i], k = s1.draft_kept[i] - s0.draft_kept[i];
+ if (r <= 0) break;
+ char b[64];
+ std::snprintf(b, sizeof b, " %d: %lld/%lld %.0f%%", i + 1, k, r, 100.0 * (double) k / (double) r);
+ pp += b;
+ }
+ return pp;
+}
+
+// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their
+// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow
+// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.
+bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,
+ float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,
+ int& accepted, std::string& err) {
+ out.clear();
+ offered = accepted = 0;
+ if (spec && room > 1 && m.draft_block() > 0) {
+ std::vector<int> drafts;
+ std::vector<float> conf;
+ if (!m.draft(tok, drafts, conf, err, temperature, top_p, top_k, min_p, seed, counter)) return false;
+ int keep = 0;
+ while (keep < (int) drafts.size() && keep < room - 1 && conf[(size_t) keep] >= conf_min) ++keep;
+ if (keep > 0) {
+ drafts.resize((size_t) keep);
+ int acc = 0, next = -1;
+ if (!m.verify(tok, drafts, temperature, top_p, top_k, min_p, seed, counter, acc, next, err)) return false;
+ counter += (unsigned long long) acc + 1;
+ for (int i = 0; i < acc; ++i) out.push_back(drafts[(size_t) i]);
+ out.push_back(next);
+ offered = keep;
+ accepted = acc;
+ return true;
+ }
+ }
+ if (!m.step(tok, true, err)) return false;
+ out.push_back(m.sample(temperature, top_p, top_k, min_p, seed, counter++));
+ return true;
+}
+
+struct Req {
+ long long max_new = 0;
+ float temperature = 0.0f, top_p = 1.0f, min_p = 0.0f;
+ int top_k = 64;
+ unsigned long long seed = 0;
+ int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)
+ float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)
+ std::string file; // GENI: the pictures
+ std::vector<int> ids;
+};
+
+// A prompt's picture: where its span starts, its pixels (the server's resize: RGB rows, both sides multiples of the
+// patch size), a hash of them; once encoded, its span's rows
+struct Pic {
+ int64_t start = 0;
+ int width = 0, height = 0;
+ uint64_t hash = 0;
+ size_t len = 0; // the span (0: not known yet)
+ std::vector<uint8_t> rgb;
+};
+
+// GENI's file: int32 'DSI1', int32 count, then per picture int64 start, int32 width, int32 height and its
+// height x width x 3 RGB bytes, in prompt order
+bool read_pics(const std::string& path, std::vector<Pic>& v, std::string& err) {
+ std::FILE* f = std::fopen(path.c_str(), "rb");
+ if (f == nullptr) {
+ err = "cannot open the pictures' file " + path;
+ return false;
+ }
+ char magic[4] = {};
+ int32_t n = 0;
+ bool ok = std::fread(magic, 1, 4, f) == 4 && std::memcmp(magic, "DSI1", 4) == 0 && std::fread(&n, 4, 1, f) == 1 &&
+ n >= 0 && n <= 4096;
+ for (int i = 0; ok && i < n; ++i) {
+ Pic p;
+ int32_t wh[2] = {};
+ ok = std::fread(&p.start, 8, 1, f) == 1 && std::fread(wh, 4, 2, f) == 2 && wh[0] > 0 && wh[1] > 0 &&
+ (int64_t) wh[0] * wh[1] <= (64 << 20);
+ if (!ok) break;
+ p.width = wh[0], p.height = wh[1];
+ p.rgb.resize((size_t) p.width * p.height * 3);
+ ok = std::fread(p.rgb.data(), 1, p.rgb.size(), f) == p.rgb.size();
+ uint64_t h = 1469598103934665603ull; // FNV-1a over the size and the pixels
+ for (int32_t x : wh)
+ for (int b = 0; b < 4; ++b) h = (h ^ (uint8_t) (x >> (8 * b))) * 1099511628211ull;
+ for (uint8_t b : p.rgb) h = (h ^ b) * 1099511628211ull;
+ p.hash = h;
+ v.push_back(std::move(p));
+ }
+ std::fclose(f);
+ if (!ok) err = path + ": not a pictures' file (DSI1)";
+ return ok;
+}
+
+// the encoded spans, the latest few (a conversation sends its pictures with every turn)
+struct SpanCache {
+ std::deque<std::pair<uint64_t, std::shared_ptr<const std::vector<uint16_t>>>> q;
+ std::shared_ptr<const std::vector<uint16_t>> get(strata::ds4::Model& m, const Pic& p, std::string& err) {
+ for (size_t i = 0; i < q.size(); ++i)
+ if (q[i].first == p.hash) {
+ auto r = q[i].second;
+ q.erase(q.begin() + (long) i);
+ q.emplace_front(p.hash, r);
+ return r;
+ }
+ auto rows = std::make_shared<std::vector<uint16_t>>();
+ if (!m.encode_image(p.rgb.data(), p.width, p.height, *rows, err)) return nullptr;
+ q.emplace_front(p.hash, rows);
+ if (q.size() > 8) q.pop_back();
+ return rows;
+ }
+};
+
+bool parse_gen(const std::string& line, Req& r, bool geni) {
+ const char* p = line.c_str() + (geni ? 5 : 4);
+ char* e = nullptr;
+ r.max_new = std::strtoll(p, &e, 10);
+ if (e == p) return false;
+ p = e;
+ for (;;) {
+ while (*p == ' ') ++p;
+ const char* s = p;
+ while (*p && *p != ' ') ++p;
+ if (p == s) break;
+ const std::string tok(s, (size_t) (p - s));
+ const size_t eq = tok.find('=');
+ if (eq == std::string::npos && geni && r.file.empty()) {
+ r.file = tok;
+ continue;
+ }
+ if (eq == std::string::npos) {
+ r.ids = parse_ids(tok);
+ break;
+ }
+ const std::string k = tok.substr(0, eq), v = tok.substr(eq + 1);
+ if (k == "temperature") r.temperature = std::strtof(v.c_str(), nullptr);
+ else if (k == "top_p") r.top_p = std::strtof(v.c_str(), nullptr);
+ else if (k == "top_k") r.top_k = std::atoi(v.c_str());
+ else if (k == "min_p") r.min_p = std::strtof(v.c_str(), nullptr);
+ else if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);
+ else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;
+ else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);
+ }
+ return !r.ids.empty() && (!geni || !r.file.empty());
+}
+
+int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min) {
+ std::vector<int> eos = parse_ids(eos_list);
+ for (const std::string& l : m.info_lines()) std::printf("%s\n", l.c_str());
+ std::printf("READY %lld stop\n", (long long) m.max_context());
+ std::fflush(stdout);
+ std::fprintf(stderr, "maya ds4 serve: GEN <max_new> [keys] <ids>; GENI <max_new> [keys] <pictures> <ids>; STOP, QUIT\n");
+ Lines in;
+ in.start();
+ std::string line;
+ unsigned long long counter = 0;
+ std::vector<Pic> held; // the pictures the state holds (start, hash, span)
+ SpanCache spans;
+ const int img_tok = m.image_token();
+ while (in.next(line)) {
+ if (line == "QUIT") break;
+ const bool geni = line.rfind("GENI ", 0) == 0;
+ if (!geni && line.rfind("GEN ", 0) != 0) {
+ std::printf("ERR unsupported command '%s' (GEN, GENI, STOP and QUIT)\n", line.substr(0, line.find(' ')).c_str());
+ std::fflush(stdout);
+ continue;
+ }
+ in.stop.store(false);
+ Req r;
+ if (!parse_gen(line, r, geni)) {
+ std::printf("ERR expected: GEN <max_new> [keys] <id,id,...> or GENI <max_new> [keys] <file> <id,id,...>\n");
+ std::fflush(stdout);
+ continue;
+ }
+ const double t0 = now_ms();
+ std::vector<Pic> pics;
+ {
+ std::string e;
+ if (geni && img_tok < 0) e = "this engine has no vision tower, so it cannot read pictures";
+ else if (geni && read_pics(r.file, pics, e))
+ for (const Pic& q : pics)
+ if (q.start < 0 || (size_t) q.start >= r.ids.size() || r.ids[(size_t) q.start] != img_tok)
+ e = "a picture's span does not start at an image token of the prompt";
+ if (!e.empty()) {
+ std::printf("ERR %s\n", e.c_str());
+ std::printf("DONE 0 %zu 0.0 0.0 cancel 0 0 0 0 0\n", r.ids.size());
+ std::fflush(stdout);
+ continue;
+ }
+ }
+ const auto& st = m.tokens();
+ const size_t n_state = st.size();
+ size_t c = 0;
+ while (c < n_state && c < r.ids.size() && st[c] == r.ids[c]) ++c;
+ // the same tokens with another picture (or none) do not share the state from that picture's span on
+ auto same = [](const std::vector<Pic>& v, const Pic& q) {
+ for (const Pic& x : v)
+ if (x.start == q.start && x.hash == q.hash) return true;
+ return false;
+ };
+ for (const Pic& q : pics)
+ if ((size_t) q.start < c && !same(held, q)) c = (size_t) q.start;
+ for (const Pic& q : held)
+ if ((size_t) q.start < c && !same(pics, q)) c = (size_t) q.start;
+ size_t from = 0;
+ if (c == n_state && n_state < r.ids.size()) from = n_state;
+ else if (m.restorable((int64_t) std::min(c, r.ids.size() - 1)) >= 0 &&
+ m.restore((int64_t) std::min(c, r.ids.size() - 1)))
+ from = (size_t) m.pos();
+ else m.reset();
+ if (from == 0) held.clear();
+ if ((int64_t) r.ids.size() + std::max(0LL, r.max_new) > m.max_context()) {
+ std::printf("ERR the prompt (%zu tokens) and max_new %lld are more than the context %lld\n", r.ids.size(),
+ r.max_new, (long long) m.max_context());
+ std::fflush(stdout);
+ continue;
+ }
+ std::string err;
+ bool ok = true;
+ // the pictures whose spans the prompt reads (the rest the state holds): encoded (or the cache's), checked
+ if (!pics.empty()) {
+ std::vector<strata::ds4::Image> imgs;
+ const double te = now_ms();
+ int encoded = 0;
+ for (Pic& q : pics) {
+ for (const Pic& x : held)
+ if (x.start == q.start && x.hash == q.hash) q.len = x.len;
+ if (q.len > 0 && (size_t) q.start + q.len <= from) continue;
+ std::shared_ptr<const std::vector<uint16_t>> rows = spans.get(m, q, err);
+ if (!rows) {
+ ok = false;
+ break;
+ }
+ ++encoded;
+ q.len = rows->size() / (size_t) m.dim();
+ bool fits = (size_t) q.start + q.len <= r.ids.size() - 1;
+ for (size_t i = 0; fits && i < q.len; ++i) fits = r.ids[(size_t) q.start + i] == img_tok;
+ if (!fits) {
+ err = "a picture's span (" + std::to_string(q.len) + " tokens at " + std::to_string(q.start) +
+ ") is not a run of image tokens in the prompt";
+ ok = false;
+ break;
+ }
+ imgs.push_back({q.start, rows});
+ }
+ if (ok) {
+ m.set_images(std::move(imgs));
+ std::fprintf(stderr, "maya ds4 serve: %zu pictures, %d read (%.0f ms)\n", pics.size(), encoded,
+ now_ms() - te);
+ }
+ for (Pic& q : pics) q.rgb = {}; // (only the start, hash and span stay)
+ }
+ // the prompt in windows (PP <read> <total> <ms> <tok/s> <reuse> every 64 tokens), a checkpoint before its last
+ // token and one after it: the next turn re-renders this one, and shares its start up to one of them
+ const size_t total = r.ids.size();
+ size_t done = from, next_pp = from + 64;
+ m.set_progress([&](int64_t d) { // (a run of chunks: each one through every layer)
+ const double ms = now_ms() - t0;
+ std::printf("PP %zu %zu %.1f %.1f %zu\n", done + (size_t) d, r.ids.size(), ms,
+ ms > 0 ? 1000.0 * (double) (done + (size_t) d - from) / ms : 0.0, from);
+ std::fflush(stdout);
+ });
+ // (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)
+ static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;
+ while (ok && done < total) {
+ const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);
+ // (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)
+ const bool fin = done + n == total;
+ if (fin && last_window) {
+ ok = (n < 2 || m.feed(r.ids.data() + done, (int) n - 1, 0, err));
+ if (ok) m.checkpoint(1);
+ ok = ok && m.feed(r.ids.data() + done + n - 1, 1, 1, err);
+ } else
+ ok = fin ? m.feed_mark(r.ids.data() + done, (int) n, 1, 1, err)
+ : m.feed(r.ids.data() + done, (int) n, 0, err);
+ done += n;
+ if (ok && (done >= next_pp || done == total)) {
+ const double ms = now_ms() - t0;
+ std::printf("PP %zu %zu %.1f %.1f %zu\n", done, r.ids.size(), ms,
+ ms > 0 ? 1000.0 * (double) (done - from) / ms : 0.0, from);
+ std::fflush(stdout);
+ next_pp = done + 64;
+ }
+ }
+ m.set_images({});
+ if (!ok) {
+ std::fprintf(stderr, "ds4: %s\n", err.c_str());
+ std::printf("ERR %s\n", err.c_str());
+ std::printf("DONE 0 %zu 0.0 0.0 cancel 0 0 0 0 0\n", r.ids.size());
+ std::fflush(stdout);
+ m.reset();
+ held.clear();
+ continue;
+ }
+ held = std::move(pics);
+ m.checkpoint(0);
+ const double t1 = now_ms();
+ long long produced = 0;
+ const char* finish = "length";
+ const long long limit = r.max_new > 0 ? r.max_new : m.max_context() - (long long) r.ids.size();
+ long long d_off = 0, d_acc = 0;
+ bool cool = false;
+ int tok = m.sample(r.temperature, r.top_p, r.top_k, r.min_p, r.seed, counter++);
+ std::vector<int> got;
+ // STAT for the Monitor every 8 tokens and at the end, over this answer so far: the GLM engine's keys (speed, the
+ // VRAM hit, promotions a token) and DeepSeek's own - experts a token on the CPU lane, MB a token the storage
+ // read (Linux), the n-gram memory's (Engram) read time a token, the drafts' acceptance
+ const strata::ds4::Stats s_dec = m.stats();
+ const long long io_dec = io_read_bytes();
+ auto stat_line = [&]() {
+ const strata::ds4::Stats& s = m.stats();
+ const double ms = now_ms() - t1;
+ const double fed = (double) std::max<int64_t>(1, s.tokens - s_dec.tokens);
+ const double ex = (double) (s.experts - s_dec.experts), cx = (double) (s.cpu_experts - s_dec.cpu_experts);
+ std::string out = "STAT";
+ char b[160];
+ std::snprintf(b, sizeof b, " tok_s=%.2f ms_tok=%.2f vram_hit=%.4f cpu=%.3f ram=%.3f promo=%.3f engram_ms=%.3f",
+ ms > 0 ? 1000.0 * (double) produced / ms : 0.0, produced > 0 ? ms / (double) produced : 0.0,
+ ex > 0 ? 1.0 - cx / ex : 1.0, cx / fed, (double) (s.ram_experts - s_dec.ram_experts) / fed,
+ (double) (s.swaps - s_dec.swaps) / fed, (s.engram_ms - s_dec.engram_ms) / fed);
+ out += b;
+ const long long io = io_read_bytes();
+ if (io >= 0 && io_dec >= 0) {
+ std::snprintf(b, sizeof b, " disk_mb=%.3f", (double) (io - io_dec) / 1e6 / fed);
+ out += b;
+ }
+ if (d_off > 0) {
+ std::snprintf(b, sizeof b, " drafts=%.3f", (double) d_acc / (double) d_off);
+ out += b;
+ }
+ if (s.skipped > s_dec.skipped) { // (STRATA_DS4_DISK_SKIP_W: experts left out a token)
+ std::snprintf(b, sizeof b, " skipped=%.3f", (double) (s.skipped - s_dec.skipped) / fed);
+ out += b;
+ }
+ std::printf("%s\n", out.c_str());
+ };
+ for (;;) {
+ if (in.stop.load()) {
+ finish = "cancel";
+ break;
+ }
+ std::printf("T %d\n", tok);
+ ++produced;
+ if (produced % 8 == 0) stat_line();
+ std::fflush(stdout);
+ bool end = false;
+ for (int e : eos) end = end || tok == e;
+ if (end) {
+ finish = "stop";
+ break;
+ }
+ if (produced >= limit || m.pos() >= m.max_context()) break;
+ if (got.empty()) { // the next round: the drafts (if any) and the token after them
+ const int room = (int) std::min<long long>(limit - produced, m.max_context() - m.pos());
+ int off = 0, acc = 0;
+ // a round whose drafts were all below the confidence floor: the next one does without the drafter
+ // (a request's spec / dspark_conf keys over the engine's: the speed calibration measures both ways)
+ const bool try_spec = (r.spec < 0 ? spec : r.spec != 0) && !cool;
+ if (!round(m, tok, room, try_spec, r.conf >= 0 ? r.conf : conf_min, r.temperature, r.top_p, r.top_k,
+ r.min_p, r.seed, counter, got,
+ off, acc, err)) {
+ std::fprintf(stderr, "ds4: %s\n", err.c_str());
+ finish = "cancel";
+ break;
+ }
+ d_off += off;
+ d_acc += acc;
+ cool = try_spec && off == 0;
+ std::reverse(got.begin(), got.end());
+ }
+ tok = got.back();
+ got.pop_back();
+ }
+ // (a stop within accepted drafts leaves the state past it: the next prompt shares less with it, and goes back to
+ // the checkpoint at this prompt's end)
+ stat_line();
+ const double t2 = now_ms();
+ std::printf("DONE %lld %zu %.1f %.1f %s %lld %lld %zu 0 0\n", produced, r.ids.size(), t1 - t0, t2 - t1, finish,
+ d_acc, d_off, from);
+ std::fflush(stdout);
+ const auto& s = m.stats();
+ std::fprintf(stderr,
+ "maya ds4 serve: prompt %zu tokens = %zu reused + %zu read in %.0f ms (%.1f tok/s), %lld generated "
+ "in %.0f ms (%.2f tok/s); CPU lane %.1f%% of the experts\n",
+ r.ids.size(), from, r.ids.size() - from, t1 - t0,
+ t1 > t0 ? 1000.0 * (double) (r.ids.size() - from) / (t1 - t0) : 0.0, produced, t2 - t1,
+ t2 > t1 ? 1000.0 * (double) produced / (t2 - t1) : 0.0,
+ s.experts ? 100.0 * (double) s.cpu_experts / (double) s.experts : 0.0);
+ if (d_off > 0) std::fprintf(stderr, "maya ds4 serve: drafts %lld of %lld kept; %s\n", d_acc, d_off,
+ per_position(s_dec, s).c_str());
+ m.save_profile();
+ }
+ m.save_profile();
+ return 0;
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ strata::platform::env_aliases(); // (MAYA_* settings under their older names too)
+ strata::ds4::Options o;
+ std::string ids_s, dump, eos = "1";
+ int n = 0, window = 16;
+ bool spec = true;
+ float temp = 0.0f; // the CLI's sampling (0: greedy)
+ unsigned long long cli_seed = 1;
+ bool serve_mode = false;
+ std::string images, enc_rgb, enc_out;
+ int enc_w = 0, enc_h = 0;
+ for (int i = 1; i < argc; ++i) {
+ const std::string a = argv[i];
+ auto val = [&](const char* what) -> std::string {
+ if (i + 1 >= argc) {
+ std::fprintf(stderr, "%s needs a value\n", what);
+ std::exit(2);
+ }
+ return argv[++i];
+ };
+ if (a == "--pack") o.pack = val("--pack");
+ else if (a == "--gpus") {
+ for (int g : parse_ids(val("--gpus"))) o.gpus.push_back(g);
+ } else if (a == "--max-context") o.max_context = std::atoll(val("--max-context").c_str());
+ else if (a == "--reserve-mib") o.reserve_mib = std::atoll(val("--reserve-mib").c_str());
+ else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());
+ else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());
+ else if (a == "--expert-profile") o.profile = val("--expert-profile");
+ else if (a == "--eos") eos = val("--eos");
+ else if (a == "--no-expert-cache") o.expert_cache = false;
+ else if (a == "--dspark") {
+ const std::string v = val("--dspark");
+ spec = v != "off";
+ o.dspark = v != "none"; // none: not even loaded
+ } else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());
+ else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));
+ else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));
+ else if (a == "--threads") o.threads = val("--threads") != "off";
+ else if (a == "--vision") o.vision = val("--vision") != "off";
+ else if (a == "--mmap-experts") {
+ const std::string v = val("--mmap-experts");
+ o.mmap_experts = v == "on" ? 1 : v == "off" ? 0 : -1;
+ } else if (a == "--images") images = val("--images");
+ else if (a == "--encode-image") enc_rgb = val("--encode-image");
+ else if (a == "--dump-image") enc_out = val("--dump-image");
+ else if (a == "--size") {
+ const std::string v = val("--size");
+ enc_w = std::atoi(v.c_str());
+ enc_h = v.find('x') != std::string::npos ? std::atoi(v.c_str() + v.find('x') + 1) : 0;
+ }
+ else if (a == "--temperature") temp = (float) std::atof(val("--temperature").c_str());
+ else if (a == "--seed") cli_seed = std::strtoull(val("--seed").c_str(), nullptr, 10);
+ else if (a == "--verbose" || a == "-v") o.verbose = true;
+ else if (a == "--ids") ids_s = val("--ids");
+ else if (a == "--n") n = std::atoi(val("--n").c_str());
+ else if (a == "--window") window = std::max(1, std::atoi(val("--window").c_str())); // (> 16: the chunked prompt path)
+ else if (a == "--dump-logits") dump = val("--dump-logits");
+ else if (a == "--serve") serve_mode = true;
+ else if (a == "--layer-split") (void) val("--layer-split"); // the server's (this engine balances its own split)
+ else if (a == "--help" || a == "-h") {
+ usage();
+ return 0;
+ } else {
+ std::fprintf(stderr, "unknown option %s\n", a.c_str());
+ usage();
+ return 2;
+ }
+ }
+ if (o.pack.empty() || (!serve_mode && ids_s.empty() && enc_rgb.empty())) {
+ usage();
+ return 2;
+ }
+ if (o.profile.empty()) o.profile = o.pack + "/expert-profile.bin"; // "none": no profile
+ if (o.profile == "none") o.profile.clear();
+ o.vision_only = !enc_rgb.empty();
+ strata::ds4::Model m;
+ std::string err;
+ if (!m.load(o, err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ if (!o.vision_only) std::fprintf(stderr, "%s", m.describe().c_str());
+ if (serve_mode) return serve(m, eos, spec, o.dspark_conf);
+
+ if (!enc_rgb.empty()) { // the vision tower alone: a picture's span rows (BF16 [rows][dim]) into --dump-image
+ std::vector<uint8_t> px((size_t) std::max(0, enc_w) * std::max(0, enc_h) * 3);
+ std::FILE* f = std::fopen(enc_rgb.c_str(), "rb");
+ const bool got = f != nullptr && !px.empty() && std::fread(px.data(), 1, px.size(), f) == px.size();
+ if (f != nullptr) std::fclose(f);
+ if (!got) {
+ std::fprintf(stderr, "maya-ds4: %s: expected %d x %d RGB bytes (--size WxH)\n", enc_rgb.c_str(), enc_w, enc_h);
+ return 1;
+ }
+ std::vector<uint16_t> span;
+ for (int k = 0; k < 2; ++k) { // (the second: warm)
+ const double te = now_ms();
+ if (!m.encode_image(px.data(), enc_w, enc_h, span, err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ std::fprintf(stderr, "vision: %d x %d pixels -> a span of %zu rows in %.0f ms\n", enc_w, enc_h,
+ span.size() / (size_t) m.dim(), now_ms() - te);
+ }
+ std::FILE* o2 = enc_out.empty() ? nullptr : std::fopen(enc_out.c_str(), "wb");
+ if (o2 != nullptr) {
+ std::fwrite(span.data(), 2, span.size(), o2);
+ std::fclose(o2);
+ }
+ return 0;
+ }
+ const std::vector<int> ids = parse_ids(ids_s);
+ if (!images.empty()) { // --images: a GENI file's pictures, for the prompt's spans
+ std::vector<Pic> pics;
+ if (!read_pics(images, pics, err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ std::vector<strata::ds4::Image> imgs;
+ for (const Pic& q : pics) {
+ auto rows = std::make_shared<std::vector<uint16_t>>();
+ const double te = now_ms();
+ if (!m.encode_image(q.rgb.data(), q.width, q.height, *rows, err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ std::fprintf(stderr, "picture at %lld: %d x %d pixels, %zu tokens, %.0f ms\n", (long long) q.start, q.width,
+ q.height, rows->size() / (size_t) m.dim(), now_ms() - te);
+ imgs.push_back({q.start, rows});
+ }
+ m.set_images(std::move(imgs));
+ }
+ std::FILE* df = dump.empty() ? nullptr : std::fopen(dump.c_str(), "wb");
+ std::vector<float> lg;
+ const double t0 = now_ms();
+ for (size_t a = 0; a < ids.size();) {
+ const int T = (int) std::min<size_t>((size_t) window, ids.size() - a);
+ const bool last = a + (size_t) T == ids.size();
+ if (!m.feed(ids.data() + a, T, df != nullptr ? T : (last ? 1 : 0), err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ if (df != nullptr && m.logits(lg)) std::fwrite(lg.data(), 4, lg.size(), df);
+ a += (size_t) T;
+ }
+ m.set_images({});
+ const double t1 = now_ms();
+ std::fprintf(stderr, "prompt: %zu tokens in %.0f ms (%.2f tok/s, windows of %d)\n", ids.size(), t1 - t0,
+ 1000.0 * (double) ids.size() / (t1 - t0), window);
+ const strata::ds4::Stats s0 = m.stats();
+ if (s0.eg_rows > 0)
+ std::fprintf(stderr, "prompt's Engram rows: %lld, %.1f%% without a read (repeats and the row cache), %lld read\n",
+ (long long) s0.eg_rows, 100.0 * (double) s0.eg_hits / (double) s0.eg_rows, (long long) s0.eg_reads);
+ const long long io0 = io_read_bytes();
+ std::vector<int> out;
+ unsigned long long counter = 0;
+ long long d_off = 0, d_acc = 0, rounds = 0;
+ bool cool = false;
+ if (n > 0) {
+ int tok = m.sample(temp, 1.0f, temp > 0 ? 64 : 1, 0.0f, cli_seed, counter++);
+ std::vector<int> got;
+ for (;;) {
+ out.push_back(tok);
+ std::printf("%d,", tok);
+ std::fflush(stdout);
+ if (tok == 1 || (int) out.size() >= n) break;
+ if (got.empty()) {
+ int off = 0, acc = 0;
+ const bool try_spec = spec && !cool;
+ if (!round(m, tok, n - (int) out.size(), try_spec, o.dspark_conf, temp, 1.0f, temp > 0 ? 64 : 1, 0.0f,
+ cli_seed, counter, got, off, acc, err)) {
+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());
+ return 1;
+ }
+ d_off += off;
+ d_acc += acc;
+ ++rounds;
+ cool = try_spec && off == 0;
+ std::reverse(got.begin(), got.end());
+ }
+ tok = got.back();
+ got.pop_back();
+ }
+ std::printf("\n");
+ if (df != nullptr && m.logits(lg)) std::fwrite(lg.data(), 4, lg.size(), df);
+ }
+ if (spec && rounds > 0) {
+ const auto& s1 = m.stats();
+ const double nv = (double) std::max<int64_t>(1, s1.drafts_rounds - s0.drafts_rounds);
+ std::fprintf(stderr, "dspark: %lld rounds, %lld drafts offered, %lld accepted (%.2f tokens a round); drafting "
+ "%.2f ms a round, verifying %.1f ms in all (%lld windows)\n",
+ rounds, d_off, d_acc, (double) out.size() / (double) rounds, (s1.draft_ms - s0.draft_ms) / nv,
+ s1.verify_ms - s0.verify_ms, (long long) (s1.drafts_offered > 0 ? 0 : 0));
+ std::fprintf(stderr, "dspark %s\n", per_position(s0, s1).c_str());
+ }
+ const double t2 = now_ms();
+ const auto& s = m.stats();
+ const double nt = std::max<double>(1.0, (double) (s.tokens - s0.tokens));
+ std::fprintf(stderr, "decode: %zu tokens in %.0f ms (%.2f tok/s); CPU lane: %lld of %lld experts; the cache moved %lld "
+ "(%lld in all)\n", out.size(), t2 - t1,
+ out.size() > 1 ? 1000.0 * (double) (out.size() - 1) / (t2 - t1) : 0.0,
+ (long long) (s.cpu_experts - s0.cpu_experts), (long long) (s.experts - s0.experts),
+ (long long) (s.swaps - s0.swaps), (long long) s.swaps);
+ std::fprintf(stderr, "decode per token: layers %.2f ms (of it: routing waits %.2f, CPU lane %.2f, the rest host launches), "
+ "head wait %.2f, Engram reads %.2f\n",
+ (s.layer_ms - s0.layer_ms) / nt, (s.gpu_wait_ms - s0.gpu_wait_ms) / nt, (s.cpu_ms - s0.cpu_ms) / nt,
+ (s.head_ms - s0.head_ms) / nt, (s.engram_ms - s0.engram_ms) / nt);
+ if (s.eg_rows > s0.eg_rows) { // the Engram rows: how many the row cache served, the reads' time behind the layers
+ const double rows = (double) (s.eg_rows - s0.eg_rows);
+ std::fprintf(stderr, "Engram per token: %.0f rows, %.1f%% without a read (the row cache holds %lld of %lld), "
+ "%.1f read, the reads %.2f ms from asking to the last one (left to wait: %.2f)\n",
+ rows / nt, 100.0 * (double) (s.eg_hits - s0.eg_hits) / rows, (long long) s.eg_cached,
+ (long long) s.eg_cache_cap, (double) (s.eg_reads - s0.eg_reads) / nt, (s.eg_io_ms - s0.eg_io_ms) / nt,
+ (s.engram_ms - s0.engram_ms) / nt);
+ }
+ const long long io1 = io_read_bytes();
+ if (io0 >= 0 && io1 >= 0)
+ std::fprintf(stderr, "decode reads from the disk: %.1f MB a token\n", (double) (io1 - io0) / 1e6 / nt);
+ if (s.skipped > s0.skipped) // STRATA_DS4_DISK_SKIP_W
+ std::fprintf(stderr, "decode left out %.1f experts a token not in RAM (%.1f%% of the routed)\n",
+ (double) (s.skipped - s0.skipped) / nt,
+ 100.0 * (double) (s.skipped - s0.skipped) / std::max<double>(1.0, (double) (s.experts - s0.experts)));
+ if (s.ahead_picks > s0.ahead_picks) { // lookahead
+ const double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);
+ std::fprintf(stderr, "lookahead: the guesses a layer before held %.1f%% of the picks (their top k %.1f%%, two layers "
+ "before %.1f%%); file-read picks asked for early %.1f%% (%lld of %lld); %.1f experts asked "
+ "for early a token\n",
+ 100.0 * (double) (s.ahead_hit - s0.ahead_hit) / p, 100.0 * (double) (s.ahead_hit_k - s0.ahead_hit_k) / p,
+ 100.0 * (double) (s.ahead_hit2 - s0.ahead_hit2) / p,
+ c > 0 ? 100.0 * (double) (s.ahead_cold_hit - s0.ahead_cold_hit) / c : 0.0,
+ (long long) (s.ahead_cold_hit - s0.ahead_cold_hit), (long long) (s.ahead_cold - s0.ahead_cold),
+ (double) (s.ahead_pf - s0.ahead_pf) / nt);
+ std::string rk = "lookahead: picked by the guess's rank (1 ..):";
+ for (int i = 0; i < 16; ++i) {
+ const long long n = (long long) (s.ahead_rank_n[i] - s0.ahead_rank_n[i]);
+ if (n <= 0) break;
+ char b[24];
+ std::snprintf(b, sizeof b, " %.0f%%", 100.0 * (double) (s.ahead_rank_hit[i] - s0.ahead_rank_hit[i]) / (double) n);
+ rk += b;
+ }
+ std::fprintf(stderr, "%s\n", rk.c_str());
+ }
+ if (s.prof_tokens > s0.prof_tokens) { // DS4_PROF
+ const double pt = (double) (s.prof_tokens - s0.prof_tokens);
+ double sum = 0;
+ for (int i = 0; i < 6; ++i) sum += (s.prof_ms[i] - s0.prof_ms[i]) / pt;
+ std::fprintf(stderr, "GPU per token (DS4_PROF): %.2f ms - between layers %.2f, attention %.2f, routing + shared "
+ "expert %.2f, experts gate/up %.2f, waiting for the CPU lane %.2f, down + post %.2f\n", sum,
+ (s.prof_ms[0] - s0.prof_ms[0]) / pt, (s.prof_ms[1] - s0.prof_ms[1]) / pt,
+ (s.prof_ms[2] - s0.prof_ms[2]) / pt, (s.prof_ms[3] - s0.prof_ms[3]) / pt,
+ (s.prof_ms[4] - s0.prof_ms[4]) / pt, (s.prof_ms[5] - s0.prof_ms[5]) / pt);
+ }
+ if (df != nullptr) std::fclose(df);
+ m.save_profile();
+ return 0;
+}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
new file mode 100644
index 000000000..eb86182de
--- /dev/null
+++ b/src/dsv41/ds4_model.cu
@@ -0,0 +1,7786 @@
+// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).
+//
+// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):
+// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables
+// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)
+// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus
+// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the
+// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b
+// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])
+// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert
+// h = post * moe + comb^T h; pre = f_pre
+// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
+#include "ds4_model.hpp"
+#include "ds4_engram_io.hpp"
+
+#include "ds4_gguf.cuh"
+#include "ds4_gguf_cpu.hpp"
+#include "ds4_kernels.cuh"
+#include "ds4_os.hpp"
+#include "ds4_pool.hpp"
+#include "ds4_vision.cuh"
+#include "strata/kernels/cpu/exl3_cpu.hpp"
+#include "strata/kernels/exl3.hpp"
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/kernels/glm_fast.hpp"
+#include "strata/kernels/sampler.hpp"
+
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <array>
+#include <condition_variable>
+#include <deque>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <map>
+#include <mutex>
+#include <numeric>
+#include <sstream>
+#include <thread>
+#include <unordered_map>
+
+#if defined(_MSC_VER)
+#define strtok_r strtok_s
+#endif
+
+namespace strata::ds4 {
+namespace x3 = strata::kernels::exl3;
+namespace kc = strata::kernels::cpu;
+namespace gf = strata::kernels::glmf;
+namespace gg = strata::ds4::gguf;
+
+#define CK(call) \
+ do { \
+ cudaError_t e_ = (call); \
+ if (e_ != cudaSuccess) { \
+ err = std::string(#call) + ": " + cudaGetErrorString(e_); \
+ return false; \
+ } \
+ } while (0)
+
+namespace {
+
+double now_ms() {
+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+
+size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }
+
+// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:
+// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):
+// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before
+void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {
+ const int b = whole / 128, q = b / np, r = b % np;
+ auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };
+ int at = 0;
+ for (int i = 0; i < p; ++i) at += blocks(i);
+ off = 128 * at;
+ n = 128 * blocks(p);
+}
+
+std::string read_file(const std::string& p) {
+ std::ifstream f(p);
+ std::stringstream ss;
+ ss << f.rdbuf();
+ return ss.str();
+}
+std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
+ std::vector<int> v;
+ std::stringstream ss(s);
+ std::string part;
+ while (std::getline(ss, part, ',')) {
+ if (part.empty() || part == "\n") continue;
+ const size_t dash = part.find('-');
+ const int a = std::atoi(part.c_str());
+ const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);
+ for (int c = a; c <= b; ++c) v.push_back(c);
+ }
+ return v;
+}
+// each NUMA node's physical cores (the first logical CPU of each core), in order
+std::vector<std::vector<int>> node_cores() {
+#if defined(_WIN32)
+ std::vector<std::vector<int>> nodes = os::node_cores();
+#else
+ std::vector<std::vector<int>> nodes;
+ for (int n = 0; n < 64; ++n) {
+ const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");
+ if (cl.empty()) break;
+ std::vector<int> cores;
+ for (int c : cpu_list(cl)) {
+ const std::vector<int> sib = cpu_list(
+ read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));
+ if (sib.empty() || sib[0] == c) cores.push_back(c);
+ }
+ if (!cores.empty()) nodes.push_back(cores);
+ }
+#endif
+ if (nodes.empty()) {
+ std::vector<int> all;
+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);
+ nodes.push_back(all);
+ }
+ return nodes;
+}
+// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages
+void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }
+void pin_self(int cpu) { os::pin_self(cpu); }
+// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root
+// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known
+uint64_t ram_avail() {
+#if defined(_WIN32)
+ return os::ram_avail();
+#endif
+ uint64_t avail = ~uint64_t{0};
+ {
+ std::istringstream mi(read_file("/proc/meminfo"));
+ std::string k;
+ uint64_t kb = 0;
+ while (mi >> k >> kb) {
+ if (k == "MemAvailable:") avail = kb << 10;
+ mi.ignore(64, '\n');
+ }
+ }
+ std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."
+ const size_t at = cg.find("0::");
+ if (at == std::string::npos) return avail;
+ cg = cg.substr(at + 3);
+ cg = cg.substr(0, cg.find('\n'));
+ for (;;) {
+ const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);
+ const std::string mx = read_file(dir + "/memory.max");
+ if (!mx.empty() && mx.compare(0, 3, "max") != 0) {
+ const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);
+ const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);
+ uint64_t file = 0;
+ std::istringstream st(read_file(dir + "/memory.stat"));
+ std::string k;
+ uint64_t v = 0;
+ while (st >> k >> v)
+ if (k == "file") file = v;
+ const uint64_t used = cur > file ? cur - file : 0;
+ avail = std::min(avail, lim > used ? lim - used : 0);
+ }
+ if (cg.empty() || cg == "/") break;
+ const size_t sl = cg.rfind('/');
+ cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);
+ }
+ return avail;
+}
+
+float bf16_to_f32(uint16_t b) {
+ uint32_t u = (uint32_t) b << 16;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+float f16_to_f32(uint16_t h) {
+ const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
+ uint32_t u;
+ if (e == 0) {
+ if (m == 0) u = s;
+ else { // subnormal
+ int ee = -1;
+ uint32_t mm = m;
+ do {
+ ++ee;
+ mm <<= 1;
+ } while ((mm & 0x400) == 0);
+ u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;
+ }
+ } else if (e == 31) u = s | 0x7f800000u | m << 13;
+ else u = s | (e - 15 + 127) << 23 | m << 13;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+
+// ------------------------------------------------------------------ the pack
+struct Tensor {
+ std::string dtype;
+ int file = -1;
+ uint64_t off = 0, bytes = 0;
+ std::vector<int64_t> shape;
+};
+
+struct Pack {
+ std::string dir, model_dir;
+ std::unordered_map<std::string, std::string> kv;
+ std::vector<std::string> files;
+ std::vector<os::fd_t> fds;
+ std::unordered_map<std::string, Tensor> t;
+ int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;
+ std::vector<int32_t> eg_map;
+ std::vector<int64_t> eg_mult, eg_primes, eg_off;
+ // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the
+ // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)
+ bool gguf = false;
+
+ std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)
+ std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)
+ std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)
+
+ ~Pack() {
+ for (auto& m : maps) os::unmap(m.first, m.second);
+ for (os::fd_t fd : fds) os::close_fd(fd);
+ for (os::fd_t fd : dfds) os::close_fd(fd);
+ for (os::fd_t fd : pfds) os::close_fd(fd);
+ }
+ // file i mapped whole, read-only (once; not thread-safe: called while loading)
+ const uint8_t* map(int file) {
+ if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});
+ auto& m = maps[(size_t) file];
+ if (m.first == nullptr) {
+ const uint64_t size = os::file_size(fds[(size_t) file]);
+ void* p = os::map_ro(fds[(size_t) file], size);
+ if (p == nullptr) return nullptr;
+ m = {(uint8_t*) p, (size_t) size};
+ }
+ return m.first;
+ }
+
+ bool load(const std::string& d, std::string& err) {
+ dir = d;
+ {
+ std::ifstream f(d + "/ds4.txt");
+ if (!f) {
+ err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";
+ return false;
+ }
+ std::string line;
+ while (std::getline(f, line)) {
+ const size_t sp = line.find(' ');
+ if (sp == std::string::npos) continue;
+ kv[line.substr(0, sp)] = line.substr(sp + 1);
+ }
+ }
+ model_dir = str("model_dir");
+ std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");
+ if (f == nullptr) {
+ err = d + "/tensors.txt: not found";
+ return false;
+ }
+ std::vector<char> buf(1 << 16);
+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
+ char* s = buf.data();
+ if (s[0] == 'f') { // file <i> <name>
+ char name[4096];
+ int i = 0;
+ if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
+ if ((int) files.size() <= i) files.resize((size_t) i + 1);
+ files[(size_t) i] = name;
+ }
+ continue;
+ }
+ if (s[0] != 't') continue;
+ char* save = nullptr;
+ ::strtok_r(s, " \n", &save);
+ const char* name = ::strtok_r(nullptr, " \n", &save);
+ const char* dt = ::strtok_r(nullptr, " \n", &save);
+ const char* fi = ::strtok_r(nullptr, " \n", &save);
+ const char* off = ::strtok_r(nullptr, " \n", &save);
+ const char* nb = ::strtok_r(nullptr, " \n", &save);
+ if (name == nullptr || nb == nullptr) continue;
+ Tensor x;
+ x.dtype = dt;
+ x.file = std::atoi(fi);
+ x.off = std::strtoull(off, nullptr, 10);
+ x.bytes = std::strtoull(nb, nullptr, 10);
+ while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));
+ t.emplace(name, std::move(x));
+ }
+ std::fclose(f);
+ for (const std::string& fn : files) {
+ const std::string p = model_dir + "/" + fn;
+ const os::fd_t fd = os::open_ro(p);
+ if (fd == os::kBadFd) {
+ err = p + ": cannot open";
+ return false;
+ }
+ fds.push_back(fd);
+ }
+ gguf = str("format") == "gguf";
+ if (gguf)
+ for (size_t i = 0; i < fds.size(); ++i) {
+ if (map((int) i) == nullptr) {
+ err = model_dir + "/" + files[i] + ": cannot map";
+ return false;
+ }
+ dfds.push_back(os::open_direct(model_dir + "/" + files[i]));
+ pfds.push_back(os::open_par(model_dir + "/" + files[i]));
+ }
+ // the Engram tables
+ std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");
+ if (e == nullptr) {
+ err = d + "/engram.bin: not found";
+ return false;
+ }
+ char magic[4];
+ int32_t hdr[6];
+ bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&
+ std::fread(hdr, 4, 6, e) == 6;
+ if (ok) {
+ eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];
+ const int cols = (eg_ngram - 1) * eg_heads;
+ eg_map.resize((size_t) eg_vocab);
+ eg_mult.resize((size_t) eg_layers * eg_ngram);
+ eg_primes.resize((size_t) eg_layers * cols);
+ eg_off.resize((size_t) eg_layers * cols);
+ ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&
+ std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&
+ std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&
+ std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();
+ }
+ std::fclose(e);
+ if (!ok) {
+ err = d + "/engram.bin: unreadable";
+ return false;
+ }
+ return true;
+ }
+
+ std::string str(const std::string& k, const std::string& def = "") const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : it->second;
+ }
+ double num(const std::string& k, double def) const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : std::atof(it->second.c_str());
+ }
+ std::vector<int> list(const std::string& k) const {
+ std::vector<int> v;
+ std::stringstream ss(str(k));
+ std::string x;
+ while (std::getline(ss, x, ','))
+ if (!x.empty()) v.push_back(std::atoi(x.c_str()));
+ return v;
+ }
+ const Tensor* get(const std::string& n) const {
+ auto it = t.find(n);
+ return it == t.end() ? nullptr : &it->second;
+ }
+ bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)
+ if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)
+ return os::pread_par(pfds[(size_t) file], dst, n, off);
+ return os::pread_full(fds[(size_t) file], dst, n, off);
+ }
+ bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }
+ const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)
+};
+
+// ------------------------------------------------------------------ geometry
+struct Geo {
+ int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,
+ k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,
+ cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;
+ float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;
+ double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;
+ std::vector<int> ratios, kv_src, ix_src, eg_layers;
+ // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks
+ int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;
+ std::vector<int> d_targets;
+
+ void from(const Pack& p) {
+ vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);
+ nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);
+ hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);
+ q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);
+ o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);
+ k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);
+ win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);
+ sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);
+ ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);
+ cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);
+ cand_blk = (int) p.num("candidate_block_size", 0);
+ orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);
+ eos = (int) p.num("eos_token_id", 1);
+ eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);
+ route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);
+ rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);
+ factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);
+ beta_slow = p.num("rope_scaling.beta_slow", 1);
+ ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");
+ ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");
+ d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);
+ d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);
+ d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");
+ // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)
+ n_mtp = d_block > 0 && (int) ratios.size() > nl &&
+ (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)
+ ? (int) ratios.size() - nl : 0;
+ }
+};
+
+// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them
+std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {
+ std::vector<float> f((size_t) dim / 2);
+ for (int i = 0; i < dim / 2; ++i) {
+ const float e = (float) (2 * i) / (float) dim;
+ f[(size_t) i] = 1.0f / std::pow((float) base, e);
+ }
+ if (original > 0) {
+ auto corr = [&](double rot) {
+ return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)
+ };
+ const double low = std::max(std::floor(corr(beta_fast)), 0.0);
+ const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);
+ const float den = (float) std::max(high - low, 1e-3);
+ for (int i = 0; i < dim / 2; ++i) {
+ float ramp = ((float) i - (float) low) / den;
+ ramp = std::min(1.0f, std::max(0.0f, ramp));
+ const float smooth = 1.0f - ramp;
+ f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;
+ }
+ }
+ return f;
+}
+
+// ------------------------------------------------------------------ devices, layers
+struct Dev {
+ int id = 0;
+ cudaStream_t st = nullptr;
+ cudaEvent_t ev = nullptr;
+ size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;
+ std::vector<void*> allocs;
+ // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)
+ float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,
+ *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,
+ *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,
+ *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,
+ *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,
+ *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,
+ *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,
+ *ed = nullptr;
+ float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;
+ unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)
+ uint8_t* eg_rows = nullptr;
+ uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows
+ AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)
+ // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the
+ // driver's staging, ~1.7 ms a copy): h | pre | topk | cand
+ uint8_t* bounce = nullptr;
+ cudaEvent_t ev_out = nullptr;
+ float* tap = nullptr; // a window's stream means before a DSpark target layer
+ // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's
+ // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window
+ // MoE's grouping scratch
+ int grp = 0, mi = 0;
+ unsigned ar_seq = 0;
+ unsigned* xcnt = nullptr;
+ // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
+ // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
+ x3::Mat head;
+ float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;
+ unsigned head_seq = 0;
+ // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the
+ // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)
+ uint16_t *mk_share = nullptr, *mk_erow = nullptr;
+ int* mk_ints = nullptr;
+ int mk_v0 = 0, mk_vn = 0;
+ uint8_t* hand = nullptr;
+ unsigned* hand_flag = nullptr;
+ unsigned long long* grp_ptr = nullptr;
+ int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+};
+
+// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of
+// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS
+struct Chunk {
+ int C = 0, A = 0, AI = 0;
+ float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,
+ *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,
+ *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,
+ *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,
+ *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;
+ uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,
+ *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;
+ size_t w16_elems = 0;
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;
+ uint8_t* eg_rows = nullptr;
+ int* bounds_pin = nullptr; // pinned: the sort's bounds, read back
+ // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,
+ // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)
+ static constexpr int kParts = 4;
+ uint8_t* ring = nullptr;
+ size_t ring_stride = 0;
+ int ring_n = 0;
+ cudaStream_t cs = nullptr;
+ cudaEvent_t filled[kParts] = {}, freed[kParts] = {};
+ // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of
+ // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,
+ // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s
+ uint8_t* stage = nullptr;
+ size_t stage_stride = 0;
+ std::unique_ptr<Pool> readers;
+ cublasHandle_t blas = nullptr;
+ // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each
+ // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)
+ uint8_t* hand[2] = {nullptr, nullptr};
+ unsigned* hand_flag[2] = {nullptr, nullptr};
+ unsigned* hand_ack = nullptr;
+ unsigned arb_seq = 0; // the group's chunk all-reduces
+ // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)
+ uint16_t* emb_host[2] = {nullptr, nullptr};
+ uint8_t* eg_host[2] = {nullptr, nullptr};
+ cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};
+ double stream_bytes = 0; // the experts streamed in, in all
+ // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a
+ // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the
+ // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once
+ // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io
+ // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits
+ // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot
+ // i % ring_n; the compute takes the items as they land, in runs of what has.
+ struct Item {
+ int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)
+ int mi = 0; // the device's member index in the layer's group
+ const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read
+ const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)
+ uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there
+ int64_t after = -1; // (layer-major) the items up to this one read first
+ std::atomic<int> left{0}; // parts still being read
+ std::atomic<bool> bad{false};
+ };
+ bool flow = false;
+ int dev = -1; // the device (index into devs) this chunk state belongs to
+ std::unique_ptr<IoQueue> io;
+ std::deque<Item> items; // items[0] is item `base`; the indices below are absolute
+ size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;
+ uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)
+ int S = 0; // staging slots, stage_stride apart
+ std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)
+ std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use
+ int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
+ size_t spec_begin = 0;
+ std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
+ int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
+ int *bc_pin = nullptr, *bc_dev = nullptr;
+ uint64_t n_batch = 0;
+ double wait_ms = 0; // the compute waiting for a read
+ uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;
+ uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started
+ // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through
+ // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in
+ // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for
+ // its later chunks, so each is read once a run
+ bool lm = false, lm_pinned = false;
+ uint8_t* lm_state = nullptr;
+ size_t lm_state_bytes = 0;
+ uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side
+ size_t lc_bytes = 0;
+ int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all
+ size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read
+ std::vector<int> lc_of; // expert -> its slot (-1: none)
+ std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it
+ uint64_t it_lc = 0; // items taken from the layer cache
+};
+
+struct Layer {
+ int il = 0, dev = 0, ratio = 0;
+ // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the
+ // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert
+ // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)
+ int grp = 0, nm = 1, mi = 0;
+ std::vector<Layer> mates;
+ x3::ExpertLayout hlay;
+ size_t hblob = 0;
+ int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)
+ // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more
+ // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their
+ // attention adds nothing to the sum
+ int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;
+ // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's
+ // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member
+ int* vs_pin = nullptr;
+ int* vs_dev = nullptr;
+ std::vector<int> host_list;
+ std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."
+ int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)
+ bool mtp = false; // a DSpark block (index stage = il - n_layers)
+ bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;
+ int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)
+ int ik_src = -1; // the kv source whose index keys its indexer reads
+ int eg = -1; // engram layer index, or -1
+ x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;
+ float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,
+ *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,
+ *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,
+ *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)
+ uint16_t* i_wproj = nullptr;
+ float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)
+ uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)
+ float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)
+ // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row
+ uint8_t* snap_win = nullptr;
+ float *snap_kv = nullptr, *snap_sc = nullptr;
+ // routed experts
+ x3::ExpertLayout lay;
+ size_t blob = 0; // bytes of one expert's blob
+ int n_vram = 0;
+ uint8_t* vram_base = nullptr;
+ std::vector<int> slot; // expert -> VRAM slot or -1
+ std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
+ std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)
+ // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),
+ // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)
+ struct Half {
+ size_t trellis[3], suh[3], svh[3], bytes;
+ int k[3], n[3];
+ } hl;
+ std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes
+ // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it
+ // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)
+ int n_slots = 0, spare_vram = -1;
+ std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};
+ std::vector<float> heat;
+ std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked
+ bool swap_pending = false;
+ unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes
+ // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and
+ // the window's raw rows (kv, then score) to replay the accepted ones
+ uint8_t* ring_bak = nullptr;
+ float *c_snap = nullptr, *c_rows = nullptr;
+ unsigned long long* vram_tab = nullptr; // device [n_exp]
+ std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
+ std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
+ std::vector<uint64_t> region_bytes;
+ std::vector<int> region_file;
+ size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region
+ size_t piece_bytes[3][3];
+ size_t dense_bytes = 0;
+ // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +
+ // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane
+ int mx_file[3] = {-1, -1, -1};
+ uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};
+};
+
+struct Src {
+ int layer = 0, ratio = 1;
+ int64_t cap = 0;
+ // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)
+ std::vector<uint8_t*> ckv, ik;
+ uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices
+ cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;
+ // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows
+ // (ckv rows, then ik rows at kWin * hd) and its flag
+ bool remote = false;
+ uint8_t* x_stage = nullptr;
+ unsigned* x_flag = nullptr;
+ // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements
+ // [slot][device] (32 apart); readers: the devices outside the source's group that read it
+ uint8_t* x_stage1 = nullptr;
+ unsigned* x_flag1 = nullptr;
+ unsigned* x_ack = nullptr;
+ std::vector<int> readers;
+ uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }
+ unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }
+};
+
+// a host thread that enqueues one device's work (threads mode)
+struct Worker {
+ std::thread th;
+ std::mutex mu;
+ std::condition_variable cv;
+ std::function<void()> job;
+ bool busy = false, quit = false;
+ void start() {
+ th = std::thread([this] {
+ for (;;) {
+ std::function<void()> j;
+ {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return quit || job != nullptr; });
+ if (quit) return;
+ j = std::move(job);
+ job = nullptr;
+ }
+ j();
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ busy = false;
+ }
+ cv.notify_all();
+ }
+ });
+ }
+ void submit(std::function<void()> f) {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ job = std::move(f);
+ busy = true;
+ }
+ cv.notify_all();
+ }
+ void wait() {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return !busy; });
+ }
+ bool idle() {
+ std::lock_guard<std::mutex> lk(mu);
+ return !busy;
+ }
+ void stop() {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ quit = true;
+ }
+ cv.notify_all();
+ if (th.joinable()) th.join();
+ }
+};
+
+struct Group {
+ std::vector<int> mem; // device indices, member order (member 0 the lead)
+ ArGroup ar, arb; // a window's all-reduces, a chunk's
+ // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the
+ // lead's acknowledgements (host-mapped)
+ float* lg_stage = nullptr;
+ unsigned *lg_flag = nullptr, *lg_ack = nullptr;
+};
+
+constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers
+
+// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the
+// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate
+// blocks as each layer sees them, the window's hand-off seq
+struct WinPlan {
+ bool par = false, async = false;
+ unsigned win = 0;
+ int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2
+ int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)
+ int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)
+ unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)
+ unsigned seq[kMaxL] = {};
+ int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};
+};
+
+} // namespace
+
+// ------------------------------------------------------------------ the model
+struct Model::Impl {
+ Options o;
+ Pack pk;
+ Geo g;
+ std::vector<Dev> devs;
+ std::vector<Layer> L;
+ std::vector<Src> srcs;
+ std::vector<Group> groups;
+ std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device
+ bool par = false; // threads mode (the windows: decode)
+ bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked
+ unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)
+ // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows
+ int chunk = 0, chunk_min = 512;
+ static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)
+ // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk
+ // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers
+ // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token
+ static constexpr int kMaxChunk = 4 * kVlRows;
+ std::vector<std::unique_ptr<Chunk>> chunks; // per device
+ size_t eg_cap = kWin; // tokens h_eg holds
+ int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
+ bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
+ std::vector<float> chunk_logits;
+ std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
+ std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
+ std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
+ // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
+ // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took
+ struct PathModel {
+ bool on = false;
+ double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)
+ // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and
+ // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)
+ double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};
+ int64_t kn[2][4] = {};
+ double win_s = 0; // windows: seconds a window took, so far
+ int64_t win_n = 0;
+ int last_cls = 0, last_nw = 0; // the last pick's size class, its windows
+ double m = 1; // the picks' spread, against the profile's
+ int64_t n_m = 0;
+ int last = -1; // the last pick: 1 the flow, 0 windows
+ double last_pred = 0; // its prediction (s, before its k)
+ } pm;
+ static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)
+ static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)
+ std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through
+ WinPlan wplan;
+ int head_dev = 0;
+ x3::Mat head;
+ float* norm_w = nullptr; // on the head device
+ // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the
+ // confidence head (on the head device: the drafts' logits come from the trunk's head)
+ x3::Mat d_main_proj;
+ float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
+ uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
+ // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
+ // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
+ // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
+ bool mk_split = false;
+ uint16_t* h_mk_embed = nullptr;
+ unsigned* mk_slots = nullptr;
+ unsigned mk_seq = 0;
+ int d_dev = 0; // the drafter's first block's device
+ bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)
+ float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps
+ cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it
+ float* d_confv = nullptr; // on the head device
+ int* d_ints = nullptr; // on the head device: drafts in, picks out
+ float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;
+ // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for
+ // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()
+ // checks it is its own)
+ strata::kernels::SpecList* d_specq = nullptr;
+ std::vector<strata::kernels::SpecList> h_specq = std::vector<strata::kernels::SpecList>(kWin);
+ bool drafts_q = false;
+ float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;
+ int dq_top_k = 0;
+ uint64_t dq_seed = 0, dq_counter = 0;
+ // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the
+ // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5
+ // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the
+ // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs
+ // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)
+ bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;
+ float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;
+ int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;
+ std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;
+ float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)
+ uint16_t* h_dump_ring = nullptr;
+
+ uint16_t* embed_pinned = nullptr;
+ std::vector<float> freqs_win, freqs_comp;
+ std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores
+ std::vector<std::vector<int>> cores; // each node's physical cores
+ int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)
+ // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps
+ // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)
+ bool file_experts = false;
+ uint64_t warm_bytes = 0;
+ static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)
+ uint8_t* file_part(const Layer& l, int e) const {
+ return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];
+ }
+ // host side of the CPU lane, a window's rows
+ float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;
+ int* h_idx = nullptr;
+ uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales
+ // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one
+ // an Engram layer: asked for when the window's tokens are known, collected right before that layer)
+ EgReader eg_io;
+ EgRowCache eg_cache;
+ std::mutex eg_cache_mu;
+ std::vector<std::unique_ptr<EgBatch>> eg_win;
+ std::vector<uint8_t> eg_q6k; // Engram layer li's table in ggml Q6_K (a row one superblock a 256 values), else FP8
+ std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
+ uint16_t* h_emb = nullptr; // the window's embedding rows
+ std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
+ // the state
+ int64_t pos = 0;
+ std::vector<int32_t> toks, ctoks;
+ struct Check { // the state at a position: every window ring, the compressors' partial groups
+ int64_t pos = -1;
+ std::vector<std::vector<uint8_t>> win;
+ std::vector<std::vector<float>> comp;
+ };
+ Check ck[2]; // the end of the last prompt, and the position before it
+ std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
+ Stats st;
+ int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
+ // (plan_window's: the windows' plans are made in order)
+ int n_logits = 0; // the last forward's logit rows
+ // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread
+ // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum
+ Req* reqs = nullptr;
+ Resp* resps = nullptr;
+ unsigned seq_issued = 0;
+ std::atomic<int> layer_of[kRing];
+ std::thread service;
+ std::atomic<bool> quit{false};
+ bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
+ // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
+ // before the wait for the CPU lane - the same bits, no faster)
+ bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
+ int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this
+ // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same
+ // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files
+ // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next
+ // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the
+ // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's
+ // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).
+ int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;
+ int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;
+ // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)
+ bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;
+ std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;
+ // 8 asked for (prefetch)
+ std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)
+ std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for
+ std::thread pf_th;
+ std::mutex pf_mu;
+ std::condition_variable pf_cv;
+ std::vector<std::pair<const void*, size_t>> pf_q;
+ // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster
+ // on one V100 - more for a PC to try: each its staging buffers and
+ // copy streams), switched between windows
+ struct Swap {
+ int layer = 0, in = -1, out = -1;
+ int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM
+ };
+ static constexpr int kMaxSwaps = 8;
+ // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay
+ // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)
+ static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }
+ const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),
+ c_decay = env_f("DS4_CACHE_DECAY", 0.9f);
+ const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));
+ // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the
+ // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the
+ // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the
+ // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in
+ // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4
+ // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.
+ struct RamSwap {
+ int layer = 0, in = -1, out = -1, slot = -1;
+ };
+ std::vector<uint8_t*> ram_slots, ram_chunks;
+ std::vector<size_t> ram_chunk_bytes;
+ bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)
+ size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers
+ std::vector<int> ram_free;
+ std::deque<RamSwap> ram_queue;
+ std::vector<RamSwap> ram_ready;
+ std::condition_variable ram_cv;
+ std::thread ram_th;
+ int ram_inflight = 0, ram_held = 0;
+ bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;
+ // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a
+ // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route
+ // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every
+ // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the
+ // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left
+ // out still counts as used (the expert cache, the profile).
+ const float disk_skip_w = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");
+ return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;
+ }();
+ const int disk_skip_from = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
+ return v != nullptr ? std::max(0, std::atoi(v)) : 0;
+ }();
+ bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r)
+ if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;
+ return true;
+ }
+ return !file_experts || os::in_ram(file_part(l, e), l.blob);
+ }
+ // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads
+ // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between
+ // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of
+ // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'
+ // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes
+ // the RAM place of the one it takes in)
+ // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private
+ // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever
+ // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the
+ // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds
+ // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the
+ // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold
+ // them (the log's WARNING, a NOTE and INFO for the server)
+ double commit_kept = 0, pagefile_hint = 0;
+ double commit_margin() const {
+ const os::Commit c = os::commit();
+ return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);
+ }
+ // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)
+ double commit_room() const {
+ const os::Commit c = os::commit();
+ double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;
+ if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);
+ return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());
+ }
+ // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)
+ bool commit_charged(const Dev& d) const {
+ if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;
+#if defined(_WIN32)
+ int v = 0;
+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return false;
+ }
+ return v != 0;
+#else
+ (void) d;
+ return false;
+#endif
+ }
+ bool copy_mode = false;
+ std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts
+ std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes
+ std::vector<int> copy_spare; // layer -> its spare slot (-1: none)
+ std::mutex sw_mu;
+ std::condition_variable sw_cv;
+ std::deque<Swap> sw_queue;
+ std::vector<Swap> sw_ready;
+ struct SwapBufs {
+ uint8_t *a = nullptr, *b = nullptr, *h = nullptr;
+ std::vector<cudaStream_t> st; // a copy stream a device
+ };
+ std::vector<SwapBufs> sw_bufs;
+ std::vector<std::thread> cache_ths;
+ int sw_inflight = 0;
+ int64_t decay_tokens = 0;
+ bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;
+ std::vector<float> host_logits;
+ std::vector<void*> host_allocs;
+ std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;
+ // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or
+ // chunk path (tests: where the two part)
+ std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;
+ // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),
+ // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the
+ // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias
+ // [n_exp] (F32)
+ std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;
+ // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,
+ // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]
+ // (F16) and y rows [rows dim] (F32)
+ int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;
+ std::FILE* dump_y = [] {
+ const char* v = std::getenv("DS4_DUMP_Y");
+ const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;
+ return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;
+ }();
+ std::mutex dump_mu;
+ // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its
+ // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU
+ // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it
+ bool prof_on = std::getenv("DS4_PROF") != nullptr;
+ struct ProfMark {
+ const Dev* d;
+ cudaEvent_t ev;
+ int stage;
+ };
+ std::mutex prof_mu;
+ std::vector<ProfMark> prof_marks;
+ std::vector<cudaEvent_t> prof_pool;
+ bool prof_window = false; // (decode windows only: T == 1)
+ // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
+ // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
+ bool prof_chunk = false;
+ double cprof_ms[4] = {};
+ void prof_mark(const Dev& d, int stage) {
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ cudaEvent_t e = nullptr;
+ if (prof_pool.empty()) cudaEventCreate(&e);
+ else {
+ e = prof_pool.back();
+ prof_pool.pop_back();
+ }
+ cudaEventRecord(e, d.st);
+ prof_marks.push_back({&d, e, stage});
+ }
+ void prof_collect() {
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ std::vector<std::pair<const Dev*, cudaEvent_t>> last;
+ for (const ProfMark& m : prof_marks) {
+ auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
+ if (it != last.end()) {
+ float ms = 0.0f;
+ if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
+ (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
+ it->second = m.ev;
+ } else {
+ last.push_back({m.d, m.ev});
+ }
+ }
+ for (const ProfMark& m : prof_marks) prof_pool.push_back(m.ev);
+ prof_marks.clear();
+ if (!prof_chunk) ++st.prof_tokens;
+ }
+ void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
+ std::vector<float> v((size_t) T * g.hc * g.dim);
+ cudaSetDevice(d.id);
+ cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[3] = {il, p0, T};
+ std::fwrite(hdr, 8, 3, dump_l);
+ std::fwrite(v.data(), 4, v.size(), dump_l);
+ std::fflush(dump_l);
+ }
+
+ ~Impl() {
+ eg_io.stop(); // (its threads write into h_eg)
+ for (auto& w : workers) w->stop();
+ for (size_t i = 0; i < chunks.size(); ++i)
+ if (chunks[i] && chunks[i]->blas) {
+ cudaSetDevice(devs[i].id);
+ cublasDestroy(chunks[i]->blas);
+ }
+ quit.store(true);
+ if (service.joinable()) service.join();
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ }
+ pf_cv.notify_all();
+ if (pf_th.joinable()) pf_th.join();
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ }
+ sw_cv.notify_all();
+ ram_cv.notify_all();
+ for (std::thread& t : cache_ths)
+ if (t.joinable()) t.join();
+ if (ram_th.joinable()) ram_th.join();
+ for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ for (void* p : d.allocs) cudaFree(p);
+ if (d.st) cudaStreamDestroy(d.st);
+ if (d.ev) cudaEventDestroy(d.ev);
+ }
+ for (void* p : host_allocs) cudaFreeHost(p);
+ for (Layer& l : L)
+ for (uint8_t* b : l.host)
+ if (b != nullptr) {
+ (void) b; // host blobs live in one mapping per layer, unmapped below
+ }
+ for (auto& m : host_maps) {
+ if (chunk > 0) cudaHostUnregister(m.first);
+ os::anon_free(m.first, m.second);
+ }
+ }
+ std::vector<std::pair<void*, size_t>> host_maps;
+
+ template <class T>
+ bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {
+ void* v = nullptr;
+ cudaSetDevice(d.id);
+ const size_t b = align256(n * sizeof(T));
+ if (cudaMalloc(&v, b) != cudaSuccess) {
+ err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st
+ // does not order against - a zeroing still queued there could wipe the weights just copied
+ if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {
+ cudaFree(v);
+ err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ if (acct) *acct += b;
+ else d.scratch += b;
+ p = (T*) v;
+ return true;
+ }
+ static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ // bytes of a VRAM slot of l's experts (a member's part of each, in a group)
+ static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }
+ // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64
+ // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share
+ int head_members(int nm) const {
+ int na = nm;
+ while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;
+ return na;
+ }
+ // host-mapped memory (the same address on the host and every device), zeroed
+ template <class T>
+ bool hmap(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";
+ return false;
+ }
+ std::memset(v, 0, n * sizeof(T) + 64);
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+ template <class T>
+ bool halloc(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {
+ err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";
+ return false;
+ }
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+
+ // ---- weights
+ bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {
+ std::vector<uint8_t> tmp(t.bytes);
+ if (!pk.read(t, tmp.data())) {
+ err = "read failed";
+ return false;
+ }
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
+ return true;
+ }
+ // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
+ // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
+ // standalone [n][k] matrix
+ bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ const bool q8 = t.dtype == "Q8_0";
+ if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)
+ auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };
+ std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +
+ std::to_string(np) + " parts";
+ return false;
+ }
+ const size_t rb = bytes_of((size_t) k);
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = bytes_of((size_t) wd);
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r)
+ std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = q8 ? (int) bytes_of((size_t) k) : k;
+ return true;
+ }
+ // a Q8_0 tensor's values as F32 (host)
+ static std::vector<float> q8_values(const uint8_t* q, size_t n) {
+ std::vector<float> v(n);
+ for (size_t b = 0; b < n / 32; ++b) {
+ const uint8_t* p = q + b * gg::kQ8Bytes;
+ const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));
+ for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];
+ }
+ return v;
+ }
+ // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone
+ // matrix of its own (the rotations' 128-blocks stay whole)
+ // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix
+ // splits evenly, its groups only as wide as that goes)
+ bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
+ bool cols = true, int rot = 0) {
+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0"))
+ return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
+ const Tensor* tr = pk.get(pre + ".trellis");
+ const Tensor* su = pk.get(pre + ".suh");
+ const Tensor* sv = pk.get(pre + ".svh");
+ if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {
+ err = pre + ": not an EXL3 matrix in the pack";
+ return false;
+ }
+ if (pk.get(pre + ".mul1") == nullptr) {
+ err = pre + ": only the mul1 codebook is covered";
+ return false;
+ }
+ m.k = (int) tr->shape[0] * 16;
+ m.n = (int) tr->shape[1] * 16;
+ m.K = (int) tr->shape[2] / 16;
+ m.cb = 2;
+ m.ld = 0;
+ if (np > 1) {
+ const int whole = cols ? m.n : m.k;
+ int at = 0, w = 0;
+ if (whole % 128 != 0 || whole / 128 < np) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +
+ std::to_string(np) + " parts of 128-blocks";
+ return false;
+ }
+ split128(whole, np, part, at, w, rot);
+ std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);
+ if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));
+ std::vector<uint8_t> t1, u1, v1;
+ if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)
+ const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;
+ t1.resize((size_t) (m.k / 16) * nth * tile);
+ for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)
+ std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);
+ u1 = u0;
+ v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));
+ m.n = w;
+ } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)
+ const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;
+ t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));
+ u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));
+ v1 = v0;
+ m.k = w;
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||
+ !dalloc(d, dv, v1.size() / 2, err, acct))
+ return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||
+ !dalloc(d, dv, sv->bytes / 2, err, acct))
+ return false;
+ if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ if (!x3::supported(m)) {
+ err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +
+ " not covered by the EXL3 kernels";
+ return false;
+ }
+ return true;
+ }
+ // a float tensor (F32 / F16 / BF16) as F32 on the device
+ bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {
+ const Tensor* t = pk.get(name);
+ if (t == nullptr) {
+ err = name + ": not in the pack";
+ return false;
+ }
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ size_t n = 0;
+ std::vector<float> v;
+ if (t->dtype == "F32") {
+ n = t->bytes / 4;
+ v.resize(n);
+ std::memcpy(v.data(), raw.data(), t->bytes);
+ } else if (t->dtype == "F16" || t->dtype == "BF16") {
+ n = t->bytes / 2;
+ v.resize(n);
+ const uint16_t* s = (const uint16_t*) raw.data();
+ for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);
+ } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)
+ n = t->bytes / gg::kQ8Bytes * 32;
+ v = q8_values(raw.data(), n);
+ } else {
+ err = name + ": dtype " + t->dtype + " is not a float";
+ return false;
+ }
+ if (want != 0 && n != want) {
+ err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);
+ return false;
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));
+ return true;
+ }
+ bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {
+ const Tensor* t = pk.get(name);
+ if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)
+ const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();
+ std::vector<uint16_t> h(n);
+ for (size_t i = 0; i < n; ++i) {
+ float v;
+ if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);
+ else if (t->dtype == "Q8_0") v = q[i];
+ else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);
+ h[i] = strata::kernels::f16_from_f32(v);
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));
+ return true;
+ }
+ if (t == nullptr || t->dtype != "F16") {
+ err = name + ": not an F16 tensor in the pack";
+ return false;
+ }
+ if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;
+ return upload(d, *t, p, err);
+ }
+
+ size_t mat_bytes(const std::string& pre) const {
+ if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
+ size_t b = 0;
+ for (const char* s : {".trellis", ".suh", ".svh"})
+ if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
+ return b;
+ }
+ size_t tensor_bytes(const std::string& name) const {
+ const Tensor* t = pk.get(name);
+ return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
+ }
+
+ // ---- expert regions and blob layout
+ // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every
+ // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
+ // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
+ static int gg_type_of(const std::string& dtype) {
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23})
+ if (dtype == gg::type_name(t)) return t;
+ return -1;
+ }
+ bool expert_layout_mx(Layer& l, std::string& err) {
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;
+ const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");
+ const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;
+ const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;
+ const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;
+ if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {
+ err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +
+ "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";
+ return false;
+ }
+ l.mx_file[r] = t0->file;
+ l.mx_off[r] = t0->off;
+ l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");
+ if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||
+ t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {
+ err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";
+ return false;
+ }
+ }
+ l.lay.trellis[r] = off;
+ l.lay.suh[r] = l.lay.svh[r] = off;
+ l.lay.K[r] = ty;
+ off += align256(bytes);
+ }
+ l.lay.fmt = gg::kFmtMXFP4;
+ if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
+ gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||
+ gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {
+ err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +
+ " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";
+ return false;
+ }
+ l.blob = off;
+ return true;
+ }
+ const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
+ return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
+ : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
+ return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);
+ }
+ size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)
+ return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);
+ }
+ // (copy_mode) matrix r's rows, and the rows of one NUMA part of it
+ int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }
+ size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }
+ // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them
+ const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {
+ if (copy_mode && l.ram[(size_t) e] >= 0)
+ return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ }
+ // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)
+ void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);
+ }
+ }
+ void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);
+ }
+ }
+ // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down
+ // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back
+ bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {
+ if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);
+ else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);
+ return true;
+ }
+ return read_mx_file(l, e, blob);
+ }
+ bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)
+ for (int r = 0; r < 3; ++r)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),
+ blob + l.lay.trellis[r]))
+ return false;
+ return true;
+ }
+ void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);
+ }
+ void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);
+ }
+
+ bool expert_layout(Layer& l, std::string& err) {
+ if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)
+ return expert_layout_mx(l, err);
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ const char* pieces[3] = {".trellis", ".suh", ".svh"};
+ l.region_off.resize((size_t) l.n_exp);
+ l.region_bytes.resize((size_t) l.n_exp);
+ l.region_file.resize((size_t) l.n_exp);
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");
+ if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {
+ err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";
+ return false;
+ }
+ l.lay.K[r] = (int) tr->shape[2] / 16;
+ l.lay.trellis[r] = off;
+ off += align256(tr->bytes);
+ l.lay.suh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);
+ l.lay.svh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);
+ }
+ l.lay.cb = 2;
+ l.blob = off;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const std::string E = P + std::to_string(e) + ".";
+ uint64_t lo = UINT64_MAX, hi = 0;
+ int file = -1;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const Tensor* t = pk.get(E + parts[r] + pieces[q]);
+ if (t == nullptr) {
+ err = E + parts[r] + pieces[q] + ": missing";
+ return false;
+ }
+ if (file < 0) file = t->file;
+ if (t->file != file) {
+ err = E + ": split across files";
+ return false;
+ }
+ lo = std::min(lo, t->off);
+ hi = std::max(hi, t->off + t->bytes);
+ if (e == 0) l.piece_bytes[r][q] = t->bytes;
+ else if (t->bytes != l.piece_bytes[r][q]) {
+ err = E + parts[r] + ": not the shape of expert 0's";
+ return false;
+ }
+ }
+ l.region_off[(size_t) e] = lo;
+ l.region_bytes[(size_t) e] = hi - lo;
+ l.region_file[(size_t) e] = file;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;
+ if (e == 0) l.piece_src[r][q] = src;
+ else if (src != l.piece_src[r][q]) {
+ err = E + ": the pieces lie differently from expert 0's";
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ // the region of expert e (read from the file) -> its blob
+ void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};
+ for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);
+ }
+ }
+
+ // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)
+ void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = region + l.piece_src[r][0];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // ---- loading
+ bool load(const Options& opt, std::string& err) {
+ o = opt;
+ const double t0 = now_ms();
+ if (!pk.load(o.pack, err)) return false;
+ g.from(pk);
+ if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {
+ err = o.pack + "/ds4.txt: incomplete geometry";
+ return false;
+ }
+ if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {
+ err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +
+ std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";
+ return false;
+ }
+ freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);
+ freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);
+
+ // devices
+ int ndev = 0;
+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
+ err = "no CUDA device";
+ return false;
+ }
+ std::vector<int> ids = o.gpus;
+ if (ids.empty())
+ for (int i = 0; i < ndev; ++i) ids.push_back(i);
+ devs.resize(ids.size());
+ for (size_t i = 0; i < ids.size(); ++i) {
+ Dev& d = devs[i];
+ d.id = ids[i];
+ CK(cudaSetDevice(d.id));
+ CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));
+ CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));
+ if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {
+ err = "exl3 scratch on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ CK(cudaMemGetInfo(&d.free0, &d.total));
+ }
+ // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each
+ // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)
+ int tp = o.tp;
+ if (pk.gguf && tp > 1) {
+ int blk = 32;
+ for (const auto& kv : pk.t) {
+ const std::string& n = kv.first;
+ if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||
+ n.compare(n.size() - 9, 9, "w2.weight") != 0)
+ continue;
+ const int ty = gg_type_of(kv.second.dtype);
+ if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
+ }
+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % 128 != 0 || g.heads % tp != 0 ||
+ g.o_groups % tp != 0))
+ --tp;
+ if (tp != o.tp)
+ std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "
+ "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);
+ }
+ // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone
+ // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the
+ // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the
+ // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down
+ // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
+ par = o.threads || tp > 1;
+ prompt_par = par || o.tp > 1;
+ const bool tp_all = tp > 1 && tp == (int) devs.size();
+ for (size_t i = 0; i < devs.size();) {
+ Group G;
+ G.mem.push_back((int) i);
+ size_t j = i + 1;
+ while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))
+ G.mem.push_back((int) j++);
+ if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own
+ for (int m : G.mem) {
+ Group one;
+ one.mem.push_back(m);
+ groups.push_back(one);
+ }
+ } else groups.push_back(G);
+ i = j;
+ }
+ for (size_t gi = 0; gi < groups.size(); ++gi)
+ for (size_t m = 0; m < groups[gi].mem.size(); ++m) {
+ devs[(size_t) groups[gi].mem[m]].grp = (int) gi;
+ devs[(size_t) groups[gi].mem[m]].mi = (int) m;
+ }
+ // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the
+ // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM
+ // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place
+ // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times
+ if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM
+ // after VRAM's are in, by --mmap-experts)
+ else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;
+ else {
+ double exp_b = 0, other_b = 0, room = 0;
+ for (const auto& kt : pk.t) {
+ const std::string& n = kt.first;
+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
+ (!o.dspark && n.rfind("mtp.", 0) == 0))
+ continue;
+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
+ }
+ for (const Dev& d : devs)
+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -
+ (prompt_par ? 1.5 : 0.4) * (1 << 30)) *
+ o.vram_experts;
+ file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
+ }
+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;
+ const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)
+ if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
+ if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
+ if (chunk > 0) {
+ chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));
+ eg_cap = (size_t) chunk;
+ tap_cap = std::max(kWin, 128);
+ } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)
+ std::fprintf(stderr,
+ "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"
+ " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"
+ " 461 / 1010 in chunks of %d) - %s\n",
+ kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,
+ o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"
+ : "leave --tp 1 out, or pass --threads on, for chunks");
+ // the vision tower on the first GPU, before the split: the layers get what it leaves
+ if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,
+ if (o.vision_only) return false; // and the server is told pictures are off - info_lines)
+ std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "
+ "text only\n", err.c_str());
+ vision_err = err.empty() ? "unknown reason" : err;
+ err.clear();
+ vt.release();
+ img_tok = -1;
+ }
+ if (o.vision_only) return true;
+
+ // layers: roles and the sources they read
+ if (!o.dspark) g.n_mtp = 0;
+ L.resize((size_t) (g.nl + g.n_mtp));
+ int last_src = -1;
+ for (int il = 0; il < g.nl + g.n_mtp; ++il) {
+ Layer& l = L[(size_t) il];
+ l.il = il;
+ l.mtp = il >= g.nl;
+ l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
+ l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
+ l.k_exp = l.mtp ? g.d_k : g.k_exp;
+ l.ratio = g.ratios[(size_t) il];
+ if (l.mtp) { // window attention only: no compressor, indexer or Engram
+ if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
+ err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +
+ std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";
+ return false;
+ }
+ if (!expert_layout(l, err)) return false;
+ continue;
+ }
+ l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();
+ l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();
+ l.owns_k = l.kv_src;
+ l.cand_src = il == g.cand_src;
+ l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;
+ if (l.kv_src) {
+ Src s;
+ s.layer = il;
+ s.ratio = l.ratio;
+ s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;
+ srcs.push_back(s);
+ last_src = (int) srcs.size() - 1;
+ }
+ if (l.ratio > 0) {
+ if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {
+ err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +
+ " without a KV source of that ratio before it";
+ return false;
+ }
+ l.src = last_src;
+ l.ik_src = last_src;
+ }
+ auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);
+ if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());
+ if (!expert_layout(l, err)) return false;
+ }
+
+ // the split: each layer's dense weights + its experts, the head on the last GPU
+ for (Layer& l : L) {
+ const std::string& P = l.name;
+ size_t b = 0;
+ if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");
+ for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",
+ "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",
+ "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})
+ b += mat_bytes(P + m);
+ for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));
+ b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");
+ b += (size_t) g.win * win_rb() + (2u << 20);
+ l.dense_bytes = b;
+ }
+ const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);
+ const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;
+ const size_t kv_per_src_dev = 0; // counted after the split
+ (void) kv_per_src_dev;
+ std::vector<double> avail(groups.size(), 0.0);
+ double avail_sum = 0;
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ for (int m : groups[gi].mem)
+ avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));
+ avail_sum += avail[gi];
+ }
+ double need_sum = (double) head_b;
+ for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ {
+ // contiguous layer ranges, each group's share of the total by its free memory
+ size_t d = 0;
+ double cum = 0, target = avail[0] / avail_sum * need_sum;
+ std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers
+ for (Layer& l : L)
+ if (l.mtp) order.push_back(&l);
+ for (Layer& l : L)
+ if (!l.mtp) order.push_back(&l);
+ for (Layer* lp : order) {
+ Layer& l = *lp;
+ const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ if (cum + lb / 2 > target && d + 1 < groups.size()) {
+ ++d;
+ target += avail[d] / avail_sum * need_sum;
+ }
+ l.grp = (int) d;
+ cum += lb;
+ }
+ // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its
+ // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage
+ // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all
+ if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {
+ const int G = (int) groups.size(), NL = g.nl;
+ std::vector<double> cap(groups.size());
+ for (int gi = 0; gi < G; ++gi) {
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ double mn = 1e300;
+ for (int m : groups[(size_t) gi].mem)
+ mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -
+ 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)
+ cap[(size_t) gi] = mn * nm;
+ }
+ for (const Layer& l : L)
+ if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ cap[(size_t) G - 1] -= (double) head_b;
+ std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense
+ for (int il = 0; il < NL; ++il) {
+ pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;
+ pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;
+ }
+ auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);
+ return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;
+ };
+ // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)
+ std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));
+ std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));
+ for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);
+ for (int gi = 1; gi < G; ++gi)
+ for (int b = gi + 1; b <= NL; ++b)
+ for (int a = gi; a < b; ++a) {
+ const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));
+ if (v < best[(size_t) gi][(size_t) b]) {
+ best[(size_t) gi][(size_t) b] = v;
+ cut[(size_t) gi][(size_t) b] = a;
+ }
+ }
+ int b = NL;
+ for (int gi = G - 1; gi >= 0; --gi) {
+ const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;
+ for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;
+ b = a;
+ }
+ }
+ // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)
+ if (const char* sp = std::getenv("DS4_SPLIT")) {
+ std::vector<int> n;
+ for (const char* q = sp; *q;) {
+ n.push_back(std::atoi(q));
+ while (*q && *q != ',') ++q;
+ if (*q == ',') ++q;
+ }
+ int il = 0;
+ for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)
+ for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;
+ for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;
+ }
+ for (Layer& l : L) {
+ l.dev = groups[(size_t) l.grp].mem[0];
+ l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)
+ }
+ head_dev = L[(size_t) g.nl - 1].dev;
+ d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;
+ }
+ // the groups' members: copies of each layer's device state, a member's part of an expert
+ for (Layer& l : L) {
+ l.ff0 = 0;
+ l.ffn = g.ff;
+ l.ah0 = l.og0 = 0;
+ l.ahn = g.heads;
+ l.ogn = g.o_groups;
+ if (l.nm == 1) continue;
+ const int na = head_members(l.nm);
+ if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";
+ return false;
+ }
+ for (int m = 1; m < l.nm; ++m) {
+ Layer c = l;
+ c.mates.clear();
+ c.mi = m;
+ c.dev = groups[(size_t) l.grp].mem[(size_t) m];
+ l.mates.push_back(std::move(c));
+ }
+ // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even
+ // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its
+ // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)
+ const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const int ha = m - (l.nm - na); // (its place among the members with heads)
+ x.ahn = ha < 0 ? 0 : g.heads / na;
+ x.ah0 = ha < 0 ? 0 : ha * x.ahn;
+ x.ogn = ha < 0 ? 0 : g.o_groups / na;
+ x.og0 = ha < 0 ? 0 : ha * x.ogn;
+ if (gguf_x) {
+ x.ffn = g.ff / l.nm;
+ x.ff0 = m * x.ffn;
+ } else
+ split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;
+ x.hlay.K[r] = l.lay.K[r];
+ x.hlay.trellis[r] = off;
+ if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks
+ const int ty = gg::expert_type(l.lay, r);
+ if (k % gg::type_block(ty) != 0) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +
+ " blocks do not split";
+ return false;
+ }
+ x.hlay.suh[r] = x.hlay.svh[r] = off;
+ off += align256((size_t) n * gg::type_row_bytes(ty, k));
+ continue;
+ }
+ off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));
+ x.hlay.suh[r] = off;
+ off += align256((size_t) k * 2);
+ x.hlay.svh[r] = off;
+ off += align256((size_t) n * 2);
+ }
+ x.hlay.cb = l.lay.cb;
+ x.hlay.fmt = l.lay.fmt;
+ x.hblob = off;
+ }
+ }
+ // which devices hold each source's compressed KV / index keys
+ for (Src& s : srcs) {
+ s.ckv.assign(devs.size(), nullptr);
+ s.ik.assign(devs.size(), nullptr);
+ }
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ const Layer& l = member(l0, m);
+ if (l.src >= 0) {
+ srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;
+ }
+ if (l.ix_src && l.ik_src >= 0) {
+ srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;
+ }
+ }
+
+ // per device: the scratch, the caches, the dense weights
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ const int D = g.dim, H = g.hc;
+ const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&
+ dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&
+ dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&
+ dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&
+ dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&
+ dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&
+ dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&
+ dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&
+ dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&
+ dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&
+ dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&
+ dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&
+ dalloc(d, d.rw, W * kPicks, err) &&
+ dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.moe_out, W * D, err) &&
+ dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&
+ dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&
+ dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&
+ dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&
+ dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&
+ dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
+ dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
+ dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
+ if (!ok) return false;
+ d.attn_scr.max_t = kAttnScratchT;
+ d.attn_scr.cap = (int) topk_stride();
+ if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;
+ if (!g.eg_layers.empty()) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&
+ dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);
+ if (!ok) return false;
+ }
+ if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
+ if (!halloc(d.bounce, bounce_bytes(), err)) return false;
+ if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
+ !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
+ !dalloc(d, d.n_groups, 4, err))
+ return false;
+ if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;
+ if (g.n_mtp > 0 && (int) di == head_dev &&
+ (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))
+ return false;
+ CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));
+ for (Src& s : srcs) {
+ if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)
+ if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))
+ return false;
+ CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));
+ CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));
+ CK(cudaSetDevice(d.id));
+ }
+ if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))
+ return false;
+ if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))
+ return false;
+ }
+ }
+ for (Src& sr : srcs) {
+ if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))
+ return false;
+ if (sr.remote && chunk > 0 &&
+ (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||
+ !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))
+ return false;
+ const int sg = L[(size_t) sr.layer].grp;
+ for (size_t di = 0; di < devs.size(); ++di)
+ if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);
+ }
+ for (Group& G : groups) {
+ G.ar.nm = (int) G.mem.size();
+ G.ar.slots = 4;
+ G.ar.stride = (size_t) kWin * g.dim;
+ if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))
+ return false;
+ // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)
+ if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))
+ return false;
+ if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&
+ (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||
+ !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))
+ return false;
+ G.arb = G.ar; // (a chunk's: two slots of chunk rows)
+ G.arb.slots = 2;
+ G.arb.stride = (size_t) chunk * g.dim;
+ if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))
+ return false;
+ if (chunk > 0 && G.arb.nm > 2 &&
+ (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))
+ return false;
+ }
+ if (g.n_mtp > 0) {
+ if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;
+ CK(cudaSetDevice(devs[(size_t) d_dev].id));
+ CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));
+ CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));
+ }
+ // the dense weights, each device's layers by a thread of its own
+ {
+ std::vector<std::string> errs(devs.size());
+ std::vector<std::thread> th;
+ for (size_t di = 0; di < devs.size(); ++di)
+ th.emplace_back([this, di, &errs] {
+ std::string e;
+ if (!load_dense((int) di, e)) errs[di] = e;
+ });
+ for (auto& t : th) t.join();
+ for (auto& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);
+ // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E
+ // host-mapped, the steps' slots
+ // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)
+ bool mk_rows = g.n_mtp > 0 && head_nm() > 1;
+ for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;
+ if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {
+ const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");
+ if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {
+ err = "the markov embed: not a BF16 [vocab][rank] table";
+ return false;
+ }
+ if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||
+ !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {
+ if (err.empty()) err = "the markov embed: read failed";
+ return false;
+ }
+ devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)
+ mk_split = true;
+ }
+
+ // the embedding table, pinned on the host
+ {
+ const Tensor* t = pk.get("embed.weight");
+ const size_t nv = (size_t) g.vocab * g.dim;
+ const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
+ if (t == nullptr || (!q8 && (t->dtype != "BF16" || t->bytes != nv * 2))) {
+ err = "embed.weight: not a BF16 (or Q8_0) [vocab][dim] table";
+ return false;
+ }
+ if (!halloc(embed_pinned, nv, err)) return false;
+ if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
+ std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);
+ for (size_t c = 0; c < (size_t) g.dim; ++c) {
+ uint32_t u;
+ std::memcpy(&u, &v[c], 4);
+ u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even
+ embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);
+ }
+ }
+ } else if (!pk.read(*t, embed_pinned)) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ }
+ // the host side of the CPU lane and the Engram rows
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||
+ !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) ||
+ !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||
+ !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))
+ return false;
+ if (pk.eg_layers > 0) {
+ // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);
+ // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
+ const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
+ const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
+ int nth = (int) std::min(16u, std::max(4u, hw / 2));
+ if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
+ const uint64_t avail = ram_avail();
+ uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
+ if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);
+ eg_cache.init((size_t) ((mb << 20) / row), row);
+ std::vector<std::string> paths;
+ for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
+ eg_io.start(paths, nth);
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ eg_win.emplace_back(new EgBatch());
+ const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");
+ const bool q6k = tw != nullptr && tw->dtype == "Q6_K";
+ if (q6k && pk.eg_hd % 256 != 0) {
+ err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " +
+ std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";
+ return false;
+ }
+ eg_q6k.push_back(q6k ? 1 : 0);
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
+ "their layer; row cache %llu MB (%zu rows)\n",
+ nth, (unsigned long long) mb, eg_cache.capacity());
+ }
+ prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);
+ if (!o.profile.empty()) {
+ std::FILE* f = std::fopen(o.profile.c_str(), "rb");
+ if (f != nullptr) {
+ std::vector<uint32_t> p(prof.size());
+ const size_t n = std::fread(p.data(), 4, p.size(), f);
+ if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());
+ std::fclose(f);
+ }
+ }
+
+ // the chunked prompt path's buffers (before the experts: they take what is left)
+ if (chunk > 0 && !alloc_chunks(err)) return false;
+ // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM
+ // now leaves over
+ if (file_experts) {
+ double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;
+ }
+ for (const Layer& l : L) all += (double) l.blob * l.n_exp;
+ const double need = std::max(0.0, all - room);
+ const uint64_t avail = ram_avail();
+ warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
+ if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
+ std::fprintf(stderr,
+ "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
+ "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
+ need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"
+ : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",
+ avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);
+ }
+ // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in
+ // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)
+ cores = node_cores();
+ NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());
+ if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;
+ if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)
+ {
+ const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;
+ c_xh.resize((size_t) NH * S * 2 * E);
+ c_xsb.resize((size_t) NH * S * 2 * (E / 128));
+ c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));
+ c_xhd.resize(S * FF);
+ c_xsd.resize(S * (FF / 128));
+ c_accd.resize((size_t) NH * S * DQ * (E / NH));
+ c_yd.resize(S * E);
+ }
+ // the experts: VRAM slots in what each device has left, the rest on the host
+ if (!place_experts(err)) return false;
+ if (!load_experts(err)) return false;
+ if (pk.gguf) make_pools();
+ if (chunk > 0) {
+ // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
+ const double tr = now_ms();
+ for (auto& m : host_maps)
+ if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {
+ err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";
+ return false;
+ }
+ if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);
+ if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)
+ size_t mr = 0;
+ for (const Layer& l : L)
+ if (!l.mtp) {
+ for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)
+ mr = std::max(mr, l.blob);
+ for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));
+ }
+ }
+ const char* nr = std::getenv("DS4_READERS");
+ const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);
+ const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;
+ // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the
+ // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the
+ // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)
+ const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ auto& cp = chunks[di];
+ if (cp == nullptr || cp->ring == nullptr) continue;
+ cp->stage_stride = (mr + 4095) & ~(size_t) 4095;
+ // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)
+ bool fl = flow_on;
+ int n_exp_max = 0;
+ for (const Layer& l : L)
+ if (!l.mtp)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ // (a group's members: each its share of the down's blocks in every row)
+ fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&
+ (l.nm == 1 || (flow_tp && (g.ff / l.nm) %
+ std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));
+ n_exp_max = std::max(n_exp_max, l.n_exp);
+ }
+ if (fl) // (each part read unbuffered from its offset rounded down: a page more each)
+ cp->stage_stride = std::max(cp->stage_stride, flow_stride());
+ cp->S = cp->ring_n;
+ if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;
+ cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);
+ if (!fl) continue;
+ cudaSetDevice(devs[di].id);
+ cp->flow = true;
+ cp->dev = (int) di;
+ cp->io = std::make_unique<IoQueue>(n_readers);
+ cp->stage_item.assign((size_t) cp->S, -1);
+ cp->stage_ev.resize((size_t) cp->S);
+ cp->ring_fill.resize((size_t) cp->ring_n);
+ cp->ring_free.resize((size_t) cp->ring_n);
+ for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})
+ for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ // (a region is written again only bc_n batches later - more than a layer's items, so after that
+ // layer's bounds wait, which every copy from it precedes)
+ cp->bc_n = n_exp_max + 2;
+ cp->bc_w = cp->ring_n + 1;
+ if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||
+ !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))
+ return false;
+ }
+ // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one
+ // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each
+ // window, a page fault at a time) - DS4_CHUNK_MIN still overrides
+ bool all_flow = true;
+ for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);
+ if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;
+ }
+ for (Layer& l : L) {
+ if (l.mtp) continue;
+ for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)
+ Layer& x = member(l, m);
+ if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;
+ if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;
+ }
+ }
+ }
+ {
+ void* m = nullptr;
+ if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's requests";
+ return false;
+ }
+ host_allocs.push_back(m);
+ reqs = (Req*) m;
+ if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's answers";
+ return false;
+ }
+ host_allocs.push_back(m);
+ resps = (Resp*) m;
+ std::memset((void*) reqs, 0, sizeof(Req) * kRing);
+ std::memset((void*) resps, 0, sizeof(Resp) * kRing);
+ for (auto& x : layer_of) x.store(-1);
+ ahead_fl.assign(L.size() * 1024, 0);
+ ahead_rank.assign(L.size() * 1024, 0);
+ ahead_for.assign(L.size(), 0);
+ const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;
+ service = std::thread([this, core] {
+ if (core >= 0) pin_self(core);
+ serve_lane();
+ });
+ if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });
+ }
+ if (o.expert_cache) {
+ size_t maxb = 0;
+ for (const Layer& l : L) maxb = std::max(maxb, l.blob);
+ const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));
+ sw_bufs.resize((size_t) nt);
+ for (SwapBufs& B : sw_bufs) {
+ if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStream_t cs = nullptr;
+ CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
+ B.st.push_back(cs);
+ }
+ }
+ for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ }
+ if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)
+ for (size_t di = 0; di < devs.size(); ++di) {
+ workers.push_back(std::make_unique<Worker>());
+ workers.back()->start();
+ }
+ calibrate_paths();
+ if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
+ reset();
+ return true;
+ }
+
+
+ // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed
+ // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)
+ size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }
+ size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
+ size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
+
+ // ---- the chunked prompt path
+ size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
+ size_t chunk_hand_bytes() const {
+ return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+
+ bool alloc_chunks(std::string& err) {
+ chunks.resize(devs.size());
+ const int D = g.dim, H = g.hc, HD = g.hd;
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ cudaSetDevice(d.id);
+ chunks[di] = std::make_unique<Chunk>();
+ Chunk& c = *chunks[di];
+ const int nm = (int) groups[(size_t) d.grp].mem.size();
+ const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);
+ int FF = 0, ff0_ = 0;
+ split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
+ const size_t C = (size_t) chunk;
+ c.C = chunk;
+ c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
+ // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
+ c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
+ c.w16_elems = (size_t) 8 << 20;
+ bool has_eg = false, has_first = false;
+ size_t stride = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di && !l.mtp) {
+ has_eg = has_eg || l.eg >= 0;
+ has_first = has_first || l.il == 0;
+ stride = std::max(stride, align256(unit(member(l, m))));
+ }
+ const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&
+ dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&
+ dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&
+ dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&
+ dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&
+ dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&
+ dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&
+ dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&
+ dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&
+ dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&
+ dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&
+ dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&
+ dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&
+ dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&
+ dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&
+ dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
+ dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
+ dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
+ dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
+ halloc(c.bounds_pin, 2048, err);
+ if (!ok) return false;
+ if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
+ if (has_eg) {
+ const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);
+ if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||
+ !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))
+ return false;
+ }
+ if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches
+ const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;
+ c.ring_stride = stride;
+ c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);
+ if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;
+ }
+ CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));
+ for (int i = 0; i < Chunk::kParts; ++i) {
+ CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));
+ }
+ if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {
+ err = "cuBLAS on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ for (int sl = 0; sl < 2; ++sl) {
+ if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;
+ if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;
+ if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;
+ CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));
+ }
+ if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;
+ }
+ return true;
+ }
+
+ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS
+ bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
+ std::string& err) {
+ if (rows <= 0) return true;
+ const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
+ for (int c0 = 0; c0 < m.n; c0 += cols) {
+ const int nc = std::min(cols, m.n - c0);
+ if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
+ else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
+ const float one = 1.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
+ m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx: status " + std::to_string((int) st);
+ return false;
+ }
+ }
+ return true;
+ }
+ // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)
+ bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,
+ float* y, int ldy, std::string& err) {
+ const float zero = 0.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,
+ x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);
+ return false;
+ }
+ return true;
+ }
+ // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale
+ bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {
+ const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
+ const float one = 1.0f, zero = 0.0f;
+ if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=
+ CUBLAS_STATUS_SUCCESS) {
+ err = "cublasSgemm (mHC mixes)";
+ return false;
+ }
+ hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);
+ return true;
+ }
+
+ // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full
+ // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file
+ // region read into pinned staging)
+ bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,
+ const uint8_t* src = nullptr) {
+ if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into
+ // pinned staging as its blob) or from the file's mapping
+ auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };
+ bool ok = true;
+ if (l.nm == 1) {
+ if (src != nullptr)
+ ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ else
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ } else {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,
+ (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,
+ (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+ const auto& parts = l.hh[(size_t) e];
+ const int D = g.dim, FF = g.ff;
+ auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {
+ return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {
+ return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ bool ok = true;
+ if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets
+ const uint8_t* p = src != nullptr ? src : parts[0];
+ if (l.nm == 1) {
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&
+ cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&
+ cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);
+ } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;
+ ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,
+ nt * tile, nth * tile, (size_t) D / 16) &&
+ cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&
+ cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];
+ ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,
+ (size_t) (fn / 16) * row) &&
+ cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&
+ cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);
+ }
+ } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,
+ (size_t) (k / 16));
+ ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ }
+ } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of
+ // both parts' columns
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)
+ for (int X = 0; X < 2; ++X) {
+ const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);
+ if (a >= b) continue;
+ ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,
+ (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&
+ cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,
+ (size_t) (b - a) * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);
+ }
+ const size_t tile = (size_t) 32 * l.lay.K[2];
+ const int nh = D / 2;
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,
+ (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));
+ ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+
+ // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk
+ // in layer order; the copies run at most kParts batches ahead of the compute
+ struct Batch {
+ int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range
+ };
+ struct Streamer {
+ std::vector<Batch> b;
+ size_t cp = 0, kp = 0; // the next batch to copy, to compute
+ std::atomic<size_t> done{0}; // cp, for read_ahead's thread
+ };
+ // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel
+ // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -
+ // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone
+ void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {
+ static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *
+ 1048576.0;
+ std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied
+ double ahead = 0;
+ for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {
+ const size_t done = S.done.load(std::memory_order_relaxed);
+ while (!issued.empty() && issued.front().first < done) {
+ ahead -= issued.front().second;
+ issued.pop_front();
+ }
+ if (ahead > cap) {
+ std::this_thread::sleep_for(std::chrono::microseconds(500));
+ continue;
+ }
+ const Batch& bt = S.b[q];
+ const Layer& l0 = L[(size_t) bt.il];
+ double nb = 0;
+ if (l0.dev == di)
+ for (int i = 0; i < bt.n; ++i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4) {
+ if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);
+ nb += (double) l0.blob;
+ continue;
+ }
+ os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);
+ nb += (double) l0.region_bytes[(size_t) e];
+ }
+ issued.push_back({q++, nb});
+ ahead += nb;
+ }
+ }
+ bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {
+ const int P = Chunk::kParts;
+ while (S.cp < S.b.size() && S.cp < S.kp + P) {
+ const Batch& bt = S.b[S.cp];
+ const int half = (int) (S.cp % P);
+ const Layer& l0 = L[(size_t) bt.il];
+ int mi = 0;
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di) mi = m;
+ if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));
+ uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots
+ if (c.stage != nullptr) {
+ if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)
+ sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;
+ std::atomic<bool> bad{false};
+ c.readers->run(bt.n, [&](int i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)
+ : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],
+ l0.region_bytes[(size_t) e],
+ sb + (size_t) i * c.stage_stride))
+ bad.store(true);
+ });
+ if (bad.load()) {
+ err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";
+ return false;
+ }
+ }
+ for (int i = 0; i < bt.n; ++i) {
+ uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;
+ if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,
+ sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))
+ return false;
+ c.stream_bytes += (double) unit(member(l0, mi));
+ }
+ CK(cudaEventRecord(c.filled[half], c.cs));
+ ++S.cp;
+ S.done.store(S.cp, std::memory_order_relaxed);
+ }
+ return true;
+ }
+
+ // ---- the flow (see Chunk)
+ // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute
+ // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing
+ static void kick(cudaStream_t s) {
+#if defined(_WIN32) && defined(STRATA_USE_HIP)
+ (void) cudaStreamQuery(s);
+ (void) cudaGetLastError();
+#else
+ (void) s;
+#endif
+ }
+ size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
+ return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
+ }
+ size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }
+ static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }
+ // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from
+ // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of
+ // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole
+ size_t mx_read_off(const Layer& l, int m, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : 0;
+ }
+ size_t mx_read_len(const Layer& l, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : mx_part_bytes(l, r);
+ }
+ // the device's member index in L0's group (0 alone)
+ int flow_mi(const Chunk& c, const Layer& L0) const {
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == c.dev) return m;
+ return 0;
+ }
+ void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {
+ c.items.emplace_back();
+ Chunk::Item& it = c.items.back();
+ it.il = l.il;
+ it.mi = mi;
+ it.k = k;
+ it.e = e;
+ it.v = v;
+ const bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ if (in_ram && ram_pinned) { // (DMA'd from where it lies)
+ const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];
+ for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);
+ ++c.it_ram;
+ return;
+ }
+ if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)
+ it.ram = copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];
+ bool hit = false;
+ if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after
+ const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;
+ if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again
+ c.lc_il = l.il; // once that layer's items are all read - `after` below)
+ c.lc_used = 0;
+ c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);
+ c.lc_first = i;
+ std::fill(c.lc_of.begin(), c.lc_of.end(), -1);
+ }
+ const int j = c.lc_of[(size_t) e];
+ if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)
+ it.ram = c.lc + (size_t) j * stride;
+ it.after = c.lc_writer[(size_t) j];
+ hit = true;
+ } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)
+ const int jn = c.lc_used++;
+ c.lc_of[(size_t) e] = jn;
+ c.lc_writer[(size_t) jn] = (int64_t) i;
+ it.keep = c.lc + (size_t) jn * stride;
+ it.after = (int64_t) c.lc_first - 1;
+ }
+ }
+ it.stage = (int) (c.n_file++ % (uint64_t) c.S);
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);
+ it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);
+ sb += flow_region(r);
+ }
+ it.left.store(in_ram && copy_mode ? 3 * NH : 3, std::memory_order_relaxed);
+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
+ }
+ // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance
+ // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the
+ // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -
+ // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is
+ // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer
+ // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the
+ // rest of its picks are streamed in)
+ // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same
+ // profile and the same routing - and each reads only its part of every expert)
+ void flow_spec(Chunk& c, int k, int il, int T) {
+ static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;
+ Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ int* vs = member(l, mi).vs_pin;
+ c.spec_k = k;
+ c.spec_il = il;
+ c.spec_begin = c.base + c.items.size();
+ const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += pr[e];
+ std::vector<std::pair<double, int>> cand;
+ const double picks = (double) l.k_exp * T;
+ // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE
+ // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole
+ // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376
+ // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and
+ // 21 of 1568, the early reads 9-12% faster there)
+ static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;
+ const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;
+ for (int e : l.host_list) {
+ if (!early) break;
+ const double p = sum > 0 ? pr[e] / sum : 0.0;
+ const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));
+ if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {
+ return a.first > b.first;
+ });
+ if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));
+ std::vector<char> is((size_t) l.n_exp, 0);
+ int j = 0;
+ for (const auto& x : cand) {
+ vs[x.second] = l.n_slots + j++;
+ is[(size_t) x.second] = 1;
+ }
+ for (int e : l.host_list)
+ if (!is[(size_t) e]) vs[e] = l.n_slots + j++;
+ for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);
+ c.n_spec = (int) cand.size();
+ c.it_spec += cand.size();
+ }
+ // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones
+ void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {
+ const Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ const int* vs = member(l, mi).vs_pin;
+ for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {
+ const int v = fit(c, i).v;
+ if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;
+ }
+ for (int e : l.host_list) { // (index order: their virtual slots ascending)
+ const int v = vs[e];
+ if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
+ }
+ }
+ // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)
+ bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
+ const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
+ const size_t i = (size_t) (it - c.fl_layers.begin());
+ if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
+ if (k + 1 < (int) c.fl_T.size()) {
+ k2 = k + 1;
+ il2 = il;
+ return true;
+ }
+ if (i + 1 >= c.fl_layers.size()) return false;
+ k2 = 0;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (i + 1 < c.fl_layers.size()) {
+ k2 = k;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {
+ k2 = k + 1;
+ il2 = c.fl_layers[0];
+ return true;
+ }
+ return false;
+ }
+ // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each
+ // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now
+ bool flow_pump(Chunk& c, std::string& err) {
+ const size_t end = c.base + c.items.size();
+ IoQueue::Job jobs[6];
+ while (c.n_read < end) {
+ Chunk::Item& it = fit(c, c.n_read);
+ if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot
+ // written again once the layer before's items are all read (none still copies out of it)
+ bool wait = false;
+ if (it.keep == nullptr)
+ wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;
+ else {
+ while (c.lc_chk < c.n_read &&
+ (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))
+ ++c.lc_chk;
+ wait = c.lc_chk <= (size_t) it.after;
+ }
+ if (wait) break;
+ }
+ if (it.stage >= 0) {
+ const int64_t prev = c.stage_item[(size_t) it.stage];
+ if (prev >= 0) {
+ if ((size_t) prev >= c.n_dma) break;
+ const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);
+ if (q == cudaErrorNotReady) break;
+ CK(q);
+ }
+ c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)
+ int nj = 0;
+ for (int r = 0; r < 3; ++r) {
+ if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X) {
+ IoQueue::Job& j = jobs[nj++];
+ j = IoQueue::Job{};
+ j.len = nb;
+ j.dst = sb + (size_t) X * nb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ }
+ sb += flow_region(r);
+ continue;
+ }
+ IoQueue::Job& j = jobs[nj++];
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);
+ j.len = mx_read_len(l, r);
+ j.dst = sb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;
+ j.map = pk.at(l.mx_file[r], j.off);
+ j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;
+ sb += flow_region(r);
+ }
+ c.io->push(jobs, nj);
+ }
+ ++c.n_read;
+ }
+ const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;
+ struct Kick { // (every way out: what was queued is submitted)
+ cudaStream_t s;
+ const size_t &a, b;
+ ~Kick() {
+ if (a != b) kick(s);
+ }
+ } kq{c.cs, c.n_dma, dma0};
+ while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {
+ Chunk::Item& it = fit(c, c.n_dma);
+ if (it.left.load(std::memory_order_acquire) != 0) break;
+ if (it.bad.load(std::memory_order_acquire)) {
+ err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +
+ " from the model's files failed";
+ return false;
+ }
+ const size_t r = c.n_dma % R;
+ if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* dst = c.ring + r * c.ring_stride;
+ const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;
+ for (int p = 0; p < 2; ++p)
+ CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));
+ if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)
+ const int t2 = gg::expert_type(l.lay, 2);
+ const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);
+ CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,
+ cudaMemcpyHostToDevice, c.cs));
+ } else
+ CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));
+ CK(cudaEventRecord(c.ring_fill[r], c.cs));
+ if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));
+ c.stream_bytes += (double) unit(l);
+ ++c.n_dma;
+ }
+ return true;
+ }
+ // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM
+ // ones in shares between them (the device works while the reads run)
+ bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {
+ cudaStream_t s = d.st;
+ const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)
+ if (c.spec_k != k || c.spec_il != L0.il) {
+ err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";
+ return false;
+ }
+ flow_rest(c, k, L0.il, b);
+ const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
+ int k2 = 0, il2 = 0;
+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);
+ const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
+ int v_done = 0;
+ while (c.n_comp < le) {
+ // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what
+ // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so
+ // the sums come out the same on every run whatever the reads' timing
+ const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});
+ if (!flow_pump(c, err)) return false;
+ if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy
+ const double tw = now_ms();
+ kick(s);
+ if (c.n_read > c.n_dma) c.io->wait_any(500);
+ else {
+ const Chunk::Item& it = fit(c, c.n_read);
+ if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));
+ }
+ c.wait_ms += now_ms() - tw;
+ continue;
+ }
+ const int n = (int) (i1 - i0);
+ int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ ++c.n_batch;
+ // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)
+ for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];
+ bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];
+ if (bc[n] > bc[0]) {
+ CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));
+ CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));
+ if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))
+ return false;
+ }
+ for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));
+ kick(s);
+ c.n_comp = i1;
+ if (!flow_pump(c, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)
+ c.items.pop_front();
+ ++c.base;
+ }
+ return true;
+ }
+
+ // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)
+ // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order
+ bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,
+ const int* b, int i0, int i1, int T, std::string& err) {
+ const int D = g.dim, FF = l.ffn;
+ while (i0 < i1) {
+ int j = i0 + 1;
+ while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;
+ const int a = b[i0], nrows = b[j] - a;
+ int max_rows = 0;
+ for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);
+ if (nrows > 0) {
+ x3::SetArgs sa;
+ sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)
+ sa.stride = stride;
+ sa.lay = l.nm > 1 ? l.hlay : l.lay;
+ sa.n_exp = j - i0;
+ sa.bounds = d_bounds + i0;
+ sa.row0 = a;
+ sa.max_rows = max_rows;
+ sa.n_embd = D;
+ sa.n_ff = FF;
+ sa.limit = g.lim;
+ if (sa.lay.fmt == gg::kFmtMXFP4) {
+ gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ gg::set_down(sa, c.h16, FF, c.y, D, d.st);
+ } else {
+ x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ x3::set_down(sa, c.h16, FF, c.y, D, d.st);
+ }
+ if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {
+ std::vector<int> bb(b + i0, b + j + 1);
+ std::vector<uint16_t> hv((size_t) nrows * FF);
+ std::vector<float> yv((size_t) nrows * D);
+ CK(cudaStreamSynchronize(d.st));
+ CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&
+ wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};
+ std::fwrite(hdr, 8, 6, dump_y);
+ std::fwrite(bb.data(), 4, bb.size(), dump_y);
+ std::fwrite(hv.data(), 2, hv.size(), dump_y);
+ std::fwrite(yv.data(), 4, yv.size(), dump_y);
+ std::fflush(dump_y);
+ }
+ moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);
+ }
+ i0 = j;
+ }
+ (void) err;
+ return true;
+ }
+
+ bool load_dense(int di, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ size_t* acct = &d.weights;
+ for (Layer& l0 : L)
+ for (int mi = 0; mi < l0.nm; ++mi) {
+ Layer& l = member(l0, mi);
+ if (l.dev != di) continue;
+ // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,
+ // of the shared expert's FFN; the rest whole)
+ const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);
+ const std::string& P = l.name;
+ const std::string A = P + "attn.";
+ bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&
+ mat(d, A + "wkv", l.wkv, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&
+ f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&
+ f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&
+ f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&
+ f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&
+ f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&
+ f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&
+ f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&
+ f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&
+ f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&
+ (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||
+ f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&
+ mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);
+ if (!ok) return false;
+ l.sink += (size_t) l.ah0;
+ for (int gi = 0; gi < l.ogn; ++gi)
+ if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;
+ if (l.kv_src) {
+ const std::string C = A + "compressor.";
+ if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;
+ if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;
+ if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ if (l.ix_src) {
+ const std::string I = A + "indexer.";
+ if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))
+ return false;
+ if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))
+ return false;
+ }
+ if (l.eg >= 0) {
+ if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;
+ if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already
+ if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;
+ } else {
+ float *qw = nullptr, *kw = nullptr;
+ if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))
+ return false;
+ std::vector<float> a((size_t) g.hc * g.dim), b(a.size());
+ CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));
+ for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product
+ CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));
+ l.eg_w = qw;
+ }
+ }
+ if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;
+ if (g.n_mtp > 0 && !l.mtp) {
+ if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;
+ if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))
+ return false;
+ }
+ if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns
+ const int hn = head_nm();
+ if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))
+ return false;
+ if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))
+ return false;
+ if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;
+ if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head
+ const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");
+ const int R = g.d_rank;
+ head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)
+ // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same
+ // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)
+ if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {
+ std::vector<uint16_t> rows((size_t) d.mk_vn * R);
+ if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {
+ err = "the markov head: read failed";
+ return false;
+ }
+ if (th->dtype == "BF16")
+ for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));
+ if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||
+ !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))
+ return false;
+ CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));
+ }
+ }
+ }
+ if (di == head_dev) {
+ head = d.head;
+ norm_w = d.norm_w;
+ d_norm = d.d_norm;
+ if (g.n_mtp > 0) {
+ const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";
+ if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))
+ return false;
+ const Tensor* te = pk.get(Z + "markov_head.embed.weight");
+ const Tensor* th = pk.get(Z + "markov_head.head.weight");
+ if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {
+ err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";
+ return false;
+ }
+ if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
+ !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))
+ return false;
+ }
+ }
+ if (g.n_mtp > 0 && di == d_dev) {
+ if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||
+ !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))
+ return false;
+ }
+ return true;
+ }
+
+ void set_vm(Layer& l, int e, int slot) {
+ uint8_t* b = l.vram_base + (size_t) slot * l.blob;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m = x3::Mat{};
+ m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)
+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
+ m.raw = b + l.lay.trellis[r];
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ }
+ return;
+ }
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);
+ m.suh = (const uint16_t*) (b + l.lay.suh[r]);
+ m.svh = (const uint16_t*) (b + l.lay.svh[r]);
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ m.K = l.lay.K[r];
+ m.cb = 2;
+ m.ld = 0;
+ }
+ }
+ // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)
+ void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = blob + l.lay.trellis[r];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+ void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ uint8_t* tr = blob + l.lay.trellis[r];
+ for (int X = 0; X < NH; ++X)
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);
+ std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],
+ (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back
+ void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,
+ blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);
+ std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);
+ std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);
+ }
+ void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,
+ src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);
+ if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);
+ std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);
+ if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);
+ }
+
+ bool place_experts(std::string& err) {
+ // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)
+ double room = 1e300, kept = 0;
+ for (const Dev& d : devs)
+ if (commit_charged(d)) room = commit_room();
+ const double room0 = room;
+ // per group: what its tightest member has left (each member holds its part of every VRAM expert of the
+ // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round
+ // the members layer by layer, so each member's total comes to about the mean's)
+ auto ucount = [](const Layer& l) {
+ double s = 0;
+ for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));
+ return s / l.nm;
+ };
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ double budget = 1e300;
+ bool charged = false;
+ for (int m : groups[gi].mem) {
+ cudaSetDevice(devs[(size_t) m].id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *
+ o.vram_experts);
+ charged = charged || commit_charged(devs[(size_t) m]);
+ }
+ if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)
+ double all = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;
+ const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);
+ kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;
+ budget = capped;
+ room = std::max(0.0, room - std::min(budget, all) * nm);
+ }
+ double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)
+ for (Layer& l : L)
+ if (l.grp == (int) gi && l.mtp) {
+ l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));
+ budget_t -= (double) l.n_vram * ucount(l);
+ }
+ double want = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi && !l.mtp) want += (double) ucount(l) * l.n_exp;
+ const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
+ for (Layer& l : L) {
+ if (l.grp != (int) gi || l.mtp) continue;
+ l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
+ // never more than fits: the floor of each layer's share can still round up the total
+ while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
+ }
+ double used = 0;
+ for (Layer& l : L)
+ if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
+ // what the floors left: whole experts, layer by layer
+ for (Layer& l : L) {
+ if (l.grp != (int) gi) continue;
+ while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
+ ++l.n_vram;
+ used += (double) ucount(l);
+ }
+ }
+ }
+ if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it
+ const os::Commit c = os::commit();
+ const double gib = (double) (1ull << 30);
+ const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;
+ commit_kept = kept;
+ pagefile_hint = std::ceil((pf + kept) / gib);
+ std::fprintf(stderr,
+ "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "
+ "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "
+ "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "
+ "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",
+ kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);
+ } else if (room0 < 1e300 && o.verbose) {
+ std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "
+ "programs)\n", room0 / 1e9, commit_margin() / 1e9);
+ }
+ for (Layer& l : L) {
+ // the hottest experts of the profile first, then by index
+ std::vector<int> order((size_t) l.n_exp);
+ std::iota(order.begin(), order.end(), 0);
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });
+ l.slot.assign((size_t) l.n_exp, -1);
+ l.host.assign((size_t) l.n_exp, nullptr);
+ // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps
+ l.n_slots = std::min(l.n_vram, l.n_exp);
+ if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)
+ l.n_vram = l.n_slots - 1;
+ l.spare_vram = l.n_slots - 1;
+ } else {
+ l.n_vram = l.n_slots;
+ l.spare_vram = -1;
+ }
+ for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;
+ for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {
+ Layer& x = member(l, m);
+ Dev& d = devs[(size_t) x.dev];
+ cudaSetDevice(d.id);
+ void* v = nullptr;
+ if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {
+ err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +
+ " experts on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ d.experts += (size_t) l.n_slots * unit(x);
+ x.vram_base = (uint8_t*) v;
+ }
+ l.heat.assign((size_t) l.n_exp, 0.0f);
+ { // the profile as the starting heat: its counts scaled to ~512 hits a layer
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ const double sc = sum > 0 ? 512.0 / sum : 0.0;
+ for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);
+ }
+ const int n_host = l.n_exp - l.n_vram;
+ { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;
+ l.hl.k[r] = k;
+ l.hl.n[r] = n;
+ l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;
+ off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);
+ l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;
+ off += align256((size_t) k * 2);
+ l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;
+ off += align256((size_t) n * 2);
+ }
+ l.hl.bytes = off;
+ }
+ l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});
+ if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);
+ } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (pk.map(l.region_file[(size_t) e]) == nullptr) {
+ err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";
+ return false;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) {
+ l.hh[(size_t) e][0] = file_part(l, e);
+ l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ } else if (n_host > 0) {
+ const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)
+ const size_t bytes = (size_t) n_parts * l.hl.bytes;
+ for (int X = 0; X < NH; ++X) {
+ void* m = node_map(bytes, X, NH);
+ if (m == nullptr) {
+ err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +
+ " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";
+ return false;
+ }
+ host_maps.emplace_back(m, bytes);
+ int hi = 0;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;
+ if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ l.vm.assign((size_t) l.n_exp, {});
+ if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);
+ if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0)
+ tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));
+ }
+ }
+ return true;
+ }
+
+ // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the
+ // file: the mapping reads them on first use)
+ bool load_experts_mx(std::string& err) {
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ size_t maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12; ++t)
+ th.emplace_back([&] {
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);
+ for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,
+ unit(l), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!ok) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the
+ // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them
+ struct Warm {
+ int layer, e;
+ double share;
+ };
+ std::vector<Warm> warm;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0) {
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);
+ } else {
+ warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});
+ }
+ }
+ }
+ std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });
+ if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left
+ std::vector<std::pair<int, int>> hot;
+ for (const Warm& w : warm) hot.push_back({w.layer, w.e});
+ if (!copy_experts(hot, err)) return false;
+ if (copy_mode) {
+ warm.erase(std::remove_if(warm.begin(), warm.end(),
+ [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),
+ warm.end());
+ const uint64_t av = ram_avail();
+ if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;
+ }
+ }
+ if (ram_tier && warm_bytes > 0 && !warm.empty()) {
+ // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for
+ // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not
+ // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on
+ // Windows - the engine's own pinned buffers come after this
+ const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));
+ while (ram_slots.size() < want + 1) {
+ const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());
+ uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);
+ if (c == nullptr) break;
+ ram_chunks.push_back(c);
+ ram_chunk_bytes.push_back(n * maxb);
+ for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);
+ }
+ ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;
+ for (Layer& l : L)
+ if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);
+ std::atomic<size_t> ri{0};
+ std::atomic<bool> rfail{false};
+ std::vector<std::thread> rt;
+ for (int t = 0; t < 12 && ram_held > 0; ++t)
+ rt.emplace_back([&] {
+ for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {
+ const Layer& l = L[(size_t) warm[i].layer];
+ if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ }
+ });
+ for (auto& t : rt) t.join();
+ if (rfail.load()) {
+ err = "reading the RAM tier's experts from the model's files failed";
+ return false;
+ }
+ for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;
+ if (ram_held > 0) ram_free.push_back(ram_held);
+ if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "
+ "read from the model's files (%.1f s)\n",
+ ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ size_t nw = 0;
+ for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)
+ b += (double) L[(size_t) warm[nw].layer].blob;
+ std::atomic<size_t> wi{0};
+ std::vector<std::thread> wt;
+ for (int t = 0; t < 12 && nw > 0; ++t)
+ wt.emplace_back([&] {
+ std::vector<uint8_t> buf(maxb);
+ for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());
+ });
+ for (auto& t : wt) t.join();
+ if (o.verbose && nw > 0)
+ std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,
+ nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
+ return true;
+ }
+
+ // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU
+ // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the
+ // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:
+ // none - read in place: the page cache follows a conversation's experts where a static set (the profile's
+ // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the
+ // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file
+ // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what
+ // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place
+ bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {
+ if (o.mmap_experts > 0 || hot.empty()) return true;
+ const double t0 = now_ms();
+ auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };
+ double all = 0;
+ {
+ std::vector<char> seen(L.size(), 0);
+ for (const auto& h : hot) {
+ const Layer& l = L[(size_t) h.first];
+ all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));
+ }
+ }
+ const uint64_t avail = ram_avail();
+ // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),
+ commit_room());
+ // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
+ // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
+ // on 64 GB and on 32 GB of RAM)
+ double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;
+ if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;
+ if (o.mmap_experts == 0 && all > can)
+ std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "
+ "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "
+ "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
+ if (budget <= 0) {
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "
+ "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",
+ all / 1e9, can / 1e9, kRamMargin / 1e9);
+ return true;
+ }
+ // NUMA parts: whole rows of each matrix, so any row count the nodes divide
+ int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));
+ if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)
+ nh = std::atoi(v) >= 2 ? 2 : 1;
+ if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;
+ NH = nh;
+ cp_lay.assign(L.size(), {0, 0, 0, 0});
+ for (const Layer& l : L) {
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ cp_lay[(size_t) l.il][(size_t) r] = off;
+ off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));
+ }
+ cp_lay[(size_t) l.il][3] = off;
+ }
+ // the hottest that fit the budget (a layer's first one brings its spare)
+ std::vector<int> n_copy(L.size(), 0);
+ double used = 0;
+ size_t take = 0;
+ for (; take < hot.size(); ++take) {
+ const Layer& l = L[(size_t) hot[take].first];
+ const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));
+ if (used + add > budget) break;
+ used += add;
+ ++n_copy[(size_t) l.il];
+ }
+ // each layer's slots: one block per node of (copies + spare) parts
+ std::vector<int> first(L.size(), -1);
+ copy_spare.assign(L.size(), -1);
+ for (Layer& l : L) {
+ l.ram.assign((size_t) l.n_exp, -1);
+ const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);
+ if (ns == 0) continue;
+ const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
+ std::array<uint8_t*, 2> base = {nullptr, nullptr};
+ bool ok = true;
+ for (int X = 0; X < NH && ok; ++X) {
+ base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
+ ok = base[(size_t) X] != nullptr;
+ if (ok) {
+ ram_chunks.push_back(base[(size_t) X]);
+ ram_chunk_bytes.push_back(bytes);
+ }
+ }
+ if (!ok) {
+ std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,
+ ns, (double) bytes * NH / 1e9);
+ n_copy[(size_t) l.il] = 0;
+ continue;
+ }
+ first[(size_t) l.il] = (int) ram_part.size();
+ for (int i = 0; i < ns; ++i)
+ ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});
+ if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;
+ }
+ struct Job {
+ int layer, e, slot;
+ };
+ std::vector<Job> jobs;
+ {
+ std::vector<int> next = first;
+ for (size_t i = 0; i < take; ++i) {
+ const int il = hot[i].first;
+ if (n_copy[(size_t) il] == 0) continue;
+ jobs.push_back({il, hot[i].second, next[(size_t) il]++});
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> ji{0};
+ std::atomic<bool> bad{false};
+ std::atomic<int64_t> done_b{0};
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12 && !jobs.empty(); ++t)
+ th.emplace_back([&] {
+ for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {
+ const Job& j = jobs[i];
+ const Layer& l = L[(size_t) j.layer];
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,
+ ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))
+ bad.store(true);
+ }
+ const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && i % 2048 == 0)
+ std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ });
+ for (auto& t : th) t.join();
+ if (bad.load()) {
+ err = "copying the CPU lane's experts from the model's files failed";
+ return false;
+ }
+ for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;
+ copy_mode = true;
+ ram_held = (int) jobs.size();
+ double cb = 0;
+ for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;
+ std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",
+ jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,
+ jobs.size() < hot.size() ? "; the others read in place" : "");
+ return true;
+ }
+
+ bool load_experts(std::string& err) {
+ if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);
+ // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place
+ // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their
+ // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are
+ // dropped from it again
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ for (const Layer& l : L)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ uint64_t warm = 0;
+ if (file_experts) {
+ std::vector<std::pair<double, Job>> cand;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ for (const auto& c : cand) {
+ const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];
+ if (warm + b > warm_bytes) break;
+ warm += b;
+ jobs.push_back(c.second);
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])
+ return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];
+ return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ const int nth = 12;
+ size_t maxr = 0, maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);
+ }
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < nth; ++t)
+ th.emplace_back([&] {
+ std::vector<uint8_t> region(maxr);
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ int cur_dev = -1;
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],
+ region.data())) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ if (l.slot[(size_t) e] >= 0) {
+ repack(l, region.data(), stage);
+ bool up = true;
+ for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ const Dev& d = devs[(size_t) x.dev];
+ if (cur_dev != d.id) {
+ cudaSetDevice(d.id);
+ cur_dev = d.id;
+ }
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,
+ unit(x), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!up) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);
+ failed.store(true);
+ break;
+ }
+ if (file_experts)
+ os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);
+ } else if (!file_experts) {
+ for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +
+ (int64_t) l.region_bytes[(size_t) e];
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),
+ db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ if (file_experts)
+ std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",
+ (now_ms() - t0) / 1000.0, warm / 1e9);
+ return true;
+ }
+
+ std::string describe() const {
+ std::ostringstream s;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ const Dev& d = devs[di];
+ int a = -1, b = -1, ev = 0, et = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ if (a < 0) a = l.il;
+ b = l.il;
+ ev += l.n_vram;
+ et += l.n_exp;
+ }
+ size_t fr = 0, tot = 0;
+ cudaSetDevice(d.id);
+ cudaMemGetInfo(&fr, &tot);
+ const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();
+ char tp[64] = "";
+ if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);
+ char line[384];
+ std::snprintf(line, sizeof line,
+ "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "
+ "%.2f GB, %.0f MiB free\n",
+ d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,
+ d.caches / 1e9, fr / 1048576.0);
+ s << line;
+ }
+ int nh = 0;
+ size_t hb_ = 0;
+ for (const Layer& l : L) {
+ nh += l.n_exp - l.n_vram;
+ hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;
+ }
+ char line[200];
+ if (copy_mode)
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "
+ "parts), the rest in place in the model's files; %d threads\n",
+ nh, hb_ / 1e9, ram_held, NH, cpu_threads());
+ else if (file_experts)
+ std::snprintf(line, sizeof line,
+ "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",
+ nh, hb_ / 1e9, cpu_threads());
+ else
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,
+ hb_ / 1e9, NH, cpu_threads());
+ s << line;
+ return s.str();
+ }
+
+ int cpu_threads() const {
+ int n = 0;
+ for (const auto& pl : pools) n += pl->size() - 1;
+ return n + 1;
+ }
+ // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over
+ // every node's cores (the pages lie where the cache put them)
+ void make_pools() {
+ std::vector<std::vector<int>> cs = cores;
+ if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)
+ std::vector<int> all = cs[0];
+ cs.assign((size_t) NH, {});
+ for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);
+ }
+ for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {
+ std::vector<int> c = cs[(size_t) X];
+ if (file_experts && NH == 1)
+ for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());
+ if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)
+ int nw = (int) c.size();
+ if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));
+ pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)
+ }
+ (void) gg::mx_tier(); // (measured here, not on the first token)
+ if (o.verbose) {
+ int nthr = 0;
+ for (auto& p : pools) nthr += p->size();
+ std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),
+ gg::mx_tier_report());
+ }
+ }
+
+ void reset() {
+ pos = 0;
+ toks.clear();
+ ctoks.clear();
+ ck[0].pos = ck[1].pos = -1;
+ for (Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ Layer& l = member(l0, m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);
+ if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
+ // state is indexed by position, and what lies past a restored position is written again)
+ void checkpoint(int slot) {
+ Check& c = ck[slot & 1];
+ c.pos = pos;
+ c.win.resize(L.size());
+ c.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ c.win[i].resize((size_t) g.win * win_rb());
+ cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);
+ if (l.c_kv) {
+ c.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyDeviceToHost, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
+ int best_check(int64_t limit) const {
+ int b = -1;
+ for (int i = 0; i < 2; ++i)
+ if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
+ return b;
+ }
+ int64_t restorable(int64_t limit) const {
+ const int b = best_check(limit);
+ return b < 0 ? -1 : ck[b].pos;
+ }
+
+ bool restore(int64_t limit) {
+ const int b = best_check(limit);
+ if (b < 0) return false;
+ const Check& c = ck[b];
+ for (size_t i = 0; i < L.size(); ++i)
+ for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)
+ Layer& l = member(L[i], m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);
+ if (l.c_kv) {
+ cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyHostToDevice, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ pos = c.pos;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
+ if (o2.pos > pos) o2.pos = -1;
+ return true;
+ }
+
+ size_t bounce_bytes() const {
+ return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+ size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }
+ size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }
+ size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }
+ static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need
+
+ // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the
+ // host; a prompt's pictures as their spans' rows (set_images)
+ VisionTower vt;
+ VisionGeo vg;
+ int img_tok = -1;
+ std::string vision_err; // why the tower did not load (pictures off), or empty
+ std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]
+ std::vector<Image> imgs; // the pictures of the prompt being fed, by start
+ bool load_vision(std::string& err) {
+ vg.layers = (int) pk.num("vision.num_hidden_layers", 0);
+ const int tok = (int) pk.num("image_token_id", -1);
+ if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none
+ vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);
+ vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);
+ vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);
+ vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;
+ const char* mark[3] = {"image_start", "image_newline", "image_end"};
+ for (int i = 0; i < 3; ++i) {
+ const Tensor* t = pk.get(mark[i]);
+ img_mark[i].resize((size_t) g.dim);
+ if (t == nullptr || t->dtype != "BF16" || t->bytes != (uint64_t) g.dim * 2 || !pk.read(*t, img_mark[i].data())) {
+ err = std::string(mark[i]) + ": not a BF16 row of the model's width";
+ return false;
+ }
+ }
+ const double t0 = now_ms();
+ Dev& d = devs[0];
+ auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {
+ const Tensor* t = pk.get(n);
+ if (t == nullptr || t->dtype != "BF16") return false;
+ shape = t->shape;
+ return dst == nullptr || pk.read(*t, dst);
+ };
+ if (!vt.load(vg, d.id, read, err)) return false;
+ img_tok = tok;
+ CK(cudaSetDevice(d.id));
+ CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",
+ vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {
+ if (img_tok < 0) {
+ err = "this model has no vision tower (or it is off)";
+ return false;
+ }
+ std::vector<uint16_t> rows;
+ int nh = 0, nw = 0;
+ if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;
+ const size_t D = (size_t) g.dim;
+ span.resize(((size_t) nh * (nw + 1) + 2) * D);
+ uint16_t* o = span.data();
+ auto put = [&](const uint16_t* r) {
+ std::memcpy(o, r, D * 2);
+ o += D;
+ };
+ put(img_mark[0].data());
+ for (int i = 0; i < nh; ++i) {
+ for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);
+ put(img_mark[1].data());
+ }
+ put(img_mark[2].data());
+ return true;
+ }
+ // the row of a picture's span at position p (null: p is in none)
+ const uint16_t* image_row(int64_t p) const {
+ if (imgs.empty()) return nullptr;
+ auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });
+ if (it == imgs.begin()) return nullptr;
+ const Image& im = *(it - 1);
+ const int64_t r = p - im.start;
+ if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;
+ return im.rows->data() + (size_t) r * g.dim;
+ }
+ // token tk's embedding row at position p: inside a picture's span, the picture's
+ const uint16_t* emb_row(int64_t p, int tk) const {
+ const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;
+ return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;
+ }
+ bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }
+ // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)
+ int32_t ctok(int64_t p, int tk) const {
+ if (tk == img_tok && image_row(p) != nullptr) return -1;
+ return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;
+ }
+ // a chunk's image rows (route()'s bits); false: none
+ bool vl_bits(int64_t p0, int T, VlBits& b) const {
+ if (imgs.empty()) return false;
+ std::memset(&b, 0, sizeof b);
+ bool any = false;
+ for (int t = 0; t < T && t < kVlRows; ++t)
+ if (in_image(p0 + t)) {
+ b.w[t >> 5] |= 1u << (t & 31);
+ any = true;
+ }
+ return any;
+ }
+
+ // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]
+ // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a
+ // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,
+ // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.
+ void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;
+ const size_t per = (size_t) cols * (hd + hd / 32);
+ b.clear();
+ b.reads.reserve((size_t) T * cols);
+ const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
+ const Tensor* tw = pk.get(P + "weight");
+ const Tensor* ts = pk.get(P + "scale");
+ const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)
+ const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
+ int64_t hits = 0;
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (int t = 0; t < T; ++t) {
+ const int64_t p = p0 + t;
+ uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;
+ uint8_t* scales = rows + (size_t) cols * hd;
+ if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)
+ std::memset(rows, 0, per);
+ continue;
+ }
+ int64_t prod[8];
+ bool blocked = false;
+ for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)
+ blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;
+ const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];
+ prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];
+ }
+ int64_t rolling = prod[0];
+ for (int n = 1; n < pk.eg_ngram; ++n) {
+ rolling ^= prod[n];
+ for (int h = 0; h < pk.eg_heads; ++h) {
+ const int c = (n - 1) * pk.eg_heads + h;
+ const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];
+ uint8_t* rd = rows + (size_t) c * hd;
+ uint8_t* sd = scales + (size_t) c * (hd / 32);
+ const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;
+ if (const uint8_t* hit = eg_cache.find(key)) {
+ std::memcpy(rd, hit, (size_t) hd);
+ std::memcpy(sd, hit + hd, (size_t) hd / 32);
+ ++hits;
+ continue;
+ }
+ const auto fi = first.find(key);
+ if (fi != first.end()) {
+ b.dups.push_back({fi->second, {rd, sd}});
+ continue;
+ }
+ first.emplace(key, (int) b.reads.size());
+ EgRead r;
+ r.file = tw->file;
+ r.dst = rd;
+ r.scales = sd;
+ r.key = key;
+ if (mx8) {
+ r.mx8 = true;
+ r.n = (uint32_t) (hd / 32 * 33);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else if (q6k) { // (into the row's code bytes; its scale bytes unused)
+ r.n = (uint32_t) (hd / 256 * 210);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else {
+ r.n = (uint32_t) hd;
+ r.off = tw->off + (uint64_t) id * hd;
+ r.file2 = ts->file;
+ r.n2 = (uint32_t) (hd / 32);
+ r.off2 = ts->off + (uint64_t) id * (hd / 32);
+ }
+ b.reads.push_back(r);
+ }
+ }
+ }
+ }
+ eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();
+ eg_hits_n += hits + (int64_t) b.dups.size();
+ eg_reads_n += (int64_t) b.reads.size();
+ eg_io.submit(b);
+ }
+ // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)
+ void eg_finish(EgBatch& b) {
+ if (b.post.load(std::memory_order_acquire) == 2) return;
+ const int64_t t0 = now_ns();
+ eg_io.wait(b);
+ int want = 0;
+ if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {
+ const int hd = pk.eg_hd;
+ for (const auto& d : b.dups) {
+ const EgRead& r = b.reads[(size_t) d.first];
+ std::memcpy(d.second.first, r.dst, (size_t) hd);
+ std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
+ }
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (size_t i = 0; i < b.reads.size(); ++i) {
+ const EgRead& r = b.reads[i];
+ if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)
+ std::memset(r.dst, 0, (size_t) hd);
+ std::memset(r.scales, 0, (size_t) hd / 32);
+ continue;
+ }
+ if (uint8_t* c = eg_cache.put(r.key)) {
+ std::memcpy(c, r.dst, (size_t) hd);
+ std::memcpy(c + hd, r.scales, (size_t) hd / 32);
+ }
+ }
+ }
+ eg_io_ns += b.t1.load() - b.t0;
+ b.live = false;
+ b.post.store(2, std::memory_order_release);
+ } else {
+ while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();
+ }
+ eg_wait_ns += now_ns() - t0;
+ }
+ void eg_stats() {
+ st.eg_rows = eg_rows_n.load();
+ st.eg_hits = eg_hits_n.load();
+ st.eg_reads = eg_reads_n.load();
+ st.eg_io_ms = (double) eg_io_ns.load() / 1e6;
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ st.eg_cached = (int64_t) eg_cache.size();
+ st.eg_cache_cap = (int64_t) eg_cache.capacity();
+ }
+ // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)
+ void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {
+ EgBatch b;
+ eg_issue(p0, T, only, dst, b);
+ eg_finish(b);
+ }
+
+ // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its
+ // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,
+ // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's
+ // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down
+ // pieces; the down bands). yd[slot] gets each row's expert output.
+ struct Ent {
+ int e = 0, n = 0, slot0 = 0;
+ int tok[kc::kExl3CpuMaxN] = {};
+ };
+ void run_nodes(int n, const std::function<void(int)>* f) {
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);
+ pools[0]->help();
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();
+ }
+
+ // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each
+ // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),
+ // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits
+ // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds
+ // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever
+ // number of entries take it, and each job its band of h straight into the down's int8 input.
+ // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)
+ void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);
+ constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);
+ const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);
+ const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)
+ const float lim = g.lim;
+ static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");
+ { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at
+ // once, so the SSD reads them with large requests side by side while the first entries compute
+ // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost
+ // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (l.ram.empty() || l.ram[(size_t) en.e] < 0)
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ os::prefetch_cold(want);
+ }
+ // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token
+ // index past the window (none on the decode or chunk paths) gets a row of its own after them
+ uint8_t* xq = (uint8_t*) c_xh.data();
+ uint8_t* hq = xq + (size_t) kWin * xa;
+ uint8_t* xq_extra = hq + (size_t) kSlots * ha;
+ auto x_act = [&](const Ent& en, int i) -> const void* {
+ const int t = en.tok[i];
+ return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));
+ }
+ }
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];
+ float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ gp[i] = yg[i];
+ up[i] = yu[i];
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's)
+ gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);
+ gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);
+ for (int i = 0; i < en.n; ++i) {
+ float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;
+ for (int r = r0; r < r1; ++r) {
+ float gv = yg[i][r - r0], uv = yu[i][r - r0];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);
+ }
+ };
+ run_nodes(ne * GB, f);
+ if (!band_q)
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float* yp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = hq + (size_t) (en.slot0 + i) * ha;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;
+ }
+ gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);
+ };
+ run_nodes(ne * DB, f);
+ }
+ // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input
+ // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same
+ void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ constexpr int RB = 64; // rows a job
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ gg::IqCpu fm;
+ std::string ferr;
+ if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||
+ fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {
+ static std::atomic<bool> said{false};
+ if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());
+ for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);
+ return;
+ }
+ const float lim = g.lim;
+ { // (os::prefetch, as above)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (l.ram.empty() || l.ram[(size_t) en.e] < 0)
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ os::prefetch(want);
+ }
+ // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of
+ // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or
+ // chunk paths), then (x no longer needed) the slot's h
+ uint8_t* aq = (uint8_t*) c_xh.data();
+ auto x_act = [&](const Ent& en, int i) -> uint8_t* {
+ const int t = en.tok[i];
+ return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
+ }
+ }
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float* hp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any
+ // block size of the type)
+ gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,
+ r0 - X * FFh, r1 - X * FFh, lim);
+ };
+ run_nodes(ne * GB, f);
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
+ aq + (size_t) (kWin + en.slot0 + i) * E * 4);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* hp[gg::kMxCpuMaxN];
+ float* yp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)
+ }
+ gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
+ };
+ run_nodes(ne * DB, f);
+ }
+
+ void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
+ const int* ord = nullptr) {
+ if (ldx == 0) ldx = (size_t) g.dim;
+ if (l.lay.fmt == gg::kFmtMXFP4) return cpu_experts_mx(l, x, ents, ldx, ord);
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
+ const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
+ const int ne = (int) ents.size();
+ const float lim = g.lim;
+ auto part = [&](int e, int X, int r) {
+ const uint8_t* b = l.hh[(size_t) e][(size_t) X];
+ kc::Exl3Part p;
+ p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);
+ p.suh = (const uint16_t*) (b + l.hl.suh[r]);
+ p.svh = (const uint16_t*) (b + l.hl.svh[r]);
+ p.k = l.hl.k[r];
+ p.n = l.hl.n[r];
+ p.K = l.lay.K[r];
+ return p;
+ };
+ auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };
+ auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };
+ auto acc = [&](int X, int sl, int r, int q) {
+ return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;
+ };
+ auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };
+ auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };
+ auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / (2 * GQ))];
+ const int r = (j / GQ) % 2, q = j % GQ;
+ const kc::Exl3Part p = part(en.e, X, r);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ const float* xt = x + (size_t) en.tok[i] * ldx;
+ for (int b = 2 * q; b < 2 * q + 2; ++b)
+ xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);
+ xp[i] = xh(X, sl, r);
+ ap[i] = acc(X, sl, r, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);
+ };
+ run_nodes(ne * 2 * GQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / GB)];
+ const int b = j % GB;
+ const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float y[2][128], a[128], hb[128];
+ for (int r = 0; r < 2; ++r) {
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < GQ; ++q) {
+ const float* aq = acc(X, sl, r, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];
+ kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);
+ }
+ for (int c = 0; c < 128; ++c) {
+ float gv = y[0][c], uv = y[1][c];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ hb[c] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ const int gb = X * GB + b;
+ xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);
+ }
+ };
+ run_nodes(ne * GB, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DQ)];
+ const int q = j % DQ;
+ const kc::Exl3Part p = part(en.e, X, 2);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ xp[i] = xhd(en.slot0 + i);
+ ap[i] = accd(X, en.slot0 + i, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);
+ };
+ run_nodes(ne * DQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DB)];
+ const int b = j % DB;
+ const kc::Exl3Part pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float a[128];
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < DQ; ++q) {
+ const float* aq = accd(X, sl, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];
+ kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);
+ }
+ };
+ run_nodes(ne * DB, f);
+ }
+
+ // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it
+ // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs
+ // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's
+ // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:
+ // the CPU lane reads it from the file again.
+ void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot
+ for (;;) {
+ RamSwap rs;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });
+ if (quit.load()) return;
+ rs = ram_queue.front();
+ ram_queue.pop_front();
+ }
+ const Layer& l = L[(size_t) rs.layer];
+ if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {
+ for (int r = 0; r < 3; ++r) // (one copy of it in RAM)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ } else {
+ rs.in = -1; // (failed: the slot goes back)
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_ready.push_back(rs);
+ }
+ }
+
+ void cache_loop(SwapBufs& B) {
+ for (;;) {
+ Swap sw;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });
+ if (quit.load()) return;
+ sw = sw_queue.front();
+ sw_queue.pop_front();
+ }
+ Layer& l = L[(size_t) sw.layer];
+ if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)
+ swap_in_mx(l, sw, B);
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ continue;
+ }
+ parts_to_blob(l, l.hh[(size_t) sw.in], B.a);
+ if (l.nm == 1) {
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t cs = B.st[(size_t) l.dev];
+ cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);
+ if (!file_experts)
+ cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ } else { // a group: each member's part in, each member's part out
+ for (int m = 0; m < l.nm; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = B.st[(size_t) x.dev];
+ blob_to_half(l, B.a, m, B.h);
+ cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);
+ cudaStreamSynchronize(cs);
+ if (file_experts) continue;
+ cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ half_to_blob(l, B.h, m, B.b);
+ }
+ }
+ if (!file_experts)
+ for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);
+ if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte
+ std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);
+ bool ok = true;
+ for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)
+ const int e = pass == 0 ? sw.in : sw.out;
+ reg.resize(l.region_bytes[(size_t) e]);
+ pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());
+ if (pass == 0) {
+ repack(l, reg.data(), blob.data());
+ std::vector<uint8_t> dev(l.blob);
+ if (l.nm == 1)
+ cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);
+ else
+ for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)
+ std::vector<uint8_t> h(member(l, m).hblob);
+ cudaSetDevice(devs[(size_t) member(l, m).dev].id);
+ cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,
+ cudaMemcpyDeviceToHost);
+ half_to_blob(l, h.data(), m, dev.data());
+ }
+ for (int r = 0; r < 3; ++r)
+ ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&
+ std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&
+ std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;
+ } else {
+ for (int X = 0; X < NH; ++X) {
+ repack_part(l, reg.data(), X, part.data());
+ for (int r = 0; r < 3; ++r) {
+ const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];
+ ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&
+ std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&
+ std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;
+ }
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,
+ ok ? "both match the model file" : "MISMATCH");
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ }
+ }
+
+ // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);
+ // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)
+ // from the layer's spare RAM slot, which it is copied into from VRAM here
+ void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {
+ uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)
+ const std::vector<cudaStream_t>& copy_st = B.st;
+ bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till
+ // this swap's switch)
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);
+ ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ }
+ if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot
+ bool got = ok;
+ for (int m = 0; m < l.nm && got; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),
+ unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);
+ }
+ if (got) blob_to_ram(l, sw_b, sw.ram);
+ else sw.ram = -1; // (back to the file's pages then)
+ }
+ if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte
+ std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),
+ cudaMemcpyDeviceToHost);
+ if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());
+ }
+ ok = ok && read_mx_file(l, sw.in, file.data());
+ for (int r = 0; r < 3 && ok; ++r)
+ ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ bool ok_out = true;
+ if (sw.ram >= 0) {
+ ok_out = read_mx_file(l, sw.out, file.data());
+ ram_to_blob(l, sw.ram, dev.data());
+ for (int r = 0; r < 3 && ok_out; ++r)
+ ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,
+ sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");
+ } else if (!ok) {
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);
+ }
+ }
+
+ void cache_tick(int T) {
+ if (!o.expert_cache || cache_ths.empty()) return;
+ std::vector<Swap> ready;
+ std::vector<RamSwap> rready;
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ready.swap(sw_ready);
+ rready.swap(ram_ready);
+ }
+ for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)
+ --ram_inflight;
+ Layer& l = L[(size_t) rs.layer];
+ if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {
+ ram_free.push_back(rs.slot);
+ continue;
+ }
+ l.ram[(size_t) rs.in] = rs.slot;
+ if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {
+ ram_free.push_back(l.ram[(size_t) rs.out]);
+ l.ram[(size_t) rs.out] = -1;
+ }
+ ++st.ram_moves;
+ }
+ for (const Swap& sw : ready) {
+ Layer& l = L[(size_t) sw.layer];
+ const int old_slot = l.slot[(size_t) sw.out];
+ const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];
+ if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)
+ const int s_in = l.ram[(size_t) sw.in];
+ l.ram[(size_t) sw.in] = -1;
+ if (sw.ram >= 0) {
+ l.ram[(size_t) sw.out] = sw.ram;
+ copy_spare[(size_t) l.il] = s_in;
+ } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)
+ --ram_held;
+ if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;
+ }
+ } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)
+ ram_free.push_back(l.ram[(size_t) sw.in]);
+ l.ram[(size_t) sw.in] = -1;
+ }
+ l.slot[(size_t) sw.in] = l.spare_vram;
+ if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
+ l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.in] = nullptr;
+ l.slot[(size_t) sw.out] = -1;
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)
+ l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);
+ } else if (file_experts) { // (back to its pages in the file)
+ l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];
+ } else {
+ l.hh[(size_t) sw.out] = l.spare_host;
+ l.host[(size_t) sw.out] = l.spare_host[0];
+ l.spare_host = old_parts;
+ }
+ l.spare_vram = old_slot;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const Dev& dm = devs[(size_t) x.dev];
+ l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));
+ l.tab_pin[2 * m + 1] = 0ull;
+ cudaSetDevice(dm.id); // (stream order: before the next window)
+ cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);
+ cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);
+ }
+ l.swap_pending = false;
+ --sw_inflight;
+ ++st.swaps;
+ }
+ decay_tokens += T;
+ if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens
+ decay_tokens = 0;
+ for (Layer& l : L)
+ for (float& h : l.heat) h *= c_decay;
+ }
+ bool queued = false, rqueued = false;
+ for (Layer& l : L) {
+ if (l.cpu_hits.empty()) continue;
+ std::vector<int> hits;
+ hits.swap(l.cpu_hits);
+ int best = -1, vin = -1;
+ for (int e : hits)
+ if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;
+ if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in
+ int cold = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;
+ if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {
+ l.swap_pending = true;
+ ++sw_inflight;
+ vin = best;
+ Swap sw{l.il, best, cold};
+ if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_queue.push_back(sw);
+ queued = true;
+ }
+ }
+ if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;
+ int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)
+ for (int e : hits)
+ if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&
+ (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))
+ rb = e;
+ if (rb < 0) continue;
+ int rc = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;
+ const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)
+ if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;
+ RamSwap rs;
+ rs.layer = l.il;
+ rs.in = rb;
+ rs.out = room ? -1 : rc;
+ rs.slot = ram_free.back();
+ ram_free.pop_back();
+ ++ram_inflight;
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_queue.push_back(rs);
+ rqueued = true;
+ }
+ if (queued) sw_cv.notify_all();
+ if (rqueued) ram_cv.notify_one();
+ }
+
+ // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes
+ void prefetch_loop() {
+ std::vector<std::pair<const void*, size_t>> b;
+ for (;;) {
+ {
+ std::unique_lock<std::mutex> lk(pf_mu);
+ pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });
+ if (quit.load(std::memory_order_relaxed)) return;
+ b.swap(pf_q);
+ }
+ os::prefetch_cold(b); // (the resident ones would only cost the page walk)
+ b.clear();
+ }
+ }
+ // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)
+ static bool file_read(const Layer& l, int e) {
+ return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);
+ }
+ // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4
+ // experts read in place, some of them from the files now (none: nothing to read early - null)
+ const Layer* ahead_layer(const Layer& l0, int a) const {
+ const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;
+ if (j >= end || j >= (int) L.size()) return nullptr;
+ const Layer& n = L[(size_t) j];
+ if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||
+ (int) n.slot.size() < n.n_exp)
+ return nullptr;
+ for (int e = 0; e < n.n_exp; ++e)
+ if (file_read(n, e)) return &n;
+ return nullptr;
+ }
+ // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the
+ // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts
+ std::vector<std::pair<const void*, size_t>> pf_want;
+ void ahead_ask(int li, const Req& r, unsigned seq, int T) {
+ pf_want.clear();
+ for (int a = 1; a <= 2; ++a) {
+ const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;
+ if (m <= 0 || j >= (int) L.size()) continue;
+ const Layer& n = L[(size_t) j];
+ uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;
+ uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;
+ if (ahead_for[(size_t) j] != seq + (unsigned) a) {
+ std::memset(fl, 0, 1024);
+ std::memset(rk, 0, 1024);
+ ahead_for[(size_t) j] = seq + (unsigned) a;
+ }
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < m; ++i) {
+ const int e = r.ahead[t][(a - 1) * kAhead1 + i];
+ if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;
+ fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);
+ if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
+ if ((fl[e] & 8) || !file_read(n, e)) continue;
+ fl[e] |= 8;
+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
+ ++st.ahead_pf;
+ }
+ }
+ if (pf_want.empty()) return;
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());
+ }
+ pf_cv.notify_one();
+ }
+
+ // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's
+ // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.
+ void serve_lane() {
+ unsigned next = 1;
+ auto last = std::chrono::steady_clock::now();
+ std::vector<Ent> ents;
+ while (!quit.load(std::memory_order_relaxed)) {
+ Req& r = reqs[next % kRing];
+ if (*(volatile unsigned*) &r.seq != next) {
+ cpu_relax();
+ if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))
+ std::this_thread::sleep_for(std::chrono::microseconds(20));
+ continue;
+ }
+ std::atomic_thread_fence(std::memory_order_acquire);
+ const int li = layer_of[next % kRing].load(std::memory_order_acquire);
+ Resp& rp = resps[next % kRing];
+ const int T = std::max(1, std::min(kWin, r.T));
+ for (int t = 0; t < T; ++t) rp.cpu[t] = 0;
+ if (li >= 0 && li < (int) L.size()) {
+ Layer& l = L[(size_t) li];
+ uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
+ ents.clear();
+ // lookahead: the guesses made for this layer (when they are for this request), counted on decode
+ const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
+ ? ahead_fl.data() + (size_t) li * 1024 : nullptr;
+ // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row
+ float wv[kWin * kPicks];
+ int tok_of[kWin * kPicks];
+ int nslots = 0;
+ std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
+ const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
+ for (int t = 0; t < T; ++t) {
+ float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)
+ if (dsk)
+ for (int i = 0; i < l.k_exp; ++i)
+ if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = r.ids[t][i];
+ if (e < 0 || e >= l.n_exp) continue;
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (fl != nullptr) {
+ ++st.ahead_picks;
+ st.ahead_hit_k += fl[e] & 1;
+ st.ahead_hit += (fl[e] & 6) != 0;
+ st.ahead_hit2 += (fl[e] >> 2) & 1;
+ }
+ if (l.slot[(size_t) e] < 0) {
+ l.cpu_hits.push_back(e);
+ if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
+ ++st.skipped;
+ continue;
+ }
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;
+ else if (fl != nullptr) {
+ ++st.ahead_cold;
+ st.ahead_cold_hit += (fl[e] >> 3) & 1;
+ }
+ picks.push_back({e, t * kPicks + i});
+ }
+ }
+ }
+ if (fl != nullptr) { // the guesses by rank: how many were picked
+ const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;
+ for (int e = 0; e < l.n_exp && e < 1024; ++e) {
+ if (rk[e] == 0 || rk[e] > 16) continue;
+ bool hit = false;
+ for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;
+ ++st.ahead_rank_n[rk[e] - 1];
+ st.ahead_rank_hit[rk[e] - 1] += hit;
+ }
+ }
+ if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);
+ std::sort(picks.begin(), picks.end());
+ for (size_t a = 0; a < picks.size();) {
+ Ent en;
+ en.e = picks[a].first;
+ en.slot0 = nslots;
+ en.n = 0;
+ while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {
+ const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;
+ en.tok[en.n++] = t;
+ tok_of[nslots] = t;
+ wv[nslots] = r.w[t][i];
+ ++nslots;
+ ++a;
+ }
+ ents.push_back(en);
+ }
+ // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for
+ // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)
+ int ord[kWin * kPicks];
+ const int* po = nullptr;
+ if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&
+ ents.size() <= (size_t) (kWin * kPicks)) {
+ const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;
+ int n = 0;
+ for (int pass = 0; pass < 2; ++pass)
+ for (size_t q = 0; q < ents.size(); ++q) {
+ const int e = ents[q].e;
+ const bool early = !file_read(l, e) || (fa[e] & 8) != 0;
+ if (early == (pass == 0)) ord[n++] = (int) q;
+ }
+ po = ord;
+ }
+ if (!ents.empty()) {
+ const double tc = now_ms();
+ cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) {
+ bool any = false;
+ for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;
+ if (!any) continue;
+ float* o = rp.part[t];
+ for (int e = 0; e < D; ++e) o[e] = 0.0f;
+ for (int sl = 0; sl < nslots; ++sl) {
+ if (tok_of[sl] != t) continue;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];
+ }
+ rp.cpu[t] = 1;
+ }
+ st.cpu_ms += now_ms() - tc;
+ st.cpu_experts += nslots;
+ }
+ }
+ std::atomic_thread_fence(std::memory_order_release);
+ *(volatile unsigned*) &rp.seq = next;
+ ++next;
+ last = std::chrono::steady_clock::now();
+ }
+ }
+
+ // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's
+ static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {
+ bool g = false;
+ for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;
+ return g ? gg::mv(j, n, s) : x3::mv(j, n, s);
+ }
+ bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {
+ x3::MvJob j[x3::kMaxMvJobs];
+ int n = 0;
+ for (const x3::MvJob& x : jobs) j[n++] = x;
+ if (!mv_any(j, n, d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {
+ for (int a = 0; a < n; a += x3::kMaxMvJobs)
+ if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs
+ static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,
+ float alpha = 1.0f) {
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = x;
+ j.y = y;
+ j.alpha = alpha;
+ j.ncol = T;
+ j.xf_cs = x_cs;
+ j.y_cs = y_cs;
+ return j;
+ }
+
+ // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:
+ // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm
+ // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x
+ // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),
+ // the trunk's head, the markov bias token by token (greedy) and the confidence head.
+ bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {
+ if (g.n_mtp == 0 || n <= 0) return true;
+ if (n > kWin * kPicks) {
+ err = "seeding the drafter with " + std::to_string(n) + " rows";
+ return false;
+ }
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;
+ if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps
+ for (Dev& e : devs) {
+ cudaSetDevice(e.id);
+ CK(cudaStreamSynchronize(e.st));
+ }
+ cudaSetDevice(d.id);
+ const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);
+ std::fflush(dump_d);
+ }
+ for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)
+ const int na = std::min(kWin, n - a);
+ CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,
+ cudaMemcpyHostToDevice, d.st));
+ if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;
+ }
+ CK(cudaEventRecord(ev_seed, d.st));
+ Finish fm;
+ fm.w = d_main_norm;
+ fm.eps = g.eps;
+ finish(d.ed, n, D, D, fm, d.st);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (l.dev != d_dev) {
+ err = "the DSpark blocks are split across devices";
+ return false;
+ }
+ for (int a = 0; a < n; a += kWin) {
+ const int na = std::min(kWin, n - a);
+ if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) (p0 + a);
+ fk.freqs = d.freqs_win;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, na, HD, HD, fk, d.st);
+ ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);
+ }
+ }
+ return true;
+ }
+
+ bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,
+ int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ const double t0 = now_ms();
+ drafts_q = temperature > 0.0f && sample_drafts;
+ dq_temp = temperature;
+ dq_top_p = top_p;
+ dq_top_k = top_k;
+ dq_min_p = min_p;
+ dq_seed = seed;
+ dq_counter = counter;
+ const bool ok = draft_(y, out, conf, err);
+ if (!ok) drafts_q = false;
+ if (max_drafts > 0 && (int) out.size() > max_drafts) {
+ out.resize((size_t) max_drafts);
+ conf.resize((size_t) max_drafts);
+ }
+ st.draft_ms += now_ms() - t0;
+ ++st.drafts_rounds;
+ return ok;
+ }
+ bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {
+ out.clear();
+ conf.clear();
+ const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;
+ const int64_t p0 = pos;
+ if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;
+ int tk[kWin];
+ tk[0] = y;
+ for (int t = 1; t < T; ++t) tk[t] = g.d_noise;
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
+ init_pre(d.pre, T, d.st);
+ WinPlan dp; // (window attention alone: the mailbox seqs)
+ dp.async = !sync_decode && T <= async_max_t;
+ for (int i = 0; i < g.n_mtp && dp.async; ++i) {
+ const int il = g.nl + i;
+ dp.seq[il] = ++seq_issued;
+ layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;
+ if (dump_d != nullptr && h_dump == nullptr &&
+ (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))
+ return false;
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)
+ ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);
+ CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));
+ }
+ if (!layer(l, 0, p0, T, dp, err)) return false;
+ if (dump_d != nullptr) {
+ cudaSetDevice(d.id);
+ CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));
+ }
+ }
+ cudaSetDevice(d.id);
+ hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)
+ Dev& hd = devs[(size_t) head_dev];
+ // (through the host to the head's device, and to the head group's other members - the drafter's device can be
+ // the head group's lead itself, a group of every GPU)
+ if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {
+ CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(d.ev_out, d.st));
+ }
+ if (d_dev != head_dev) {
+ cudaSetDevice(hd.id);
+ CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));
+ }
+ // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)
+ const std::vector<int>& hg = groups[(size_t) hd.grp].mem;
+ const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)
+ for (int m = (int) hg.size() - 1; m >= 0; --m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ if (m > 0) {
+ CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));
+ } else
+ CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));
+ Finish fn;
+ fn.w = e.d_norm;
+ fn.eps = g.eps;
+ finish(e.ao, T, D, D, fn, e.st);
+ if (!head_share(e, m, e.ao, T, err, !mk)) return false;
+ }
+ h_idx[0] = y;
+ if (mk) {
+ // the markov steps over the group: each member its rows of the bias and its share's argmax, every member
+ // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,
+ // each step ~1/nm of their 66 MB and no gather of the shares
+ const int nm = (int) hg.size();
+ const unsigned seq = ++mk_seq;
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));
+ CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));
+ }
+ for (int i = 0; i < T; ++i)
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;
+ markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);
+ argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);
+ markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,
+ e.st);
+ }
+ cudaSetDevice(hd.id);
+ } else {
+ CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));
+ strata::kernels::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)
+ sp.greedy = false;
+ sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);
+ sp.top_p = dq_top_p;
+ sp.top_k = dq_top_k;
+ sp.min_p = dq_min_p;
+ sp.seed = dq_seed;
+ }
+ for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
+ markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
+ if (drafts_q) {
+ sp.counter = dq_counter + (uint64_t) i;
+ strata::kernels::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);
+ } else
+ strata::kernels::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);
+ }
+ }
+ confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
+ int tok[kWin + 1];
+ float cf[kWin];
+ CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];
+ for (int i = 0; i < T; ++i) cf[i] = h_w[i];
+ if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden
+ std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream
+ CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ const int64_t hdr[4] = {'D', p0, y, T};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(tok + 1, 4, (size_t) T, dump_d);
+ std::fwrite(cf, 4, (size_t) T, dump_d);
+ std::fwrite(xs.data(), 4, xs.size(), dump_d);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);
+ std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);
+ }
+ std::fflush(dump_d);
+ cudaSetDevice(hd.id);
+ }
+ for (int i = 0; i < T; ++i) {
+ out.push_back(tok[i + 1]);
+ conf.push_back(cf[i]);
+ }
+ return true;
+ }
+
+ int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,
+ const float* bias) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::kernels::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ sp.logit_bias = bias;
+ strata::kernels::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
+ int t = -1;
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.
+ // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own
+ // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution
+ // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the
+ // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.
+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {
+ const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);
+ const int T = n + 1, V = g.vocab;
+ const int64_t p0 = pos;
+ int tk[kWin];
+ tk[0] = y;
+ for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];
+ spec_win = true;
+ const double tv = now_ms();
+ const bool ok = forward(tk, T, T, err, 0);
+ st.verify_ms += now_ms() - tv;
+ spec_win = false;
+ if (!ok) return false;
+ Dev& hd = devs[(size_t) head_dev];
+ cudaSetDevice(hd.id);
+ int m = 0;
+ if (temperature <= 0.0f) {
+ strata::kernels::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ strata::kernels::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = h_idx[m];
+ } else {
+ // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)
+ // (never d itself): each row a token distributed as the model's own draw at that position, p through the
+ // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,
+ // never kept); past the first rejection the rows do not count
+ const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&
+ dq_min_p == min_p && dq_seed == seed && dq_counter == counter;
+ if (!own) { // the drafter's argmax: q a point at each draft
+ for (int i = 0; i < n; ++i) {
+ h_specq[(size_t) i].n = 1;
+ h_specq[(size_t) i].ids[0] = drafts[(size_t) i];
+ h_specq[(size_t) i].p[0] = 1.0;
+ }
+ CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::kernels::SpecList),
+ cudaMemcpyHostToDevice, hd.st));
+ }
+ for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];
+ CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));
+ strata::kernels::SamplerParams sp;
+ sp.greedy = false;
+ sp.temperature = temperature;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::kernels::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);
+ }
+ drafts_q = false;
+ rollback(p0, T, m + 1);
+ if (!seed_drafter(p0, m + 1, 0, err)) return false;
+ accepted = m;
+ st.drafts_offered += n;
+ st.drafts_accepted += m;
+ for (int i = 0; i < n && i <= m && i < 8; ++i) {
+ ++st.draft_reached[i];
+ if (i < m) ++st.draft_kept[i];
+ }
+ return true;
+ }
+
+ // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone
+ // (the compressed KV and the index keys past them are written again when those positions come)
+ void rollback(int64_t p0, int T, int keep) {
+ if (keep >= T) return;
+ // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor
+ // parallel, a millisecond of the host's before the next draft)
+ std::vector<RingBatch> rb(devs.size());
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ RingBatch& b = rb[(size_t) l.dev];
+ if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)
+ cudaSetDevice(devs[(size_t) l.dev].id);
+ ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);
+ continue;
+ }
+ b.ring[b.n] = l.win;
+ b.bak[b.n++] = l.ring_bak;
+ }
+ for (size_t di = 0; di < devs.size(); ++di)
+ if (rb[di].n > 0) {
+ cudaSetDevice(devs[di].id);
+ ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);
+ }
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ Dev& d = devs[(size_t) l.dev];
+ if (l.c_snap != nullptr) {
+ cudaSetDevice(d.id);
+ const size_t ns = (size_t) l.ratio * g.hd * 4;
+ cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);
+ compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,
+ g.hd, l.c_norm, g.eps, d.latent, d.st);
+ }
+ }
+ pos = p0 + keep;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ }
+
+ // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
+ bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }
+
+
+ // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
+ // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
+ // place, the CPU lane's streamed in batches)
+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ Chunk& c = *chunks[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ prof_mark(d, 0);
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ab = groups[(size_t) L0.grp].arb;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ // Engram, in slabs of 256 tokens
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)
+ uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;
+ CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.eg_ev[slot], s));
+ for (int s0 = 0; s0 < T; s0 += 256) {
+ const int ns = std::min(256, T - s0);
+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
+ if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
+ engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
+ }
+ }
+ // attention
+ if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
+ hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
+ hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
+ return false;
+ if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
+ if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
+ // the window rows: the ring's (positions p0 - win ..) then the chunk's
+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = c.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(c.kvr, T, HD, HD, fk, s);
+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
+ const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
+ if (snap) {
+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
+ }
+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ const uint8_t* ckv = nullptr;
+ const long long ss = (long long) o.max_context + 64;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ const int64_t g_first = p0 / R, g_end = (p0 + T) / R;
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ const float* latent = c.latent;
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(c.ckr, T, HD, HD, fc, s);
+ latent = c.ckr;
+ } else {
+ if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)
+ const int64_t ps = p0 + wp.snap_t;
+ for (int sl = 0; sl < R; ++sl) {
+ const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;
+ const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;
+ const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;
+ CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ }
+ }
+ compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);
+ }
+ }
+ const int clen_max = (int) ((p0 + T) / R);
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);
+ if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ }
+ if (clen_max > 0) {
+ if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
+ return false;
+ for (int a0 = 0; a0 < T; a0 += c.AI) {
+ const int na = std::min(c.AI, T - a0);
+ const int64_t pa = p0 + a0;
+ if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,
+ g.ix_heads * g.ix_hd, 0.0f, err))
+ return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) pa;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ Rows lim;
+ lim.gen = 1;
+ lim.p0 = pa;
+ lim.R = R;
+ lim.kcap = g.ix_topk;
+ lim.ccap = g.cand_topk;
+ lim.blk = std::max(1, g.cand_blk);
+ const int nlast = (int) ((pa + na) / R); // the slab's widest query
+ int* tk = c.topk + (size_t) a0 * topk_stride();
+ int* cd = c.cand + (size_t) a0 * cand_stride();
+ const float* iw = c.iw + (size_t) a0 * 64;
+ if (nlast <= 0) continue;
+ if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {
+ Rows cl = lim;
+ cl.gen = 2;
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,
+ cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);
+ Rows tr = lim;
+ tr.gen = 3;
+ topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);
+ } else {
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,
+ lim, c.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr = lim;
+ cr.kcap = g.cand_topk;
+ cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,
+ (int) cand_stride(), s);
+ }
+ topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);
+ }
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(const_cast<float*>(latent), ng, HD, HD, fc, s);
+ if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once
+ const int slot = wp.k % 2; // they took the chunk before last's)
+ if (wp.k >= 2)
+ for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.stage(slot);
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);
+ }
+ }
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
+ if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
+ for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
+ const int na = std::min(c.A, T - a0);
+ if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) (p0 + a0);
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(c.q, na * NHL, HD, HD, fr, s);
+ AttnRows ar;
+ ar.gen = 1;
+ ar.p0 = p0;
+ ar.t0 = a0;
+ ar.win = g.win;
+ ar.R = l.ratio;
+ ar.kcap = g.ix_topk;
+ sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
+ na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) (p0 + a0);
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(c.o, na * NHL, HD, HD, fo, s);
+ x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,
+ OG * g.o_lora, 0.0f, err))
+ return false;
+ x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
+ if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
+ }
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
+ prof_mark(d, 1);
+ // the MoE
+ if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
+ hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);
+ hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)
+ for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)
+ const int nr = std::min(kVlRows, T - r0);
+ VlBits vb;
+ const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);
+ route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,
+ c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);
+ }
+ const int n_vs = L0.n_slots + (int) L0.host_list.size();
+ CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)
+ moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);
+ CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));
+ // the shared expert into acc while the host waits for the bounds
+ if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))
+ return false;
+ swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);
+ x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);
+ if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;
+ prof_mark(d, 2);
+ CK(cudaStreamSynchronize(s));
+ std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);
+ if (dump_r != nullptr && mi == 0) {
+ const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;
+ std::vector<int> ri(np), rt(nk);
+ std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);
+ CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
+ cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};
+ std::fwrite(hdr, 8, 8, dump_r);
+ std::fwrite(ri.data(), 4, np, dump_r);
+ std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);
+ std::fwrite(b.data(), 4, b.size(), dump_r);
+ std::fwrite(rt.data(), 4, nk, dump_r);
+ std::fwrite(gl.data(), 4, gl.size(), dump_r);
+ std::fwrite(gb.data(), 4, gb.size(), dump_r);
+ std::fflush(dump_r);
+ }
+ if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const int v = L0.vs_pin[e];
+ pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ }
+ if (c.flow) {
+ if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+ // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones
+ // in shares between them (the device works while the copies run)
+ const int di = l.dev, P = Chunk::kParts;
+ int nb = 0;
+ for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;
+ int v_done = 0;
+ for (int bi = 0; bi < nb; ++bi) {
+ const Batch& bt = S.b[S.kp];
+ const int half = (int) (S.kp % P);
+ CK(cudaStreamWaitEvent(s, c.filled[half], 0));
+ const int v0 = L0.n_slots + bt.j0;
+ if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,
+ b.data(), v0, v0 + bt.n, T, err))
+ return false;
+ CK(cudaEventRecord(c.freed[half], s));
+ ++S.kp;
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+
+ // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)
+ void plan_streams() {
+ for (Layer& l : L) {
+ if (l.mtp || l.vs_pin == nullptr) continue;
+ l.host_list.clear();
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);
+ for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)
+ Layer& x = member(l, m);
+ int j = 0;
+ for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;
+ }
+ }
+ }
+
+ // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows
+ Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {c.h, c.pre, c.topk, c.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the
+ // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows
+ // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across
+ // layers and chunks
+ struct Run {
+ std::vector<int64_t> p0;
+ std::vector<int> T;
+ std::vector<unsigned> seq;
+ int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
+ };
+ // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
+ // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
+ // layers and back before the next - so a layer's experts are read from the files once for the whole run (the
+ // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk
+ bool run_dev_layers(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S; // (none: the flow streams)
+ const size_t hb = chunk_hand_bytes();
+ // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while
+ // the one before runs (slot u % 2, once unit u - 2's copy of it is done)
+ std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
+ for (int il : c.fl_layers)
+ if (L[(size_t) il].eg >= 0)
+ for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
+ if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
+ const int li = egu[u].first, k = egu[u].second;
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);
+ };
+ size_t eg_u = 0;
+ std::future<void> eg_next;
+ if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);
+ for (size_t li = 0; li < c.fl_layers.size(); ++li) {
+ const int il = c.fl_layers[li];
+ Layer& L0 = L[(size_t) il];
+ const bool last_l = li + 1 == c.fl_layers.size();
+ for (int k = 0; k < nk; ++k) {
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way
+ const double te = now_ms();
+ eg_next.get();
+ st.engram_ms += now_ms() - te;
+ wp.eg_slot = (int) (eg_u % 2);
+ if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);
+ }
+ if (li == 0) { // the chunk's input: its embedding rows
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
+ init_pre(c.pre, T, s);
+ } else { // or its state after the layer before
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));
+ }
+ if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;
+ if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
+ if (!last_l) { // the chunk's state kept for the next layer
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));
+ continue;
+ }
+ // through every layer: the chunk's end, and the logits
+ if ((size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ lm_layers.store((int) li + 1, std::memory_order_release);
+ }
+ return true;
+ }
+ bool run_dev_chunks(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S;
+ const int B = std::max(1, c.ring_n / Chunk::kParts);
+ if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)
+ CK(cudaStreamSynchronize(s));
+ CK(cudaStreamSynchronize(c.cs));
+ c.items.clear();
+ c.base = c.n_read = c.n_dma = c.n_comp = 0;
+ c.n_file = 0;
+ std::fill(c.stage_item.begin(), c.stage_item.end(), -1);
+ c.fl_layers.clear();
+ for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)
+ for (int m = 0; m < L[(size_t) il].nm; ++m)
+ if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
+ c.fl_T = run.T;
+ c.spec_k = c.spec_il = -1;
+ c.wait_ms = 0;
+ c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
+ c.lc_il = -1;
+ c.lc_used = 0;
+ c.lc_first = c.lc_chk = 0;
+ c.io_d0 = c.io->direct_bytes();
+ c.io_b0 = c.io->buffered_bytes();
+ if (!c.fl_layers.empty()) {
+ flow_spec(c, 0, c.fl_layers[0], run.T[0]);
+ if (!flow_pump(c, err)) return false;
+ }
+ if (c.lm) return run_dev_layers(di, run, err);
+ }
+ for (int k = 0; k < nk && !c.flow; ++k)
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& L0 = L[(size_t) il];
+ bool here = false;
+ for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
+ if (!here) continue;
+ for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)
+ S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});
+ }
+ if (!S.b.empty() && c.ring == nullptr) {
+ err = "no slots to stream experts into on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ std::atomic<bool> ra_stop{false};
+ std::thread ra;
+ if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });
+ struct Join { // (every way out of the run: the read-ahead ends before S does)
+ std::atomic<bool>& stop;
+ std::thread& th;
+ ~Join() {
+ stop.store(true);
+ if (th.joinable()) th.join();
+ }
+ } ra_join{ra_stop, ra};
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once
+ // chunk k - 2's copy of it is done)
+ std::vector<int> egs;
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egs, eg_per](int k) {
+ if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);
+ for (int li : egs)
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);
+ };
+ std::future<void> eg_next;
+ if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
+ for (int k = 0; k < nk; ++k) {
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way
+ const double te = now_ms();
+ eg_next.get();
+ if (d.mi == 0) st.engram_ms += now_ms() - te;
+ if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);
+ }
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
+ init_pre(c.pre, T, s);
+ } else {
+ Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);
+ set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);
+ }
+ }
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ Xfer x;
+ if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot);
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (ng > 0 && sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);
+ set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)
+ }
+ if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
+ const int nnext = L[(size_t) il + 1].nm;
+ if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);
+ xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);
+ }
+ }
+ if (di == head_dev && (size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ if (di != head_dev) continue;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ // ---- a prompt's path: the flow or windows of 16
+ // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a
+ // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in
+ // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the
+ // flow, as before)
+ void calibrate_paths() {
+ if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;
+ if (const char* v = std::getenv("DS4_PATH_PICK"))
+ if (std::atoi(v) == 0) return;
+ int di = -1;
+ for (size_t i = 0; i < chunks.size() && di < 0; ++i)
+ if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;
+ if (di < 0) return;
+ Chunk& c = *chunks[(size_t) di];
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const double t0 = now_ms();
+ // the DMA: staging -> the ring (once to warm up, once timed)
+ const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});
+ double ms = 0;
+ for (int rep = 0; rep < 2; ++rep) {
+ const double a = now_ms();
+ if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||
+ cudaStreamSynchronize(c.cs) != cudaSuccess) {
+ cudaGetLastError();
+ return;
+ }
+ ms = now_ms() - a;
+ }
+ pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);
+ // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are
+ auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {
+ const double a = now_ms();
+ left.store((int) jobs.size());
+ c.io->push(jobs.data(), (int) jobs.size());
+ while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ return now_ms() - a;
+ };
+ {
+ const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;
+ std::vector<uint8_t> src(part * 8, 1);
+ std::vector<IoQueue::Job> jobs(8);
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < 8; ++i) {
+ jobs[(size_t) i].mem = src.data() + (size_t) i * part;
+ jobs[(size_t) i].dst = c.stage + (size_t) i * part;
+ jobs[(size_t) i].len = part;
+ jobs[(size_t) i].left = &left;
+ jobs[(size_t) i].bad = &bad;
+ }
+ run_jobs(jobs, left);
+ const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);
+ pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);
+ }
+ // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:
+ // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane
+ // three times (from the disk, then in RAM: the faster of two)
+ int best = -1, most = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int n = 0;
+ for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;
+ if (n > most) most = n, best = il;
+ }
+ if (best < 0) return;
+ const Layer& l = L[(size_t) best];
+ std::vector<int> cold;
+ const int n_read = std::min(8, c.S), n_cpu = 8;
+ for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {
+ if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;
+ bool res = false;
+ for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);
+ if (!res) cold.push_back(e);
+ }
+ if ((int) cold.size() >= n_read + n_cpu) {
+ std::vector<IoQueue::Job> jobs;
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < n_read; ++i) {
+ uint8_t* sb = c.stage + (size_t) i * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ IoQueue::Job j;
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];
+ j.len = mx_part_bytes(l, r);
+ j.dst = sb;
+ j.left = &left;
+ j.bad = &bad;
+ j.map = pk.at(l.mx_file[r], j.off);
+ jobs.push_back(j);
+ sb += flow_region(r);
+ }
+ }
+ const double rd = run_jobs(jobs, left);
+ if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);
+ }
+ { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM
+ const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;
+ std::vector<Ent> ents;
+ for (int i = 0; i < n_cpu; ++i) {
+ Ent en;
+ en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;
+ if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)
+ for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)
+ if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;
+ if (en.e < 0) break;
+ en.n = 1;
+ en.slot0 = i;
+ en.tok[0] = 0;
+ ents.push_back(en);
+ }
+ if (ents.empty()) return;
+ std::vector<float> x((size_t) g.dim);
+ for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);
+ const double bytes = (double) ents.size() * (double) l.blob;
+ double a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ const double t_first = now_ms() - a;
+ double t_warm = 1e30;
+ for (int rep = 0; rep < 2; ++rep) {
+ a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ t_warm = std::min(t_warm, now_ms() - a);
+ }
+ pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);
+ if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);
+ }
+ pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;
+ if (o.verbose) {
+ char dk[32] = "not measured", cc[32] = "not measured";
+ if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);
+ if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);
+ std::fprintf(stderr,
+ "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "
+ "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",
+ pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);
+ }
+ }
+ // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's
+ // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)
+ double pm_rate(const Layer& l, int& n_cold) const {
+ const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0, cold = 0;
+ n_cold = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ sum += pr[e];
+ if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;
+ }
+ if (n_cold == 0) return 0.0;
+ const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;
+ return pm.m * (double) l.k_exp * share / n_cold;
+ }
+ // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))
+ double pm_picks(int n) const {
+ double d = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int nc = 0;
+ const double a = pm_rate(L[(size_t) il], nc);
+ d += nc * (1.0 - std::exp(-a * n));
+ }
+ return d;
+ }
+ // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation
+ // that (bisection over its log; a running mean of the log)
+ void pm_learn(int n, double picked) {
+ if (!pm.on || n < 2 || picked < 1) return;
+ const double m0 = pm.m;
+ double lo = std::log(1e-3), hi = std::log(1e3);
+ for (int it = 0; it < 40; ++it) {
+ pm.m = std::exp(0.5 * (lo + hi));
+ (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);
+ }
+ const double obs = 0.5 * (lo + hi);
+ pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);
+ ++pm.n_m;
+ }
+ // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's
+ // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -
+ // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:
+ // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at
+ // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
+ // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
+ // faster (by half, with less than kPickRes in RAM); said with --verbose
+ bool pick_path(int n) {
+ pm.last = -1;
+ if (!pm.on || n >= kPickMax) return true;
+ double f_res = 0, f_cold = 0, w = 0;
+ const int nw = n / kWin, rem = n % kWin;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ int nc = 0;
+ const double a = pm_rate(l, nc);
+ if (nc == 0) continue;
+ int sampled = 0, res = 0, seen = 0;
+ const int step = std::max(1, nc / 32);
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;
+ bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ if (!r) {
+ r = true;
+ for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);
+ }
+ ++sampled;
+ res += r;
+ }
+ const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;
+ auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };
+ f_res += share * u(n) * b;
+ f_cold += (1.0 - share) * u(n) * b;
+ w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;
+ }
+ const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;
+ const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);
+ const double t_flow = f_res / r_fr + f_cold / r_fc;
+ const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;
+ const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;
+ // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the
+ // fixed costs of every layer on every GPU again, which a pass of the flow pays once)
+ const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;
+ // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane
+ // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would
+ // cost the most; windows win where they are all in RAM)
+ const double res_share = f_res / std::max(1.0, f_res + f_cold);
+ const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));
+ pm.last = flow ? 1 : 0;
+ pm.last_cls = cls;
+ pm.last_nw = n_win;
+ pm.last_pred = flow ? t_flow : t_win;
+ if (o.verbose)
+ std::fprintf(stderr,
+ "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "
+ "experts' %.1f GB in RAM)\n",
+ n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,
+ 100.0 * res_share, (f_res + f_cold) / 1e9);
+ return flow;
+ }
+ // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more
+ static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }
+ // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs
+ // is mostly the fixed costs a window pays again and again)
+ double pm_k(int c) const {
+ for (int dc = 0; dc < 4; ++dc)
+ for (int cc : {c - dc, c + dc})
+ if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];
+ return 1.0;
+ }
+ // n tokens through windows of 16, the last one's logits_rows rows of logits
+ bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {
+ logits_all = false;
+ for (int a = 0; a < n; a += kWin) {
+ const int T = std::min(kWin, n - a);
+ if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;
+ }
+ return true;
+ }
+ // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken
+ // / predicted), windows' time a window - running means
+ void path_done(double ms) {
+ if (pm.last < 0 || pm.last_pred <= 0) return;
+ if (pm.last == 0) {
+ const double w = ms / 1000.0 / std::max(1, pm.last_nw);
+ pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;
+ ++pm.win_n;
+ } else {
+ const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));
+ double& k = pm.k[0][pm.last_cls];
+ int64_t& n = pm.kn[0][pm.last_cls];
+ k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;
+ ++n;
+ }
+ pm.last = -1;
+ }
+ // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the
+ // first would read from the files again more than the chunks' states moved to host memory and back at every layer
+ // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,
+ // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB
+ // sets it) - at least one expert, or not layer-major
+ bool lm_begin(const Run& run) {
+ static const char* env = std::getenv("DS4_LAYER_MAJOR");
+ if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;
+ Chunk& c = *chunks[0];
+ if (!c.flow) return false;
+ size_t need = 0, n_exp = 0, blob_max = 0;
+ double file_b = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ size_t ne = 0;
+ for (int e : l.host_list)
+ if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;
+ const size_t stride = (l.blob + 63) & ~(size_t) 63;
+ need = std::max(need, ne * stride);
+ n_exp = std::max(n_exp, (size_t) l.n_exp);
+ blob_max = std::max(blob_max, stride);
+ file_b += (double) ne * (double) l.blob;
+ }
+ int64_t rows = 0;
+ for (int T : run.T) rows += T;
+ const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;
+ const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;
+ if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;
+ size_t lc_bytes = need;
+ if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);
+ else {
+ const uint64_t avail = ram_avail();
+ if (avail != ~uint64_t{0})
+ lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);
+ }
+ if (lc_bytes < blob_max) return false;
+ void* st_p = nullptr;
+ c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;
+ if (!c.lm_pinned) {
+ cudaGetLastError();
+ st_p = os::anon_alloc(state_b, -1, 1);
+ if (st_p == nullptr) return false;
+ }
+ c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);
+ if (c.lc == nullptr) {
+ if (c.lm_pinned) cudaFreeHost(st_p);
+ else os::anon_free(st_p, state_b);
+ return false;
+ }
+ c.lm_state = (uint8_t*) st_p;
+ c.lm_state_bytes = state_b;
+ c.lc_bytes = lc_bytes;
+ c.lc_of.assign(n_exp, -1);
+ c.lc_writer.assign(n_exp, -1);
+ c.lm = true;
+ lm_layers.store(0);
+ return true;
+ }
+ void lm_end() {
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ Chunk& c = *chunks[di];
+ if (!c.lm) continue;
+ cudaSetDevice(devs[di].id);
+ cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's
+ cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)
+ for (size_t i = c.base; i < c.n_read; ++i)
+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ if (c.lm_pinned) cudaFreeHost(c.lm_state);
+ else os::anon_free(c.lm_state, c.lm_state_bytes);
+ os::anon_free(c.lc, c.lc_bytes);
+ c.lm_state = c.lc = nullptr;
+ c.lm_state_bytes = c.lc_bytes = 0;
+ c.lm = c.lm_pinned = false;
+ }
+ }
+
+ // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
+ // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
+ bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
+ if (n < 1) return true;
+ if (pos + n > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < n; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
+ Run run;
+ for (int a = 0; a < n; a += per) {
+ run.p0.push_back(pos + a);
+ run.T.push_back(std::min(per, n - a));
+ run.seq.push_back(++win_seq);
+ }
+ logits_all = logits_rows > kWin;
+ if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
+ for (int t = 0; t < n; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ plan_streams();
+ if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way
+ run.snap_pos = pos + n - 1;
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ Dev& d = devs[(size_t) l.dev];
+ if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;
+ if (l.c_kv != nullptr && l.snap_kv == nullptr &&
+ (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ }
+ const int T_last = run.T.back();
+ n_logits = logits_all ? 0 : std::min(logits_rows, T_last);
+ const double tl = now_ms();
+ { // each chunk's end on the head device, for the progress reports
+ cudaSetDevice(devs[(size_t) head_dev].id);
+ while (chunk_done.size() < run.T.size()) {
+ cudaEvent_t e = nullptr;
+ CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ chunk_done.push_back(e);
+ }
+ chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);
+ for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);
+ }
+ prof_chunk = prof_on;
+ const bool lm = lm_begin(run);
+ struct LmEnd { // (every way out: its host memory freed once the device is done with it)
+ Impl* p;
+ bool on;
+ ~LmEnd() {
+ if (on) p->lm_end();
+ }
+ } lm_end_{this, lm};
+ std::vector<std::string> errs(devs.size());
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, &run, &errs] {
+ std::string e;
+ if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ size_t reported = 0;
+ for (;;) {
+ bool all = true;
+ for (auto& w : workers) all = all && w->idle();
+ if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
+ const int nd = lm_layers.load(std::memory_order_acquire);
+ if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
+ } else if (progress) { // chunks through every layer (the last layer's device past their event)
+ while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&
+ cudaEventQuery(chunk_done[reported]) == cudaSuccess)
+ ++reported;
+ if (reported > 0 && reported < run.T.size())
+ progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);
+ }
+ if (all) break;
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ for (const std::string& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ st.layer_ms += now_ms() - tl;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ cudaSetDevice(d.id);
+ prof_collect();
+ std::fprintf(stderr,
+ "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "
+ "(with the waits for streamed ones), between layers %.0f ms\n",
+ n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);
+ for (double& v : cprof_ms) v = 0;
+ prof_chunk = false;
+ }
+ if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {
+ const double ms = now_ms() - tl;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ const Chunk& c = *chunks[di];
+ if (!c.flow) continue;
+ const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;
+ const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;
+ const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)
+ char lcs[160] = "";
+ if (c.lm)
+ std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "
+ "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);
+ std::fprintf(stderr,
+ "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "
+ "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "
+ "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "
+ "%.0f ms for reads\n",
+ devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),
+ (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,
+ (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,
+ (unsigned long long) c.it_spec_unused, c.wait_ms);
+ }
+ }
+ if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)
+ bool tp = false;
+ for (const Layer& l : L) tp = tp || l.nm > 1;
+ double picked = 0;
+ for (const auto& cp : chunks)
+ if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;
+ if (!tp) pm_learn(n, picked);
+ }
+ if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)
+ pos += n;
+ st.tokens += n;
+ cache_tick(n);
+ const int nr = std::min(T_last, tap_cap);
+ if (!seed_drafter(pos - nr, nr, 0, err)) return false;
+ if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -
+ // the next feed seeds them again from its own rows, past this position)
+ Check& ckp = ck[snap_slot & 1];
+ ckp.pos = run.snap_pos;
+ ckp.win.resize(L.size());
+ ckp.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& dv = devs[(size_t) l.dev];
+ cudaSetDevice(dv.id);
+ const bool kept = !l.mtp && l.snap_win != nullptr;
+ ckp.win[i].resize((size_t) g.win * win_rb());
+ CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),
+ cudaMemcpyDeviceToHost, dv.st));
+ if (l.c_kv) {
+ ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,
+ ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,
+ ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ }
+ }
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ }
+ return true;
+ }
+
+ // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)
+ void plan_window(int64_t p0, int T, WinPlan& wp) {
+ wp.par = par;
+ wp.async = par || (!sync_decode && T <= async_max_t);
+ wp.vl = 0;
+ for (int t = 0; t < T && !imgs.empty(); ++t)
+ if (in_image(p0 + t)) wp.vl |= 1u << t;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ if (l.ratio > 0) {
+ const int R = l.ratio;
+ int n[kWin], k[kWin], clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ n[t] = (int) ((p0 + t + 1) / R);
+ k[t] = std::min(g.ix_topk, n[t]);
+ clen_max = std::max(clen_max, n[t]);
+ }
+ if (l.ix_src) {
+ for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];
+ if (clen_max > 0 && l.cand_src)
+ for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);
+ for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];
+ }
+ for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];
+ }
+ if (wp.async) {
+ wp.seq[il] = ++seq_issued;
+ layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ }
+ }
+
+ int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }
+ // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's
+ // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows
+ void head_cols(int nm, int mi, int& v0, int& vn) const {
+ const Tensor* tw = pk.get("head.weight");
+ if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {
+ vn = g.vocab / nm;
+ v0 = mi * vn;
+ } else
+ split128(g.vocab, nm, mi, v0, vn);
+ }
+ // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's
+ // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two
+ // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather
+ // false: each member keeps its share (the drafter's markov steps over the group, mk_split)
+ bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {
+ const int V = g.vocab, D = g.dim, nm = head_nm();
+ int v0 = 0, vn = 0, Vmax = 0, z = 0;
+ head_cols(nm, mi, v0, vn);
+ head_cols(nm, 0, z, Vmax);
+ Group& G = groups[(size_t) d.grp];
+ float* out = mi == 0 ? d.logits : d.logits_part;
+ const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;
+ cudaSetDevice(d.id);
+ // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4
+ // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)
+ const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head));
+ for (int c = 0; c < rows; c += per)
+ if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))
+ return false;
+ if (nm == 1 || !gather) return true;
+ const unsigned seq = ++d.head_seq;
+ const int slot = (int) (seq % 2);
+ auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };
+ auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };
+ if (mi > 0) {
+ if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = d.logits_part;
+ x.dst[0] = stage(mi);
+ x.bytes[0] = (size_t) rows * vn * 4;
+ xfer_put(x, flag(mi), seq, d.xcnt, d.st);
+ return true;
+ }
+ for (int m = 1; m < nm; ++m) {
+ int m0 = 0, mn = 0;
+ head_cols(nm, m, m0, mn);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = stage(m);
+ x.dst[0] = d.logits_part;
+ x.bytes[0] = (size_t) rows * mn * 4;
+ xfer_get(x, flag(m), seq, d.st);
+ CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,
+ (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));
+ }
+ set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);
+ return true;
+ }
+
+ // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)
+ bool head_rows_dev(int di, int T, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const int D = g.dim, H = g.hc;
+ if (n_logits <= 0) return true;
+ const int t0 = T - n_logits;
+ hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);
+ return head_share(d, d.mi, d.x, n_logits, err);
+ }
+
+ // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read
+ // h_taps)
+ bool tap(Dev& d, int il, int T, std::string& err) {
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (g.n_mtp == 0 || it == g.d_targets.end()) return true;
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int D = g.dim;
+ cudaSetDevice(d.id);
+ hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);
+ CK(cudaStreamWaitEvent(d.st, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,
+ cudaMemcpyDeviceToHost, d.st));
+ return true;
+ }
+
+ bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {
+ if (T < 1 || T > kWin) {
+ err = "a window of " + std::to_string(T) + " tokens";
+ return false;
+ }
+ if (pos + T > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < T; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ const int64_t p0 = pos;
+ for (int t = 0; t < T; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is
+ // left of their reads by then is the window's Engram time
+ const double te = now_ms();
+ const int64_t eg_w0 = eg_wait_ns.load();
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ EgBatch& b = *eg_win[(size_t) li];
+ if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)
+ eg_issue(p0, T, li, nullptr, b);
+ }
+ st.engram_ms += now_ms() - te;
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)
+ std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);
+ plan_window(p0, T, wplan);
+ prof_window = prof_on && T == 1;
+ n_logits = std::min(logits_rows, T);
+ const double tl = now_ms();
+ if (par) {
+ if (!forward_par(p0, T, err)) return false;
+ } else if (!forward_serial(p0, T, err)) return false;
+ st.layer_ms += now_ms() - tl;
+ for (auto& b : eg_win)
+ if (b->live) eg_finish(*b);
+ st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ prof_collect();
+ pos += T;
+ st.tokens += T;
+ cache_tick(T);
+ if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;
+ return true;
+ }
+
+ // one thread: the layers device after device, the hand-offs through pinned memory and events
+ bool forward_serial(int64_t p0, int T, std::string& err) {
+ const int D = g.dim, H = g.hc;
+ int cur = L[0].dev;
+ {
+ Dev& d = devs[(size_t) cur];
+ cudaSetDevice(d.id);
+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
+ init_pre(d.pre, T, d.st);
+ }
+ std::vector<float> dump;
+ if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ if (l.dev != cur) {
+ Dev& a = devs[(size_t) cur];
+ Dev& b = devs[(size_t) l.dev];
+ cudaSetDevice(a.id);
+ const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ uint8_t* bb = a.bounce;
+ CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaEventRecord(a.ev_out, a.st));
+ cudaSetDevice(b.id);
+ CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));
+ CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));
+ cur = l.dev;
+ }
+ if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
+ if (!layer(l, 0, p0, T, wplan, err)) return false;
+ if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
+ Dev& dd = devs[(size_t) l.dev];
+ std::vector<float> hv((size_t) T * H * D);
+ cudaSetDevice(dd.id);
+ CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));
+ CK(cudaStreamSynchronize(dd.st));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,
+ (size_t) H * D * 4);
+ }
+ }
+ if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);
+ if (cur != head_dev) {
+ err = "the head is not on the last layer's GPU";
+ return false;
+ }
+ return head_rows_dev(head_dev, T, err);
+ }
+
+ // threads: each device's thread enqueues its layers (a group's members each their share) at once, the
+ // devices waiting on each other through host-mapped flags (the window's seq)
+ bool forward_par(int64_t p0, int T, std::string& err) {
+ wplan.win = ++win_seq;
+ std::vector<std::string> errs(devs.size());
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, p0, T, &errs] {
+ std::string e;
+ if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ for (auto& w : workers) w->wait();
+ for (const std::string& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ return true;
+ }
+
+ // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer
+ Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {d.h, d.pre, d.topk, d.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ embed(d.emb_rows, T, d.h, D, H, s);
+ init_pre(d.pre, T, s);
+ } else {
+ const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);
+ }
+ }
+ // the compressed KV / index keys another group's source layer wrote this window
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ if (ng == 0) continue;
+ Xfer x;
+ if (sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage;
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);
+ }
+ if (mi == 0 && !tap(d, il, T, err)) return false;
+ if (!layer(L0, mi, p0, T, wp, err)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);
+ return true;
+ }
+
+ // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,
+ // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it
+ bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,
+ size_t bytes, int from, std::string& err) {
+ bool any = false;
+ for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);
+ if (!any) return true;
+ Dev& d = devs[(size_t) from];
+ CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(ev, d.st));
+ for (size_t od = 0; od < devs.size(); ++od) {
+ if (copies[od] == nullptr || (int) od == from) continue;
+ Dev& o2 = devs[od];
+ cudaSetDevice(o2.id);
+ CK(cudaStreamWaitEvent(o2.st, ev, 0));
+ CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));
+ }
+ cudaSetDevice(d.id);
+ return true;
+ }
+
+ // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;
+ // the partial sums all-reduced across the group)
+ bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ag = groups[(size_t) L0.grp].ar;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ prof_mark(d, 0);
+ // Engram
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)
+ uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;
+ for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]
+ CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,
+ cudaMemcpyHostToDevice, s));
+ uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;
+ for (int t = 0; t < T; ++t)
+ CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
+ (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);
+ if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
+ engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
+ }
+ // attention
+ hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
+ hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
+ hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
+ {
+ x3::MvJob j[4];
+ int nj = 0;
+ j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);
+ j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);
+ if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);
+ if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);
+ if (!mv_n(d, j, nj, err)) return false;
+ }
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(d.qr, T, g.q_lora, g.q_lora, fq, s);
+ if (NHL > 0) {
+ if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) p0;
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(d.q, T * NHL, HD, HD, fr, s);
+ }
+ // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark
+ // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of
+ // that ring and to the whole block, and leaves the ring as it was
+ ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, T, HD, HD, fk, s);
+ if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
+ AttnRows ar;
+ for (int t = 0; t < T; ++t) {
+ if (l.mtp) {
+ ar.win0[t] = 0;
+ ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);
+ ar.nsel[t] = T;
+ continue;
+ }
+ const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)
+ ar.win0[t] = first;
+ ar.nwin[t] = g.win + t - first + 1;
+ ar.nsel[t] = 0;
+ }
+ const void* ckv = nullptr;
+ int ckv_fmt = kKvFp4;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ // the groups this window completes: g_first .. g_first + ng - 1
+ const int64_t g_first = p0 / R; // the first group whose last position is >= p0
+ const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(d.ckr, T, HD, HD, fc, s);
+ CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));
+ } else {
+ if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected
+ const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;
+ CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
+ }
+ compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
+ }
+ }
+ Rows lim;
+ int clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ lim.n[t] = (int) ((p0 + t + 1) / R);
+ lim.k[t] = std::min(g.ix_topk, lim.n[t]);
+ clen_max = std::max(clen_max, lim.n[t]);
+ }
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),
+ (size_t) ng * ik_rb(), l.dev, err))
+ return false;
+ }
+ if (clen_max > 0) {
+ if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) p0;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ // (iw rows are 64 apart)
+ gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);
+ const long long ss = (long long) o.max_context + 64;
+ bool restrict_ = false;
+ for (int t = 0; t < T; ++t)
+ restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;
+ if (l.uses_cand && restrict_) {
+ Rows cl; // the candidate positions of each query: its blocks x blk
+ int nmax = 0;
+ for (int t = 0; t < T; ++t) {
+ cl.n[t] = lim.n[t];
+ cl.k[t] = wp.cand[l.il][t];
+ nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);
+ }
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,
+ (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);
+ Rows tr;
+ for (int t = 0; t < T; ++t) {
+ tr.n[t] = wp.cand[l.il][t] * g.cand_blk;
+ tr.k[t] = lim.k[t];
+ }
+ topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);
+ } else {
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,
+ lim, d.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr;
+ for (int t = 0; t < T; ++t) {
+ cr.n[t] = lim.n[t];
+ cr.k[t] = g.cand_topk;
+ }
+ cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,
+ (int) cand_stride(), s);
+ }
+ topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(d.latent, ng, HD, HD, fc, s);
+ if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),
+ (size_t) ng * ckv_rb(), l.dev, err))
+ return false;
+ if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.x_stage;
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.x_stage + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
+ }
+ }
+ for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection
+ ckv = d.ext + (size_t) g.win * HD;
+ ckv_fmt = kKvBf16;
+ iota_rows(d.topk, T, T, (int) topk_stride(), s);
+ }
+ if (NHL > 0) {
+ sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,
+ 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) p0;
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(d.o, T * NHL, HD, HD, fo, s);
+ x3::MvJob j[16];
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);
+ if (!mv_n(d, j, OG, err)) return false;
+ if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;
+ } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)
+ if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
+ prof_mark(d, 1);
+ // the MoE
+ hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
+ hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);
+ hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
+ if (wp.async) {
+ // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members
+ // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the
+ // router made), and waits for the answer only before the experts' sum
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ const unsigned sq = wp.seq[l.il];
+ const int slot = (int) (sq % kRing);
+ const bool lead = mi == 0;
+ int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below
+ for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {
+ const Layer* nx = ahead_layer(L0, a);
+ if (nx == nullptr) break;
+ const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);
+ float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;
+ gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN
+ // weights instead: the same hit rate)
+ route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);
+ na[a - 1] = m;
+ }
+ route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,
+ d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],
+ na[1]);
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;
+ swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;
+ prof_mark(d, 2);
+ const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;
+ if (T == 1 && moe_token_path) {
+ x3::MoeArgs ma;
+ ma.plan_ptr = d.plan_ptr;
+ ma.plan_w = d.rw;
+ ma.cpu_part = lead ? resps[slot].part[0] : nullptr;
+ ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;
+ ma.lay = lay;
+ ma.k = l.k_exp;
+ ma.n_embd = D;
+ ma.n_ff = FF;
+ ma.limit = g.lim;
+ const bool mx = lay.fmt == gg::kFmtMXFP4;
+ if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
+ else x3::moe_gate_up(ma, d.x, d.xg, s);
+ prof_mark(d, 3);
+ if (lead) wait_seq(&resps[slot].seq, sq, s);
+ prof_mark(d, 4);
+ if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
+ else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
+ } else { // each distinct expert read once for the window's tokens
+ gf::MoeWin mw;
+ mw.plan_ptr = d.plan_ptr;
+ mw.T = T;
+ mw.k = l.k_exp;
+ mw.grp_ptr = d.grp_ptr;
+ mw.grp_start = d.grp_start;
+ mw.grp_ent = d.grp_ent;
+ mw.n_groups = d.n_groups;
+ gf::moe_win_group(mw, s);
+ x3::WinArgs wa;
+ wa.grp_ptr = d.grp_ptr;
+ wa.grp_start = d.grp_start;
+ wa.grp_ent = d.grp_ent;
+ wa.n_groups = d.n_groups;
+ wa.plan_ptr = d.plan_ptr;
+ wa.plan_w = d.rw;
+ wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;
+ wa.cpu_stride = kMaxDim;
+ wa.cpu_flag = lead ? resps[slot].cpu : nullptr;
+ wa.wait_ptr = lead ? &resps[slot].seq : nullptr;
+ wa.wait_val = sq;
+ wa.x = d.x;
+ wa.x_stride = (size_t) D;
+ wa.h = d.xg;
+ wa.part = d.ed;
+ wa.sh_out = d.sh_out;
+ wa.out = d.moe_out;
+ wa.out_stride = (size_t) D;
+ wa.lay = lay;
+ wa.T = T;
+ wa.k = l.k_exp;
+ wa.n_embd = D;
+ wa.n_ff = FF;
+ wa.limit = g.lim;
+ if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {
+ err = "exl3 win_experts failed";
+ return false;
+ }
+ }
+ if (lead) st.experts += (int64_t) T * l.k_exp;
+ if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 5);
+ return true;
+ }
+ if (nm > 1) {
+ err = "a tensor-parallel layer on the host-routed path";
+ return false;
+ }
+ // (glog rows are 1024 apart: route reads them so)
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ VlBits vb = {};
+ vb.w[0] = l.mtp ? 0u : wp.vl;
+ route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,
+ vb.w[0] != 0 ? &vb : nullptr);
+ CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaEventRecord(d.ev, s));
+ // the shared expert while the host looks at the picks
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;
+ swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;
+ const double tw = now_ms();
+ CK(cudaEventSynchronize(d.ev));
+ st.gpu_wait_ms += now_ms() - tw;
+ // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the
+ // CPU lane's entries
+ Combine cm;
+ Gather ga;
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];
+ int nvex = 0, nrows = 0;
+ std::vector<Ent> ents;
+ std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)
+ std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)
+ for (int t = 0; t < T; ++t) {
+ cm.n[t] = 0;
+ cm.cpu[t] = 0;
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = h_idx[t * kPicks + i];
+ if (e < 0 || e >= l.n_exp) {
+ err = "router picked expert " + std::to_string(e);
+ return false;
+ }
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
+ per[(size_t) e].push_back({t, i});
+ }
+ }
+ int nslots = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const auto& pe = per[(size_t) e];
+ if (pe.empty()) continue;
+ if (l.slot[(size_t) e] >= 0) {
+ vex[nvex] = e;
+ vrow0[nvex] = nrows;
+ vn[nvex] = (int) pe.size();
+ for (const auto& tp : pe) {
+ ga.src[nrows] = tp.first;
+ const int t = tp.first;
+ cm.row[t][cm.n[t]] = nrows;
+ cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];
+ ++cm.n[t];
+ ++nrows;
+ }
+ ++nvex;
+ } else {
+ for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {
+ Ent en;
+ en.e = e;
+ en.slot0 = nslots;
+ en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);
+ for (int i = 0; i < en.n; ++i) {
+ en.tok[i] = pe[a + (size_t) i].first;
+ cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});
+ cm.cpu[en.tok[i]] = 1;
+ }
+ nslots += en.n;
+ ents.push_back(en);
+ }
+ }
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ st.cpu_experts += nslots;
+ if (nrows > 0) {
+ gather_rows(d.x, ga, nrows, D, d.xe, s);
+ x3::MvJob j[2 * kWin * kPicks];
+ int nj = 0;
+ for (int i = 0; i < nvex; ++i) {
+ const auto& vm = l.vm[(size_t) vex[i]];
+ j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ }
+ if (!mv_n(d, j, nj, err)) return false;
+ swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);
+ nj = 0;
+ for (int i = 0; i < nvex; ++i)
+ j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,
+ vn[i], g.ff, D);
+ if (!mv_n(d, j, nj, err)) return false;
+ }
+ if (!ents.empty()) {
+ const double tc = now_ms();
+ cpu_experts(l, h_x, ents);
+ std::fill(h_part, h_part + (size_t) T * D, 0.0f);
+ for (int sl = 0; sl < nslots; ++sl) {
+ float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
+ const float wv = cpu_rows[(size_t) sl].second;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv * y[e];
+ }
+ st.cpu_ms += now_ms() - tc;
+ CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));
+ }
+ combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
+ return true;
+ }
+
+ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::kernels::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::kernels::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
+ sp, d.tok, d.st);
+ int t = -1;
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ bool logits(std::vector<float>& out) {
+ if (logits_all) {
+ out = chunk_logits;
+ return true;
+ }
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ out.resize((size_t) std::max(1, n_logits) * g.vocab);
+ return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
+ }
+
+ void save_profile() const {
+ if (o.profile.empty()) return;
+ const std::string tmp = o.profile + ".tmp";
+ std::FILE* f = std::fopen(tmp.c_str(), "wb");
+ if (f == nullptr) return;
+ const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();
+ std::fclose(f);
+ // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile
+ // never moved on and only .tmp files were left)
+ if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());
+ }
+
+ std::vector<std::string> info_lines() const {
+ std::vector<std::string> v;
+ int nh = 0;
+ for (const Layer& l : L) nh += l.n_exp - l.n_vram;
+ char b[512];
+ // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)
+ std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "
+ "cpu_experts=%d ram_experts=%d engine=ds4-0.1",
+ (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);
+ v.push_back(b);
+ if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)
+ std::snprintf(b, sizeof b,
+ "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "
+ "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "
+ "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);
+ v.push_back(b);
+ std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),
+ pagefile_hint);
+ v.push_back(b);
+ }
+ if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor
+ std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,
+ pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);
+ v.push_back(b);
+ }
+ if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)
+ std::snprintf(b, sizeof b,
+ "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "
+ "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",
+ vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,
+ (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),
+ (long long) (vt.device_bytes() >> 20));
+ v.push_back(b);
+ } else if (!vision_err.empty()) { // (the server offers no pictures and says why)
+ std::string why = vision_err;
+ for (char& ch : why)
+ if (ch == '\n' || ch == '\r') ch = ' ';
+ v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");
+ v.push_back("INFO vision=off");
+ }
+ return v;
+ }
+};
+
+Model::Model() : p_(new Impl) {}
+Model::~Model() = default;
+bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }
+bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }
+bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
+ const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
+ const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
+ : p.feed_windows(tokens, n, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ return p.feed_windows(tokens, n, logits_rows, err);
+}
+bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ const double t0 = now_ms();
+ if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
+ const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;
+ p.checkpoint(slot);
+ const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)
+ return ok;
+}
+int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
+void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
+int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
+ return p_->sample(t, tp, tk, mp, seed, counter);
+}
+bool Model::logits(std::vector<float>& out) { return p_->logits(out); }
+int Model::image_token() const { return p_->img_tok; }
+int Model::dim() const { return p_->g.dim; }
+bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {
+ return p_->encode_image(rgb, width, height, span, err);
+}
+void Model::set_images(std::vector<Image> images) {
+ std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });
+ p_->imgs = std::move(images);
+}
+int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }
+bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,
+ float mp, uint64_t seed, uint64_t counter) {
+ return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);
+}
+bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,
+ int& accepted, int& next, std::string& err) {
+ return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);
+}
+int64_t Model::pos() const { return p_->pos; }
+int64_t Model::max_context() const { return p_->o.max_context; }
+int Model::vocab() const { return p_->g.vocab; }
+const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
+void Model::reset() { p_->reset(); }
+void Model::checkpoint(int slot) { p_->checkpoint(slot); }
+bool Model::restore(int64_t limit) { return p_->restore(limit); }
+int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
+void Model::save_profile() const { p_->save_profile(); }
+const Stats& Model::stats() const { return p_->st; }
+std::string Model::describe() const { return p_->describe(); }
+std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
new file mode 100644
index 000000000..c6b255a5c
--- /dev/null
+++ b/src/dsv41/ds4_model.hpp
@@ -0,0 +1,153 @@
+// src/ds4/ds4_model.hpp - DeepSeek-V4.1-Flash on Maya: an EXL3 model (tools/ds4/pack.py's pack beside its files) over
+// the machine's GPUs, layer-split, with the routed experts that do not fit in VRAM on the CPU lane.
+//
+// Windows of up to 16 tokens through every layer (a prompt in windows, a decode step a window of one): each query of a
+// window sees exactly what the reference's decode path gives it (DeepSeek's inference/model.py at start_pos > 0) -
+// its own 128-token window, the compressed groups completed by its position. The state a token leaves is the
+// 128-token window ring of every layer, the compressors' partial groups, the compressed KV and index keys of the KV
+// source layers (append-only), and the Engram n-gram history; checkpoint() / restore() keep the first two (the rest is
+// truncated by position) so a conversation can go back to the end of its last prompt.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+struct Options {
+ std::string pack; // <model>/maya-ds4
+ int64_t max_context = 32768;
+ std::vector<int> gpus; // empty: every visible GPU
+ int64_t reserve_mib = 600; // VRAM left free on each GPU
+ double vram_experts = 1.0; // the share of the expert budget to use (tests: less, so the CPU lane works)
+ int cpu_threads = 0; // 0: the physical cores
+ std::string profile; // expert hit counts (read to place the hottest experts in VRAM, written back)
+ bool expert_cache = true; // experts move between VRAM and the CPU lane as they are used
+ // the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of
+ // copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold them
+ int mmap_experts = -1;
+ bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
+ float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this
+ // tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the
+ // heads, of the output projection and of every expert's FFN, summed through host-mapped memory (a GPU unlike the
+ // next stays alone); tp > 1 runs with threads (--tp 1 --threads off: one host thread, layer split only)
+ int tp = 2;
+ bool threads = false; // a host thread per GPU enqueues its layers (the hand-offs host-mapped)
+ // prompts in chunks of this many tokens through each layer at once (threads): the GEMMs on the tensor cores, the
+ // CPU lane's experts streamed to the GPUs a chunk at a time instead of computed on the host; 0: windows of 16;
+ // -1: 2048, or 8192 with the experts read in place (every chunk streams them all from the files)
+ int prefill_chunk = -1;
+ bool vision = true; // the vision tower (the checkpoint's vision.* / aligner.*), when it has one
+ bool vision_only = false; // (tests) load the vision tower alone: encode_image() and nothing else
+ bool verbose = false;
+};
+
+/// A picture in the prompt: its span's rows (every position of the span carries the image token), BF16 [span][dim]
+struct Image {
+ int64_t start = 0;
+ std::shared_ptr<const std::vector<uint16_t>> rows;
+};
+
+struct Stats {
+ int64_t tokens = 0;
+ int64_t experts = 0, cpu_experts = 0; // routed experts run, and how many of them on the CPU lane
+ double cpu_ms = 0.0, gpu_wait_ms = 0.0; // the CPU lane's time; the host's waits for the routing
+ double layer_ms = 0.0, head_ms = 0.0, engram_ms = 0.0; // the layers' host time in all; the head's wait; Engram reads
+ int64_t eg_rows = 0, eg_hits = 0, eg_reads = 0; // Engram rows asked for, served without a read, read from the files
+ double eg_io_ms = 0.0; // the windows' Engram reads, issue to last done (mostly hidden)
+ int64_t eg_cached = 0, eg_cache_cap = 0; // the row cache's rows held, its capacity
+ int64_t swaps = 0; // experts the cache moved into VRAM
+ int64_t skipped = 0; // routed experts STRATA_DS4_DISK_SKIP_W left out (light, not in RAM)
+ int64_t ram_experts = 0, ram_moves = 0; // CPU-lane experts read from the RAM tier; experts moved into it
+ double prof_ms[6] = {}; // DS4_PROF: the GPU's time in decode by stage (between layers, attention, routing +
+ int64_t prof_tokens = 0; // shared expert, experts' gate / up, the wait for the CPU lane, down + post)
+ // lookahead (decode windows of one): the picks of layers guessed a layer before (the next layer's gate on this
+ // layer's FFN input), of them in the guess's top k / in all its guesses / in the guess two layers before; the picks
+ // read from the files, of them prefetched early; experts prefetched early
+ int64_t ahead_picks = 0, ahead_hit_k = 0, ahead_hit = 0, ahead_hit2 = 0;
+ int64_t ahead_cold = 0, ahead_cold_hit = 0, ahead_pf = 0;
+ int64_t ahead_rank_n[16] = {}, ahead_rank_hit[16] = {}; // the next-layer guesses by rank, and how many were picked
+ int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;
+ // per draft position (the 1st .. 8th of a round): verified with all the drafts before it accepted, and accepted
+ int64_t draft_reached[8] = {}, draft_kept[8] = {};
+ double draft_ms = 0.0, verify_ms = 0.0; // the drafter's time, the verification windows' time
+};
+
+class Model {
+public:
+ Model();
+ ~Model();
+ bool load(const Options& o, std::string& err);
+
+ /// Feeds `token` at position pos(); with logits, the next token's logits are left on the head's device.
+ bool step(int token, bool logits, std::string& err);
+ /// Feeds n tokens (windows of up to 16, or a long run in chunks of prefill_chunk); the last `logits_rows` of them get
+ /// logits (logits_rows > 16: only a run of one chunk or window).
+ bool feed(const int* tokens, int n, int logits_rows, std::string& err);
+ /// feed() of a prompt's end that also keeps the state before its last token in checkpoint `slot` (a prompt's
+ /// last token then rides along in the run of chunks instead of a window of its own); the same as
+ /// feed(n - 1), checkpoint(slot), feed(1)
+ bool feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err);
+ /// The next token from the last feed's last logit row (greedy when temperature <= 0).
+ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter);
+ /// The last feed's logit rows on the host (rows x vocab floats).
+ bool logits(std::vector<float>& out);
+
+ /// The DSpark drafter (speculative decoding): its block size (0: not loaded), drafts for the positions after `y`
+ /// (y: the token at pos(), not fed yet) with their confidences, and the verification of [y, drafts]: how many of the
+ /// drafts stand and the token after them (the state then holds y and the accepted drafts). The drafts are the
+ /// drafter's argmax; a sampled request passes its chain (the same temperature / top_p / top_k / min_p / seed /
+ /// counter as its verify()) so that with DS4_DSPARK_SAMPLE=1 they are drawn from the drafter's distribution, as
+ /// DeepSeek's reference does. verify() keeps a draft d with probability min(1, p(d) / q(d)) over the request's
+ /// chain - the speculative-sampling rule, so every token is still distributed as the model's own draw.
+ int draft_block() const;
+ bool draft(int y, std::vector<int>& drafts, std::vector<float>& conf, std::string& err, float temperature = 0.0f,
+ float top_p = 1.0f, int top_k = 0, float min_p = 0.0f, uint64_t seed = 0, uint64_t counter = 0);
+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err);
+
+ /// Tokens a feed call takes at a time to use the chunked prompt path (all of a prompt: its chunks run as a
+ /// pipeline; 16 without it).
+ int feed_unit() const;
+ /// Called from feed() while a run of chunks goes through the layers, with the tokens done so far.
+ void set_progress(std::function<void(int64_t)> f);
+ int64_t pos() const;
+ int64_t max_context() const;
+ int vocab() const;
+ int dim() const;
+ const std::vector<int32_t>& tokens() const; // what the state has consumed, in order
+
+ void reset();
+ /// Keep the state at the current position in checkpoint slot 0 or 1 (the end of a prompt, the position before it);
+ /// restore(limit) goes back to the latest one at or before limit (-1: none).
+ void checkpoint(int slot = 0);
+ bool restore(int64_t limit);
+ int64_t restorable(int64_t limit) const;
+
+ /// The vision tower: the token every position of an image's span carries (-1: no tower).
+ int image_token() const;
+ /// An image's pixels (RGB rows, width x height, both multiples of the patch size: the server's resize) -> the
+ /// rows of its span, BF16 [n_h * (n_w + 1) + 2][dim]: image_start, n_h rows of (n_w aligner rows, image_newline),
+ /// image_end.
+ bool encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err);
+ /// The pictures of the prompt about to be fed ({}: none): positions inside a span read its rows (and route with
+ /// the image bias, and take no part in the Engram n-grams); the rest the embedding table's.
+ void set_images(std::vector<Image> images);
+
+ void save_profile() const;
+ const Stats& stats() const;
+ std::string describe() const;
+ /// INFO / VRAM lines for the server's Monitor.
+ std::vector<std::string> info_lines() const;
+
+ struct Impl;
+
+private:
+ std::unique_ptr<Impl> p_;
+};
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_mx_kern.inc b/src/dsv41/ds4_mx_kern.inc
new file mode 100644
index 000000000..24bd08213
--- /dev/null
+++ b/src/dsv41/ds4_mx_kern.inc
@@ -0,0 +1,249 @@
+// src/ds4/ds4_mx_kern.inc - the CPU lane's int8 MXFP4 rows (ds4_gguf_cpu.hpp: mx_rows), one body for the x86 tiers.
+// ds4_gguf_cpu.cpp includes it once a tier, inside its unnamed namespace, after <immintrin.h>, with
+// MXK_NS the tier's namespace
+// MXK_ATTR the tier's instruction set as a function attribute (GCC / Clang: target(...); MSVC: nothing - it takes
+// any intrinsic) on every function here, so the file is built for the baseline and each tier's code runs
+// only where ds4_gguf_cpu.cpp's CPU check says it can
+// MXK_256 or MXK_512, and MXK_VNNI (vpdpbusd; else vpmaddubsw + vpmaddwd)
+// and gets MXK_NS::rows(...) (and with MXK_256 and no MXK_VNNI, MXK_NS::quant(...)); the macros are undefined at the end.
+//
+// The arithmetic is the device's (ds4_gguf.cu, mx_dot) - a block's 32 codes times the int8 activations summed exactly
+// in integers, times (weight scale x activation scale) into F32 sums - in ONE order for every tier, the scalar one
+// included (ds4_gguf_cpu.cpp's rows_q_scalar), so every tier gives the same bits and the choice of tier never changes
+// an answer:
+// * the blocks go in groups of 8; block j of a group adds into accumulator L(j), L = {0, 4, 1, 5, 2, 6, 3, 7} (the
+// order a 256-bit hadd tree leaves the 8 blocks' totals in: lanes = blocks 0, 2, 4, 6, 1, 3, 5, 7):
+// acc[L(j)] = fma(float(total_j), dw_j * dx_j, acc[L(j)]), the groups in turn;
+// * the row is ((acc0 + acc4) + (acc2 + acc6)) + ((acc1 + acc5) + (acc3 + acc7)) (hsum256's tree).
+// The codes come out of a shuffle table as w + 12 (0..24, unsigned bytes), so the products go through vpmaddubsw /
+// vpdpbusd as u8 x s8; each block's total then gets the activation's -12 x (sum of its 32 values) back, exactly.
+//
+// The activation of a call (mx_act_bytes(k): k % 256 == 0 here; quant() / ds4_gguf_cpu.cpp's quant_scalar):
+// [0, k) int8 values in quads of 4 blocks, quad g at 128 g: the 4 blocks' values 0..15 (block j at
+// +16 j), then their 16..31 - one 256-bit step takes a pair of blocks, one 512-bit step a quad
+// [k, k + k/8) F32 scale of each block, group G (8 blocks) at 8 G in lane order (blocks 0, 2, 4, 6, 1, 3, 5, 7)
+// [k + k/8, + k/8) int32 -12 x (sum of the block's 32 values), the same order
+namespace MXK_NS {
+
+constexpr int kBlockBytes = 17; // E8M0 scale + 16 code bytes
+
+MXK_ATTR inline float hsum256(__m256 v) {
+ const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));
+ const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));
+ return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));
+}
+// the 8 weight scales of a group (its first block at p), in lane order
+MXK_ATTR inline __m256 group_dw(const uint8_t* p, const float* e8) {
+ return _mm256_setr_ps(e8[p[0]], e8[p[2 * kBlockBytes]], e8[p[4 * kBlockBytes]], e8[p[6 * kBlockBytes]],
+ e8[p[kBlockBytes]], e8[p[3 * kBlockBytes]], e8[p[5 * kBlockBytes]], e8[p[7 * kBlockBytes]]);
+}
+
+#if defined(MXK_256)
+// ---------------------------------------------------------------- 256-bit: a pair of blocks a step
+// a pair's codes (a1: values 0..15 of both blocks, a2: 16..31) times x1 / x2: 8 int32 lanes, block A's 4 then B's 4
+MXK_ATTR inline __m256i pair_dot(__m256i a1, __m256i a2, __m256i x1, __m256i x2) {
+#if defined(MXK_VNNI)
+ return _mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), a1, x1), a2, x2);
+#else
+ const __m256i p = _mm256_add_epi16(_mm256_maddubs_epi16(a1, x1), _mm256_maddubs_epi16(a2, x2));
+ return _mm256_madd_epi16(p, _mm256_set1_epi16(1));
+#endif
+}
+
+template <int N>
+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,
+ const float* e8, int pf) {
+ const __m256i lut = _mm256_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0, //
+ 12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0);
+ const __m256i m4 = _mm256_set1_epi8(0x0f);
+ const int ng = k / 256;
+ const int8_t* xq[N];
+ const float* xd[N];
+ const int32_t* xc[N];
+ for (int i = 0; i < N; ++i) {
+ const uint8_t* a = (const uint8_t*) act[i];
+ xq[i] = (const int8_t*) a;
+ xd[i] = (const float*) (a + k);
+ xc[i] = (const int32_t*) (a + k + k / 8);
+ }
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* p = w + (size_t) r * rb;
+ __m256 acc[N];
+ for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();
+ for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {
+ if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);
+ const __m256 dw = group_dw(p, e8);
+ // the group's 4 pairs of blocks (2 q, 2 q + 1) decoded once; then each activation in turn: its 4 pair dots,
+ // the hadd tree to the 8 blocks' totals (lanes: blocks 0 2 4 6 | 1 3 5 7), the FMA - only the accumulators
+ // stay live from one activation to the next (16 YMM registers on AVX2)
+ __m256i a1[4], a2[4];
+ for (int q = 0; q < 4; ++q) {
+ const uint8_t* pb = p + 2 * q * kBlockBytes;
+ const __m256i c = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*) (pb + 1))),
+ _mm_loadu_si128((const __m128i*) (pb + 18)), 1);
+ a1[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(c, m4));
+ a2[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(_mm256_srli_epi16(c, 4), m4));
+ }
+ const int o = G * 256; // (quads 2 G and 2 G + 1: pair q at (q / 2) * 128 + (q % 2) * 32)
+ for (int i = 0; i < N; ++i) {
+ const int8_t* x = xq[i] + o;
+ const __m256i s0 = pair_dot(a1[0], a2[0], _mm256_loadu_si256((const __m256i*) x),
+ _mm256_loadu_si256((const __m256i*) (x + 64)));
+ const __m256i s1 = pair_dot(a1[1], a2[1], _mm256_loadu_si256((const __m256i*) (x + 32)),
+ _mm256_loadu_si256((const __m256i*) (x + 96)));
+ const __m256i s2 = pair_dot(a1[2], a2[2], _mm256_loadu_si256((const __m256i*) (x + 128)),
+ _mm256_loadu_si256((const __m256i*) (x + 192)));
+ const __m256i s3 = pair_dot(a1[3], a2[3], _mm256_loadu_si256((const __m256i*) (x + 160)),
+ _mm256_loadu_si256((const __m256i*) (x + 224)));
+ const __m256i u = _mm256_hadd_epi32(_mm256_hadd_epi32(s0, s1), _mm256_hadd_epi32(s2, s3));
+ const __m256i tot = _mm256_add_epi32(u, _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));
+ acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);
+ }
+ }
+ for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);
+ }
+}
+
+#if !defined(MXK_VNNI)
+// blocks [b0, b1) of x (x at block b0's first value) into act, the bytes of quant_scalar (ds4_gguf_cpu.cpp): the same
+// IEEE division and round-half-even conversion, the same max
+MXK_ATTR inline void quant(const float* x, int b0, int b1, uint8_t* act, int k) {
+ int8_t* q = (int8_t*) act;
+ float* dx = (float*) (act + k);
+ int32_t* corr = (int32_t*) (act + k + k / 8);
+ const __m256 sign = _mm256_set1_ps(-0.0f);
+ const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
+ for (int b = b0; b < b1; ++b, x += 32) {
+ const __m256 v0 = _mm256_loadu_ps(x), v1 = _mm256_loadu_ps(x + 8);
+ const __m256 v2 = _mm256_loadu_ps(x + 16), v3 = _mm256_loadu_ps(x + 24);
+ const __m256 m = _mm256_max_ps(_mm256_max_ps(_mm256_andnot_ps(sign, v0), _mm256_andnot_ps(sign, v1)),
+ _mm256_max_ps(_mm256_andnot_ps(sign, v2), _mm256_andnot_ps(sign, v3)));
+ __m128 m2 = _mm_max_ps(_mm256_castps256_ps128(m), _mm256_extractf128_ps(m, 1));
+ m2 = _mm_max_ps(m2, _mm_movehl_ps(m2, m2));
+ m2 = _mm_max_ss(m2, _mm_shuffle_ps(m2, m2, 1));
+ const float am = _mm_cvtss_f32(m2);
+ const float sc = am / 127.0f;
+ int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);
+ int8_t* hi = lo + 64;
+ int s = 0;
+ if (am > 0.0f) {
+ const __m256 vs = _mm256_set1_ps(sc);
+ const __m256i q0 = _mm256_cvtps_epi32(_mm256_div_ps(v0, vs)), q1 = _mm256_cvtps_epi32(_mm256_div_ps(v1, vs));
+ const __m256i q2 = _mm256_cvtps_epi32(_mm256_div_ps(v2, vs)), q3 = _mm256_cvtps_epi32(_mm256_div_ps(v3, vs));
+ const __m256i pk = _mm256_permutevar8x32_epi32(
+ _mm256_packs_epi16(_mm256_packs_epi32(q0, q1), _mm256_packs_epi32(q2, q3)), order);
+ _mm_storeu_si128((__m128i*) lo, _mm256_castsi256_si128(pk));
+ _mm_storeu_si128((__m128i*) hi, _mm256_extracti128_si256(pk, 1));
+ __m256i t = _mm256_add_epi32(_mm256_add_epi32(q0, q1), _mm256_add_epi32(q2, q3));
+ __m128i t2 = _mm_add_epi32(_mm256_castsi256_si128(t), _mm256_extracti128_si256(t, 1));
+ t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0x4E));
+ t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0xB1));
+ s = _mm_cvtsi128_si32(t2);
+ } else {
+ _mm_storeu_si128((__m128i*) lo, _mm_setzero_si128());
+ _mm_storeu_si128((__m128i*) hi, _mm_setzero_si128());
+ }
+ const int j = b & 7, lane = 8 * (b >> 3) + (j & 1) * 4 + (j >> 1);
+ dx[lane] = sc;
+ corr[lane] = -12 * s;
+ }
+}
+#endif
+#endif // MXK_256
+
+#if defined(MXK_512)
+// ---------------------------------------------------------------- 512-bit: a quad of blocks a step
+MXK_ATTR inline __m512i quad_dot(__m512i a1, __m512i a2, __m512i x1, __m512i x2) {
+#if defined(MXK_VNNI)
+ return _mm512_dpbusd_epi32(_mm512_dpbusd_epi32(_mm512_setzero_si512(), a1, x1), a2, x2);
+#else
+ const __m512i p = _mm512_add_epi16(_mm512_maddubs_epi16(a1, x1), _mm512_maddubs_epi16(a2, x2));
+ return _mm512_madd_epi16(p, _mm512_set1_epi16(1));
+#endif
+}
+
+template <int N>
+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,
+ const float* e8, int pf) {
+ const __m512i lut = _mm512_broadcast_i32x4(_mm_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0));
+ const __m512i m4 = _mm512_set1_epi8(0x0f);
+ // a quad's 16 lanes after adjacent-pair sums: block j's halves in lanes 4 j and 4 j + 2; the group's two quads ->
+ // the 8 blocks in lane order (0 2 4 6 1 3 5 7), first halves (pa) and second halves (pb)
+ const __m512i pa = _mm512_setr_epi32(0, 8, 16, 24, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0);
+ const __m512i pb = _mm512_setr_epi32(2, 10, 18, 26, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0);
+ const int ng = k / 256;
+ const int8_t* xq[N];
+ const float* xd[N];
+ const int32_t* xc[N];
+ for (int i = 0; i < N; ++i) {
+ const uint8_t* a = (const uint8_t*) act[i];
+ xq[i] = (const int8_t*) a;
+ xd[i] = (const float*) (a + k);
+ xc[i] = (const int32_t*) (a + k + k / 8);
+ }
+ for (int r = r0; r < r1; ++r) {
+ const uint8_t* p = w + (size_t) r * rb;
+ __m256 acc[N];
+ for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();
+ for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {
+ if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);
+ const __m256 dw = group_dw(p, e8);
+ __m512i h0[N];
+ for (int qd = 0; qd < 2; ++qd) { // the group's quads: blocks 4 qd .. 4 qd + 3
+ const uint8_t* pq = p + 4 * qd * kBlockBytes;
+ __m512i c = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*) (pq + 1)));
+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 18)), 1);
+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 35)), 2);
+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 52)), 3);
+ const __m512i a1 = _mm512_shuffle_epi8(lut, _mm512_and_si512(c, m4));
+ const __m512i a2 = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(c, 4), m4));
+ const int o = (2 * G + qd) * 128;
+ for (int i = 0; i < N; ++i) {
+ const __m512i s = quad_dot(a1, a2, _mm512_loadu_si512((const void*) (xq[i] + o)),
+ _mm512_loadu_si512((const void*) (xq[i] + o + 64)));
+ const __m512i h = _mm512_add_epi32(s, _mm512_shuffle_epi32(s, (_MM_PERM_ENUM) 0xB1));
+ if (qd == 0) {
+ h0[i] = h;
+ continue;
+ }
+ const __m512i ta = _mm512_permutex2var_epi32(h0[i], pa, h), tb = _mm512_permutex2var_epi32(h0[i], pb, h);
+ const __m256i tot = _mm256_add_epi32(
+ _mm256_add_epi32(_mm512_castsi512_si256(ta), _mm512_castsi512_si256(tb)),
+ _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));
+ acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);
+ }
+ }
+ }
+ for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);
+ }
+}
+#endif // MXK_512
+
+// y[i][r - r0] for every i < n, in calls of up to 8 activations; e8 the 256 E8M0 scales as F32 (2^(e - 128)), pf the
+// bytes to prefetch ahead in a row (0: none)
+MXK_ATTR inline void rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,
+ float* const* y, const float* e8, int pf) {
+ for (int i0 = 0; i0 < n; i0 += 8) {
+ float* yy[8];
+ const int c = n - i0 < 8 ? n - i0 : 8;
+ for (int i = 0; i < c; ++i) yy[i] = y[i0 + i];
+ switch (c) {
+ case 1: rows_n<1>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 2: rows_n<2>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 3: rows_n<3>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 4: rows_n<4>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 5: rows_n<5>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 6: rows_n<6>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ case 7: rows_n<7>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ default: rows_n<8>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;
+ }
+ }
+}
+
+} // namespace MXK_NS
+
+#undef MXK_NS
+#undef MXK_ATTR
+#undef MXK_256
+#undef MXK_512
+#undef MXK_VNNI
diff --git a/src/dsv41/ds4_numerics.cuh b/src/dsv41/ds4_numerics.cuh
new file mode 100644
index 000000000..36b1f6849
--- /dev/null
+++ b/src/dsv41/ds4_numerics.cuh
@@ -0,0 +1,63 @@
+// src/ds4/ds4_numerics.cuh - the BF16 and FP8 (E4M3) conversions of maya-ds4's kernels, on CUDA through its
+// intrinsics (cuda_bf16.h / cuda_fp8.h) and on HIP in plain arithmetic that gives the same bits: BF16 round to
+// nearest even; E4M3 (OCP "fn": no infinities, max 448) decoded exactly, rounded to nearest even and saturated.
+#pragma once
+
+#include <cstdint>
+#include <cuda_runtime.h>
+#if !defined(STRATA_USE_HIP)
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+#include <cuda_fp8.h>
+#endif
+
+namespace strata::ds4 {
+
+/// F32 -> BF16 bits, round to nearest even (a NaN stays a NaN).
+__device__ __forceinline__ uint16_t bf16_rn(float f) {
+#if !defined(STRATA_USE_HIP)
+ return __bfloat16_as_ushort(__float2bfloat16_rn(f));
+#else
+ const uint32_t u = __float_as_uint(f);
+ if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 0x40u);
+ return (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);
+#endif
+}
+
+/// An E4M3 code's value (exact in F32).
+__device__ __forceinline__ float e4m3_value(uint8_t q) {
+#if !defined(STRATA_USE_HIP)
+ const __half_raw h = __nv_cvt_fp8_to_halfraw((__nv_fp8_storage_t) q, __NV_E4M3);
+ return __half2float(__half(h));
+#else
+ const uint32_t e = (q >> 3) & 15u, m = q & 7u;
+ float v;
+ if (e == 15u && m == 7u) v = __uint_as_float(0x7fc00000u); // NaN
+ else if (e == 0u) v = (float) m * 0.001953125f; // m 2^-9
+ else v = __uint_as_float(((e + 120u) << 23) | (m << 20)); // (1 + m / 8) 2^(e - 7)
+ return (q & 0x80u) ? -v : v;
+#endif
+}
+
+/// v rounded to the nearest E4M3 value (ties to even), saturated at +-448.
+__device__ __forceinline__ float e4m3_round(float v) {
+#if !defined(STRATA_USE_HIP)
+ const __nv_fp8_storage_t q = __nv_cvt_float_to_fp8(v, __NV_SATFINITE, __NV_E4M3);
+ const __half_raw h = __nv_cvt_fp8_to_halfraw(q, __NV_E4M3);
+ return __half2float(__half(h));
+#else
+ const float a = fabsf(v);
+ if (!(a == a)) return v;
+ float r;
+ if (a >= 448.0f) r = 448.0f;
+ else {
+ const int e = (int) ((__float_as_uint(a) >> 23) & 0xffu) - 127; // a's binade (a > 0)
+ const float step = e < -6 ? 0.001953125f : __uint_as_float((uint32_t) (e - 3 + 127) << 23);
+ r = a == 0.0f ? 0.0f : rintf(a / step) * step; // (a power of two: exact)
+ r = fminf(r, 448.0f);
+ }
+ return v < 0.0f ? -r : r;
+#endif
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hpp
new file mode 100644
index 000000000..2b61860c0
--- /dev/null
+++ b/src/dsv41/ds4_os.hpp
@@ -0,0 +1,436 @@
+// src/ds4/ds4_os.hpp - the few OS services maya-ds4 needs, on Linux and on Windows: files read at an offset and
+// mapped read-only (the experts read in place), page-cache hints, anonymous host memory on a NUMA node, thread pinning,
+// each NUMA node's physical cores, the RAM this process can still get. (On Windows as src/core/expert_source.cpp
+// does it for GLM: no FILE_FLAG_RANDOM_ACCESS - the page cache must keep what it read.)
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <psapi.h> // (K32QueryWorkingSetEx: kernel32's, no psapi.lib)
+#else
+#include <fcntl.h>
+#include <pthread.h>
+#include <sched.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#endif
+
+namespace strata::ds4::os {
+
+using fd_t = intptr_t;
+constexpr fd_t kBadFd = -1;
+
+#if defined(_WIN32)
+inline std::wstring wide(const std::string& s) { // UTF-8 -> UTF-16 (a path with non-ASCII characters)
+ const int n = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, nullptr, 0);
+ std::wstring w((size_t) (n > 0 ? n : 1), L'\0');
+ if (n > 0) MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, &w[0], n);
+ return w;
+}
+#endif
+
+/// A file opened for reading (kBadFd: it could not be).
+inline fd_t open_ro(const std::string& path) {
+#if defined(_WIN32)
+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL, nullptr);
+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+ const int fd = ::open(path.c_str(), O_RDONLY);
+ return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// The same file opened for unbuffered reads (Linux: O_DIRECT - no page cache, the disk straight into the buffer;
+/// kBadFd where the filesystem refuses it, or on Windows, whose unbuffered reads of a mapped file take a slow path)
+inline fd_t open_direct(const std::string& path) {
+#if defined(_WIN32)
+ (void) path;
+ return kBadFd;
+#else
+ const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);
+ return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// [off, off + n) through an unbuffered descriptor into dst, a kDirectAlign-aligned buffer of at least
+/// direct_span(off, n) bytes: the file read from off rounded down, so the bytes land at dst + (off % kDirectAlign).
+/// False if a read fails (the caller reads buffered instead).
+constexpr uint64_t kDirectAlign = 4096;
+inline uint64_t direct_span(uint64_t off, uint64_t n) {
+ const uint64_t a0 = off & ~(kDirectAlign - 1), a1 = (off + n + kDirectAlign - 1) & ~(kDirectAlign - 1);
+ return a1 - a0;
+}
+inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+ (void) f, (void) dst, (void) n, (void) off;
+ return false;
+#else
+ const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;
+ uint8_t* p = (uint8_t*) dst;
+ uint64_t got = 0;
+ while (got < need) {
+ const ssize_t r = ::pread((int) f, p + got, (size_t) std::min<uint64_t>(need - got, 1ull << 30), (off_t) (a0 + got));
+ if (r < 0) return false;
+ if (r == 0) break; // (the file's end: the last aligned read is short)
+ got += (uint64_t) r;
+ if ((uint64_t) r % kDirectAlign != 0) break;
+ }
+ return got >= want;
+#endif
+}
+/// The same file opened for reads from many threads at once (Windows runs the reads of a synchronous handle one at a
+/// time, so an overlapped one; Linux: a plain descriptor, pread is concurrent anyway)
+inline fd_t open_par(const std::string& path) {
+#if defined(_WIN32)
+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, nullptr);
+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;
+#else
+ const int fd = ::open(path.c_str(), O_RDONLY);
+ return fd < 0 ? kBadFd : (fd_t) fd;
+#endif
+}
+/// n bytes at off through open_par's handle (any thread, concurrently)
+inline bool pread_par(fd_t f, void* dst, uint64_t n, uint64_t off) {
+#if defined(_WIN32)
+ struct Ev { // (each thread its own, closed when it ends)
+ HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);
+ ~Ev() {
+ if (h != nullptr) CloseHandle(h);
+ }
+ };
+ static thread_local Ev ev;
+ if (ev.h == nullptr) return false;
+ uint8_t* p = (uint8_t*) dst;
+ while (n > 0) {
+ const DWORD want = (DWORD) std::min<uint64_t>(n, 1ull << 30);
+ OVERLAPPED ov{};
+ ov.Offset = (DWORD) (off & 0xffffffffu);
+ ov.OffsetHigh = (DWORD) (off >> 32);
+ ov.hEvent = ev.h;
+ ResetEvent(ev.h);
+ DWORD got = 0;
+ if (!ReadFile((HANDLE) f, p, want, nullptr, &ov) && GetLastError() != ERROR_IO_PENDING) return false;
+ if (!GetOverlappedResult((HANDLE) f, &ov, &got, TRUE) || got == 0) return false;
+ p += got, off += got, n -= got;
+ }
+ return true;
+#else
+ uint8_t* p = (uint8_t*) dst;
+ while (n > 0) {
+ const ssize_t r = ::pread((int) f, p, (size_t) std::min<uint64_t>(n, 1ull << 30), (off_t) off);
+ if (r <= 0) return false;
+ p += r, off += (uint64_t) r, n -= (uint64_t) r;
+ }
+ return true;
+#endif
+}
+/// `from` renamed to `to`, replacing `to` if it exists (rename(2) does; Windows' rename fails on an existing file, so
+/// MoveFileExW with MOVEFILE_REPLACE_EXISTING)
+inline bool replace_file(const std::string& from, const std::string& to) {
+#if defined(_WIN32)
+ return MoveFileExW(wide(from).c_str(), wide(to).c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0;
+#else
+ return std::rename(from.c_str(), to.c_str()) == 0;
+#endif
+}
+inline void close_fd(fd_t f) {
+ if (f == kBadFd) return;
+#if defined(_WIN32)
+ CloseHandle((HANDLE) f);
+#else
+ ::close((int) f);
+#endif
+}
+inline uint64_t file_size(fd_t f) {
+#if defined(_WIN32)
+ LARGE_INTEGER sz{};
+ return GetFileSizeEx((HANDLE) f, &sz) ? (uint64_t) sz.QuadPart : 0;
+#else
+ struct stat sb;
+ return ::fstat((int) f, &sb) == 0 && sb.st_size > 0 ? (uint64_t) sb.st_size : 0;
+#endif
+}
+/// n bytes at offset off into dst (any thread, concurrently); false if the file ends first or a read fails.
+inline bool pread_full(fd_t f, void* dst, uint64_t n, uint64_t off) {
+ uint8_t* p = (uint8_t*) dst;
+ while (n > 0) {
+ const uint64_t want = std::min<uint64_t>(n, 1ull << 30);
+#if defined(_WIN32)
+ OVERLAPPED ov{};
+ ov.Offset = (DWORD) (off & 0xffffffffu);
+ ov.OffsetHigh = (DWORD) (off >> 32);
+ DWORD got = 0;
+ if (!ReadFile((HANDLE) f, p, (DWORD) want, &got, &ov) || got == 0) return false;
+ const uint64_t r = got;
+#else
+ const ssize_t rr = ::pread((int) f, p, (size_t) want, (off_t) off);
+ if (rr <= 0) return false;
+ const uint64_t r = (uint64_t) rr;
+#endif
+ p += r, off += r, n -= r;
+ }
+ return true;
+}
+/// The whole file mapped read-only (nullptr: it could not be).
+inline void* map_ro(fd_t f, uint64_t bytes) {
+ if (bytes == 0) return nullptr;
+#if defined(_WIN32)
+ HANDLE m = CreateFileMappingW((HANDLE) f, nullptr, PAGE_READONLY, 0, 0, nullptr);
+ if (m == nullptr) return nullptr;
+ void* v = MapViewOfFile(m, FILE_MAP_READ, 0, 0, 0);
+ CloseHandle(m); // (the view keeps the mapping)
+ return v;
+#else
+ void* p = ::mmap(nullptr, (size_t) bytes, PROT_READ, MAP_SHARED, (int) f, 0);
+ return p == MAP_FAILED ? nullptr : p;
+#endif
+}
+inline void unmap(void* p, uint64_t bytes) {
+ if (p == nullptr) return;
+#if defined(_WIN32)
+ (void) bytes;
+ UnmapViewOfFile(p);
+#else
+ ::munmap(p, (size_t) bytes);
+#endif
+}
+/// A hint that [off, off + n) of the file is needed soon (need) or not again soon (!need); nothing where the OS has
+/// no such hint for a file (Windows' page cache reads ahead on its own).
+inline void advise(fd_t f, uint64_t off, uint64_t n, bool need) {
+#if defined(_WIN32)
+ (void) f, (void) off, (void) n, (void) need;
+#else
+ posix_fadvise((int) f, (off_t) off, (off_t) n, need ? POSIX_FADV_WILLNEED : POSIX_FADV_DONTNEED);
+#endif
+}
+
+/// Ranges of a file's mapping about to be read: the OS reads them in now, with large requests side by side, instead of
+/// a page fault at a time later (Windows: PrefetchVirtualMemory - cold expert-sized pieces came in 5x faster on an NVMe;
+/// Linux: madvise WILLNEED, the kernel's read-ahead of the whole range). Best effort: a failure changes nothing.
+inline void prefetch(const std::vector<std::pair<const void*, size_t>>& ranges) {
+ if (ranges.empty()) return;
+#if defined(_WIN32)
+ std::vector<WIN32_MEMORY_RANGE_ENTRY> r(ranges.size());
+ for (size_t i = 0; i < ranges.size(); ++i) {
+ r[i].VirtualAddress = const_cast<void*>(ranges[i].first);
+ r[i].NumberOfBytes = ranges[i].second;
+ }
+ PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) r.size(), r.data(), 0);
+#else
+ const uintptr_t pg = 4096;
+ for (const auto& x : ranges) {
+ const uintptr_t a = (uintptr_t) x.first, b = a & ~(pg - 1);
+ madvise((void*) b, x.second + (a - b), MADV_WILLNEED);
+ }
+#endif
+}
+
+/// Whether every page of [p, p + n) of a file's mapping is in the page cache (Linux mincore; elsewhere false: the
+/// caller reads buffered anyway, which the cache serves)
+inline bool resident(const void* p, size_t n) {
+#if defined(_WIN32)
+ (void) p, (void) n;
+ return false;
+#else
+ const uintptr_t pg = 4096, a = (uintptr_t) p, b = a & ~(pg - 1);
+ const size_t len = n + (a - b), np = (len + pg - 1) / pg;
+ thread_local std::vector<unsigned char> v;
+ if (v.size() < np) v.resize(np);
+ if (mincore((void*) b, len, v.data()) != 0) return false;
+ for (size_t i = 0; i < np; ++i)
+ if ((v[i] & 1) == 0) return false;
+ return true;
+#endif
+}
+
+/// Whether a range of a file's mapping looks resident, so that prefetch() would only cost CPU time (it walks every page:
+/// ~50 ns a 4 KiB page on Windows - 0.34 ms for a 6.3 MB expert matrix on a Strix Halo): `samples` pages spread over
+/// it all resident (Linux: mincore, the page cache; Windows: QueryWorkingSetEx, the process's working set - a page on
+/// the standby list counts as not resident, and prefetch() is what brings those back cheaply). An expert's pages are
+/// read together, so they come and go together: a wrong "yes" costs page faults, a wrong "no" a prefetch.
+inline bool resident_sampled(const void* p, size_t bytes, int samples = 8) {
+ if (bytes == 0) return true;
+ const uintptr_t pg = 4096, a = (uintptr_t) p, span = bytes - 1;
+ samples = std::max(1, std::min(samples, 16));
+#if defined(_WIN32)
+ PSAPI_WORKING_SET_EX_INFORMATION w[16] = {};
+ for (int i = 0; i < samples; ++i)
+ w[i].VirtualAddress = (PVOID) ((a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1));
+ if (!K32QueryWorkingSetEx(GetCurrentProcess(), w, (DWORD) (samples * sizeof w[0]))) return false;
+ for (int i = 0; i < samples; ++i)
+ if (!w[i].VirtualAttributes.Valid) return false;
+ return true;
+#else
+ for (int i = 0; i < samples; ++i) {
+ unsigned char v = 0;
+ const uintptr_t at = (a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1);
+ if (mincore((void*) at, pg, &v) != 0 || !(v & 1)) return false;
+ }
+ return true;
+#endif
+}
+/// prefetch() of the ranges that do not look resident (resident_sampled) - on Windows, where PrefetchVirtualMemory walks
+/// every page of a resident range (0.34 ms a 6.3 MB expert matrix on a Strix Halo); Linux's madvise(WILLNEED) on cached
+/// pages costs ~1 us a matrix (a Core i5-12600T), less than the residency check (2.5 us): every range there
+inline void prefetch_cold(const std::vector<std::pair<const void*, size_t>>& ranges) {
+#if defined(_WIN32)
+ static const bool all = [] { // (DS4_PF_ALL=1: every range, as before)
+ const char* v = std::getenv("DS4_PF_ALL");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ if (all) return prefetch(ranges);
+ std::vector<std::pair<const void*, size_t>> cold;
+ for (const auto& x : ranges)
+ if (!resident_sampled(x.first, x.second)) cold.push_back(x);
+ prefetch(cold);
+#else
+ prefetch(ranges);
+#endif
+}
+
+/// Whether [p, p + n) of a file's mapping is in RAM now: every page (Linux, one mincore), sampled pages of the working
+/// set (Windows; a page on the standby list counts as not in RAM)
+inline bool in_ram(const void* p, size_t n) {
+#if defined(_WIN32)
+ return resident_sampled(p, n, 8);
+#else
+ return resident(p, n);
+#endif
+}
+
+/// Anonymous memory on NUMA node `node` of `nodes` (node < 0 or nodes < 2: wherever the first touch puts it), on huge
+/// pages where the OS gives them; nullptr if it cannot be had.
+inline void* anon_alloc(size_t bytes, int node, int nodes) {
+#if defined(_WIN32)
+ if (node >= 0 && nodes > 1)
+ return VirtualAllocExNuma(GetCurrentProcess(), nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,
+ (DWORD) node);
+ return VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+#else
+ void* m = ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (m == MAP_FAILED) return nullptr;
+ madvise(m, bytes, MADV_HUGEPAGE);
+ if (node >= 0 && nodes > 1) {
+ unsigned long mask[4] = {0, 0, 0, 0};
+ mask[node / 64] = 1ul << (node % 64);
+ syscall(SYS_mbind, m, bytes, 2 /* MPOL_BIND */, mask, 256, 0);
+ }
+ return m;
+#endif
+}
+inline void anon_free(void* p, size_t bytes) {
+ if (p == nullptr) return;
+#if defined(_WIN32)
+ (void) bytes;
+ VirtualFree(p, 0, MEM_RELEASE);
+#else
+ ::munmap(p, bytes);
+#endif
+}
+
+/// The calling thread, or `t`, on logical CPU `cpu` (Windows: the first 64 CPUs, processor group 0).
+inline void pin_self(int cpu) {
+#if defined(_WIN32)
+ if (cpu >= 0 && cpu < 64) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR) 1 << cpu);
+#else
+ cpu_set_t set;
+ CPU_ZERO(&set);
+ CPU_SET(cpu, &set);
+ sched_setaffinity(0, sizeof set, &set);
+#endif
+}
+inline void pin_thread(std::thread& t, int cpu) {
+#if defined(_WIN32)
+ if (cpu >= 0 && cpu < 64) SetThreadAffinityMask((HANDLE) t.native_handle(), (DWORD_PTR) 1 << cpu);
+#else
+ cpu_set_t set;
+ CPU_ZERO(&set);
+ CPU_SET(cpu, &set);
+ pthread_setaffinity_np(t.native_handle(), sizeof(set), &set);
+#endif
+}
+
+#if defined(_WIN32)
+/// Each NUMA node's physical cores (the first logical CPU of each), processor group 0 - Windows' answer to
+/// /sys/devices/system/node; empty if the OS does not say.
+inline std::vector<std::vector<int>> node_cores() {
+ std::vector<std::vector<int>> nodes;
+ auto query = [](LOGICAL_PROCESSOR_RELATIONSHIP rel, std::vector<uint8_t>& buf) {
+ DWORD len = 0;
+ GetLogicalProcessorInformationEx(rel, nullptr, &len);
+ buf.resize(len);
+ return len > 0 && GetLogicalProcessorInformationEx(rel, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len);
+ };
+ std::vector<uint8_t> nb, cb;
+ if (!query(RelationNumaNode, nb) || !query(RelationProcessorCore, cb)) return nodes;
+ std::vector<KAFFINITY> masks;
+ for (size_t o = 0; o < nb.size();) {
+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (nb.data() + o);
+ if (e->NumaNode.GroupMask.Group == 0) masks.push_back(e->NumaNode.GroupMask.Mask);
+ o += e->Size;
+ }
+ nodes.resize(masks.size());
+ for (size_t o = 0; o < cb.size();) {
+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (cb.data() + o);
+ if (e->Processor.GroupCount > 0 && e->Processor.GroupMask[0].Group == 0) {
+ const KAFFINITY m = e->Processor.GroupMask[0].Mask;
+ int first = -1;
+ for (int b = 0; b < 64; ++b)
+ if ((m >> b) & 1) {
+ first = b;
+ break;
+ }
+ for (size_t n = 0; n < masks.size() && first >= 0; ++n)
+ if ((masks[n] >> first) & 1) nodes[n].push_back(first);
+ }
+ o += e->Size;
+ }
+ nodes.erase(std::remove_if(nodes.begin(), nodes.end(), [](const std::vector<int>& v) { return v.empty(); }),
+ nodes.end());
+ for (auto& v : nodes) std::sort(v.begin(), v.end());
+ return nodes;
+}
+/// The RAM this process can still get (~0: not known).
+inline uint64_t ram_avail() {
+ MEMORYSTATUSEX s{};
+ s.dwLength = sizeof(s);
+ return GlobalMemoryStatusEx(&s) ? (uint64_t) s.ullAvailPhys : ~uint64_t{0};
+}
+#endif
+
+/// Windows' commit (RAM + page file, what every private allocation is charged to - on an integrated GPU the GPU's
+/// own allocations too): the limit, what this process can still commit, the RAM; all ~0 where there is no such limit
+/// (Linux overcommits; its RAM is ram_avail's business).
+struct Commit {
+ uint64_t limit = ~uint64_t{0}, avail = ~uint64_t{0}, ram = ~uint64_t{0};
+};
+inline Commit commit() {
+ Commit c;
+#if defined(_WIN32)
+ MEMORYSTATUSEX s{};
+ s.dwLength = sizeof(s);
+ if (GlobalMemoryStatusEx(&s)) {
+ c.limit = (uint64_t) s.ullTotalPageFile;
+ c.avail = (uint64_t) s.ullAvailPageFile;
+ c.ram = (uint64_t) s.ullTotalPhys;
+ }
+#endif
+ return c;
+}
+
+} // namespace strata::ds4::os
diff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hpp
new file mode 100644
index 000000000..9888f5e3b
--- /dev/null
+++ b/src/dsv41/ds4_pool.hpp
@@ -0,0 +1,222 @@
+// src/ds4/ds4_pool.hpp - the DeepSeek engine's CPU workers: run(n, fn) runs fn(0..n-1) on the workers and the caller and
+// returns when every job is done. The design is glmfast::Workers' (src/core/glm_fast_state.hpp): one 64-bit ticket
+// (generation << 32 | next index) claimed by CAS, so a worker that wakes late for an older batch can never take a job
+// of the current one; idle workers spin for a while, then sleep on a condition variable.
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "ds4_os.hpp"
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
+
+namespace strata::ds4 {
+
+inline void cpu_relax() {
+#if defined(_MSC_VER)
+ _mm_pause();
+#elif defined(__x86_64__) || defined(_M_X64)
+ __builtin_ia32_pause();
+#endif
+}
+
+class Pool {
+public:
+ explicit Pool(int n, const std::vector<int>& cpus = {}, int spin_us = 200) : spin_us_(spin_us) {
+ for (int i = 0; i < n; ++i) {
+ th_.emplace_back([this] { loop(); });
+ if (!cpus.empty()) os::pin_thread(th_.back(), cpus[(size_t) i % cpus.size()]);
+ }
+ }
+ ~Pool() {
+ quit_.store(true);
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ }
+ cv_.notify_all();
+ for (auto& t : th_) t.join();
+ }
+ int size() const { return (int) th_.size() + 1; }
+
+ void run(int n, const std::function<void(int)>& fn) {
+ start(n, fn);
+ help();
+ wait();
+ }
+ /// The batch goes to the workers; the caller may help() and must wait() before fn goes out of scope. (Two pools
+ /// - one a NUMA node - run at once this way: start both, help one, wait for both.)
+ void start(int n, const std::function<void(int)>& fn) {
+ n_ = n;
+ if (n <= 0) return;
+ const uint64_t g = (gen_.load(std::memory_order_relaxed) + 1) & 0xffffffffu;
+ ticket_.store((g << 32) | 0x7fffffffu, std::memory_order_seq_cst);
+ fn_ = &fn;
+ total_.store(n, std::memory_order_release);
+ done_.store(0, std::memory_order_relaxed);
+ ticket_.store(g << 32, std::memory_order_release);
+ gen_.store(g, std::memory_order_release);
+ if (sleeping_.load(std::memory_order_acquire) > 0) {
+ std::lock_guard<std::mutex> lk(mu_);
+ cv_.notify_all();
+ }
+ }
+ void help() {
+ if (n_ > 0) work(gen_.load(std::memory_order_acquire));
+ }
+ void wait() {
+ if (n_ <= 0) return;
+ while (done_.load(std::memory_order_acquire) < n_) cpu_relax();
+ n_ = 0;
+ }
+
+private:
+ void work(uint64_t g) {
+ for (;;) {
+ uint64_t t = ticket_.load(std::memory_order_acquire);
+ if ((t >> 32) != g) return;
+ const int n = total_.load(std::memory_order_acquire);
+ if ((int) (t & 0xffffffffu) >= n) return;
+ if (!ticket_.compare_exchange_weak(t, t + 1, std::memory_order_acq_rel)) continue;
+ (*fn_)((int) (t & 0xffffffffu));
+ done_.fetch_add(1, std::memory_order_acq_rel);
+ }
+ }
+ void loop() {
+ uint64_t seen = gen_.load();
+ for (;;) {
+ const auto t0 = std::chrono::steady_clock::now();
+ uint64_t g;
+ for (;;) {
+ g = gen_.load(std::memory_order_acquire);
+ if (g != seen || quit_.load(std::memory_order_relaxed)) break;
+ cpu_relax();
+ if (std::chrono::steady_clock::now() - t0 > std::chrono::microseconds(spin_us_)) {
+ std::unique_lock<std::mutex> lk(mu_);
+ sleeping_.fetch_add(1);
+ cv_.wait(lk, [&] { return gen_.load() != seen || quit_.load(); });
+ sleeping_.fetch_sub(1);
+ }
+ }
+ if (quit_.load()) return;
+ seen = g;
+ work(g);
+ }
+ }
+
+ std::vector<std::thread> th_;
+ int spin_us_;
+ int n_ = 0; // the current batch's size (the caller's side)
+ const std::function<void(int)>* fn_ = nullptr;
+ alignas(64) std::atomic<uint64_t> ticket_{0};
+ alignas(64) std::atomic<uint64_t> gen_{0};
+ alignas(64) std::atomic<int> total_{0};
+ alignas(64) std::atomic<int> done_{0};
+ std::atomic<int> sleeping_{0};
+ std::atomic<bool> quit_{false};
+ std::mutex mu_;
+ std::condition_variable cv_;
+};
+
+/// File reads on threads of their own, in the order they were pushed, the pusher never waiting: the prompt path's
+/// streamed experts (each part of an expert a job), unbuffered where the job has a direct descriptor and its range is
+/// not all in the page cache already (the disk straight into dst, the cache neither used nor disturbed), buffered
+/// otherwise or when the unbuffered read fails - the bytes land at dst + off % os::kDirectAlign either way. A job's `left` is
+/// counted down when it is done (`bad` set if it failed); wait_any() sleeps until some job finished or the timeout.
+class IoQueue {
+public:
+ struct Job {
+ os::fd_t fd = os::kBadFd, dfd = os::kBadFd;
+ uint64_t off = 0, len = 0;
+ uint8_t* dst = nullptr;
+ std::atomic<int>* left = nullptr;
+ std::atomic<bool>* bad = nullptr;
+ const uint8_t* mem = nullptr; // non-null: a copy of len bytes from memory to dst instead of a read
+ const void* map = nullptr; // the range in the file's mapping: all in the page cache, it is read buffered
+ uint8_t* keep = nullptr; // non-null (a read): the len bytes read copied here as well
+ };
+ explicit IoQueue(int n) {
+ for (int i = 0; i < n; ++i) th_.emplace_back([this] { loop(); });
+ }
+ ~IoQueue() {
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ quit_ = true;
+ }
+ cv_.notify_all();
+ for (auto& t : th_) t.join();
+ }
+ void push(const Job* jobs, int n) {
+ {
+ std::lock_guard<std::mutex> lk(mu_);
+ for (int i = 0; i < n; ++i) q_.push_back(jobs[i]);
+ }
+ if (n == 1) cv_.notify_one();
+ else cv_.notify_all();
+ }
+ void wait_any(int us) {
+ std::unique_lock<std::mutex> lk(done_mu_);
+ const uint64_t seen = done_;
+ done_cv_.wait_for(lk, std::chrono::microseconds(us), [&] { return done_ != seen; });
+ }
+ double busy_ms() const { return busy_us_.load(std::memory_order_relaxed) / 1000.0; }
+ uint64_t direct_bytes() const { return direct_.load(std::memory_order_relaxed); }
+ uint64_t buffered_bytes() const { return buffered_.load(std::memory_order_relaxed); }
+
+private:
+ void loop() {
+ for (;;) {
+ Job j;
+ {
+ std::unique_lock<std::mutex> lk(mu_);
+ cv_.wait(lk, [&] { return quit_ || !q_.empty(); });
+ if (quit_ && q_.empty()) return;
+ j = q_.front();
+ q_.pop_front();
+ }
+ const auto t0 = std::chrono::steady_clock::now();
+ const uint64_t lead = j.off % os::kDirectAlign;
+ bool ok = false;
+ if (j.mem != nullptr) {
+ std::memcpy(j.dst, j.mem, (size_t) j.len);
+ ok = true;
+ } else if (j.dfd != os::kBadFd && !(j.map != nullptr && os::resident(j.map, (size_t) j.len)) &&
+ os::pread_direct(j.dfd, j.dst, j.len, j.off)) { // (the page cache's ones: buffered, below)
+ ok = true;
+ direct_.fetch_add(j.len, std::memory_order_relaxed);
+ } else if (os::pread_par(j.fd, j.dst + lead, j.len, j.off)) {
+ ok = true;
+ buffered_.fetch_add(j.len, std::memory_order_relaxed);
+ }
+ if (ok && j.keep != nullptr && j.mem == nullptr) std::memcpy(j.keep, j.dst + lead, (size_t) j.len);
+ busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::steady_clock::now() - t0).count(),
+ std::memory_order_relaxed);
+ if (!ok && j.bad != nullptr) j.bad->store(true, std::memory_order_release);
+ if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);
+ {
+ std::lock_guard<std::mutex> lk(done_mu_);
+ ++done_;
+ }
+ done_cv_.notify_all();
+ }
+ }
+ std::vector<std::thread> th_;
+ std::mutex mu_, done_mu_;
+ std::condition_variable cv_, done_cv_;
+ std::deque<Job> q_;
+ uint64_t done_ = 0;
+ bool quit_ = false;
+ std::atomic<uint64_t> busy_us_{0}, direct_{0}, buffered_{0};
+};
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_q6k.cuh b/src/dsv41/ds4_q6k.cuh
new file mode 100644
index 000000000..2eb517eb0
--- /dev/null
+++ b/src/dsv41/ds4_q6k.cuh
@@ -0,0 +1,28 @@
+// src/ds4/ds4_q6k.cuh - one value of a ggml Q6_K superblock (256 values, 210 bytes: ql[128] the low 4 bits, qh[64] the
+// high 2, scales[16] int8 a 16 values, d F16), as ggml's dequantize_row_q6_K computes it: d * scale * (q - 32), the two
+// products in that order in F32 - the same bits (the Engram tables a GGUF may hold in Q6_K).
+#pragma once
+
+#include <cuda_fp16.h>
+#include <cstdint>
+
+namespace strata::ds4 {
+
+constexpr int kQ6kBytes = 210;
+
+__device__ __forceinline__ float q6k_value(const uint8_t* __restrict__ b, int i) {
+ const uint8_t* ql = b;
+ const uint8_t* qh = b + 128;
+ const int8_t* sc = (const int8_t*) (b + 192);
+ // (the F16 scale's bits as a half: HIP's headers take no brace-initialized __half_raw from them)
+ const float d = __half2float(__ushort_as_half((unsigned short) (b[208] | (b[209] << 8))));
+ const int h = i >> 7, w = i & 127, quarter = w >> 5, l = w & 31; // (ggml's loop: half h, 32-value quarters)
+ ql += h * 64;
+ qh += h * 32;
+ sc += h * 8;
+ const int lo = quarter & 1 ? ql[l + 32] : ql[l];
+ const int q = ((quarter & 2 ? lo >> 4 : lo & 0xF) | (((qh[l] >> (2 * quarter)) & 3) << 4)) - 32;
+ return __fmul_rn(__fmul_rn(d, (float) sc[(l >> 4) + 2 * quarter]), (float) q);
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_vision.cu b/src/dsv41/ds4_vision.cu
new file mode 100644
index 000000000..b19b307d8
--- /dev/null
+++ b/src/dsv41/ds4_vision.cu
@@ -0,0 +1,517 @@
+// src/ds4/ds4_vision.cu - DeepSeek-V4.1's vision tower (see ds4_vision.cuh).
+//
+// One encode, N patches on a gh x gw grid:
+// patchify (the reference's (x / 255 - 0.5) / 0.5, BF16) -> x = patch_embed(patches) FP32 [N][1024]
+// per block, its weights in slot (b % 2) while block b + 1's copy runs:
+// xn = norm1(x); qkv = rope(wqkv(xn) + bias) (rows in pieces of rb_: the FP32 GEMM output)
+// per query piece of qb_ rows, per head: S = Q K^T / 8 (FP32), P = softmax(S) (BF16), O[:, head] = P V;
+// x[piece] += wo(O) + bias
+// xn = norm2(x); per row piece: x += w2(silu(gate) * up), gate | up = w1(xn)
+// xn = norm(x); the 3 x 3 unfold; GELU(w1 + b1); w2 + b2 -> BF16 rows (the aligner's weights stream in pieces too)
+#include "ds4_vision.cuh"
+
+#include <cublas_v2.h>
+#include "ds4_numerics.cuh"
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cfloat>
+#include <cmath>
+#include <cstring>
+
+namespace strata::ds4 {
+namespace {
+
+#define VK(call) \
+ do { \
+ cudaError_t e_ = (call); \
+ if (e_ != cudaSuccess) { \
+ err = std::string("vision: ") + #call + ": " + cudaGetErrorString(e_); \
+ return false; \
+ } \
+ } while (0)
+#define VB(call) \
+ do { \
+ cublasStatus_t e_ = (call); \
+ if (e_ != CUBLAS_STATUS_SUCCESS) { \
+ err = std::string("vision: ") + #call + ": cuBLAS status " + std::to_string((int) e_); \
+ return false; \
+ } \
+ } while (0)
+
+constexpr int kT = 256; // threads of the row kernels
+constexpr int kSmT = 512, kSmPer = 20; // softmax: a row of up to 10240 keys in registers
+constexpr size_t kWs = 16u << 20; // cuBLAS workspace
+
+__device__ __forceinline__ float bf(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }
+__device__ __forceinline__ uint16_t bfr(float f) { return bf16_rn(f); }
+
+__device__ float block_reduce(float v, bool is_max) {
+ __shared__ float red[32];
+ __shared__ float tot;
+ for (int o = 16; o > 0; o >>= 1) {
+ const float u = __shfl_xor_sync(0xffffffffu, v, o);
+ v = is_max ? fmaxf(v, u) : v + u;
+ }
+ const int lane = threadIdx.x & 31, w = threadIdx.x >> 5, nw = (blockDim.x + 31) >> 5;
+ __syncthreads(); // (an earlier call's readers of red / tot are done)
+ if (lane == 0) red[w] = v;
+ __syncthreads();
+ if (w == 0) {
+ v = lane < nw ? red[lane] : (is_max ? -FLT_MAX : 0.0f);
+ for (int o = 16; o > 0; o >>= 1) {
+ const float u = __shfl_xor_sync(0xffffffffu, v, o);
+ v = is_max ? fmaxf(v, u) : v + u;
+ }
+ if (lane == 0) tot = v;
+ }
+ __syncthreads();
+ return tot;
+}
+
+// patch i of the gh x gw grid: its 3 x p x p values, channel-major (the reference's reshape / permute)
+__global__ void patchify_kernel(const uint8_t* __restrict__ rgb, int width, int gw, int p, uint16_t* __restrict__ out) {
+ const int i = blockIdx.x, e = threadIdx.x, pp = p * p;
+ if (e >= 3 * pp) return;
+ const int c = e / pp, r = e % pp, y = (i / gw) * p + r / p, x = (i % gw) * p + r % p;
+ const float u = (float) rgb[((size_t) y * width + x) * 3 + c];
+ out[(size_t) i * 3 * pp + e] = bfr(__fdiv_rn(__fsub_rn(__fdiv_rn(u, 255.0f), 0.5f), 0.5f));
+}
+
+// x[r] += y[r] (y null: nothing) + bias
+__global__ void add_rows_kernel(float* __restrict__ x, const float* __restrict__ y, int ldy,
+ const uint16_t* __restrict__ bias, int dim) {
+ const int r = blockIdx.x;
+ for (int e = threadIdx.x; e < dim; e += blockDim.x) {
+ float v = x[(size_t) r * dim + e];
+ if (y != nullptr) v += y[(size_t) r * ldy + e];
+ if (bias != nullptr) v += bf(bias[e]);
+ x[(size_t) r * dim + e] = v;
+ }
+}
+
+__global__ void rmsnorm_kernel(const float* __restrict__ x, const uint16_t* __restrict__ w, int dim, float eps,
+ uint16_t* __restrict__ out) {
+ const int r = blockIdx.x;
+ const float* xr = x + (size_t) r * dim;
+ float ss = 0.0f;
+ for (int e = threadIdx.x; e < dim; e += blockDim.x) ss += xr[e] * xr[e];
+ ss = block_reduce(ss, false);
+ const float rs = rsqrtf(ss / (float) dim + eps);
+ for (int e = threadIdx.x; e < dim; e += blockDim.x) out[(size_t) r * dim + e] = bfr(bf(w[e]) * (xr[e] * rs));
+}
+
+// a piece's rows of wqkv's output (+ bias, rounded to BF16 as the reference's linear) into qkv, q and k rotated: the
+// head's first half pairs with its second; frequency j < hd / 4 turns with the patch's row, the rest with its column
+__global__ void qkv_rope_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int r0, int gw,
+ const float* __restrict__ inv_freq, int heads, int hd, uint16_t* __restrict__ qkv) {
+ const int r = blockIdx.x, gr = r0 + r, dim = heads * hd, half = hd / 2, nf = hd / 4;
+ const int row = gr / gw, col = gr % gw;
+ const float* yr = y + (size_t) r * 3 * dim;
+ uint16_t* o = qkv + (size_t) gr * 3 * dim;
+ for (int i = threadIdx.x; i < 2 * heads * half; i += blockDim.x) {
+ const int base = (i / half) * hd, d = i % half;
+ const float x1 = bf(bfr(yr[base + d] + bf(bias[base + d])));
+ const float x2 = bf(bfr(yr[base + d + half] + bf(bias[base + d + half])));
+ float sn, cs;
+ sincosf((float) (d < nf ? row : col) * inv_freq[d % nf], &sn, &cs);
+ o[base + d] = bfr(x1 * cs - x2 * sn);
+ o[base + d + half] = bfr(x2 * cs + x1 * sn);
+ }
+ for (int e = 2 * dim + threadIdx.x; e < 3 * dim; e += blockDim.x) o[e] = bfr(yr[e] + bf(bias[e]));
+}
+
+__global__ void __launch_bounds__(kSmT) softmax_kernel(const float* __restrict__ s, int n, uint16_t* __restrict__ p) {
+ const float* sr = s + (size_t) blockIdx.x * n;
+ uint16_t* pr = p + (size_t) blockIdx.x * n;
+ float v[kSmPer];
+ float m = -FLT_MAX;
+#pragma unroll
+ for (int k = 0; k < kSmPer; ++k) {
+ const int e = threadIdx.x + k * kSmT;
+ v[k] = e < n ? sr[e] : -FLT_MAX;
+ m = fmaxf(m, v[k]);
+ }
+ m = block_reduce(m, true);
+ float sum = 0.0f;
+#pragma unroll
+ for (int k = 0; k < kSmPer; ++k) {
+ const int e = threadIdx.x + k * kSmT;
+ v[k] = e < n ? expf(v[k] - m) : 0.0f;
+ sum += v[k];
+ }
+ const float inv = 1.0f / block_reduce(sum, false);
+#pragma unroll
+ for (int k = 0; k < kSmPer; ++k) {
+ const int e = threadIdx.x + k * kSmT;
+ if (e < n) pr[e] = bfr(v[k] * inv);
+ }
+}
+
+// mid = silu(gate) * up, gate | up the halves of w1's row (BF16 steps, as the reference's)
+__global__ void silu_mul_kernel(const float* __restrict__ y, int inter, uint16_t* __restrict__ mid) {
+ const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;
+ if (e >= inter) return;
+ const float gt = bf(bfr(y[(size_t) r * 2 * inter + e])), up = bf(bfr(y[(size_t) r * 2 * inter + inter + e]));
+ mid[(size_t) r * inter + e] = bfr(bf(bfr(gt / (1.0f + expf(-gt)))) * up);
+}
+
+// the aligner's input: block (bi, bj) of the m x m patches as one row, value c * m * m + kh * m + kw (F.unfold's
+// order), zero past the grid
+__global__ void unfold_kernel(const uint16_t* __restrict__ xn, int gh, int gw, int m, int dim, int lw,
+ uint16_t* __restrict__ un) {
+ const int l = blockIdx.x, bi = l / lw, bj = l % lw, mm = m * m;
+ for (int f = threadIdx.x; f < dim * mm; f += blockDim.x) {
+ const int c = f / mm, k = f % mm, y = bi * m + k / m, x = bj * m + k % m;
+ un[(size_t) l * dim * mm + f] = y < gh && x < gw ? xn[((size_t) y * gw + x) * dim + c] : (uint16_t) 0;
+ }
+}
+
+// out = act(bf16(y + bias)): GELU (erf) or none
+__global__ void bias_act_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int n, int gelu,
+ uint16_t* __restrict__ out) {
+ const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;
+ if (e >= n) return;
+ float v = bf(bfr(y[(size_t) r * n + e] + bf(bias[e])));
+ if (gelu) v = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f));
+ out[(size_t) r * n + e] = bfr(v);
+}
+
+int cdiv(int a, int b) { return (a + b - 1) / b; }
+
+// y[r][o] = alpha * sum_k x[r][k] W[o][k] (row-major W [out][in] with row stride ldw, x [rows][ldx], y [rows][ldy])
+bool gemm(cublasHandle_t h, const uint16_t* W, int ldw, const uint16_t* x, int ldx, void* y, cudaDataType yt, int ldy,
+ int rows, int in, int out, float alpha, std::string& err) {
+ const float beta = 0.0f;
+ VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, out, rows, in, &alpha, W, CUDA_R_16BF, ldw, x, CUDA_R_16BF, ldx, &beta,
+ y, yt, ldy, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ return true;
+}
+
+} // namespace
+
+template <class T>
+bool VisionTower::dalloc(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ const size_t b = (n * sizeof(T) + 255) / 256 * 256;
+ if (cudaMalloc(&v, b) != cudaSuccess) {
+ err = "vision: cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(at_) + " failed";
+ return false;
+ }
+ dallocs_.push_back(v);
+ dev_bytes_ += b;
+ p = (T*) v;
+ return true;
+}
+
+VisionTower::~VisionTower() { release(); }
+
+void VisionTower::release() {
+ if (at_ < 0) return;
+ int cur = 0;
+ cudaGetDevice(&cur);
+ cudaSetDevice(at_);
+ for (int i = 0; i < 2; ++i) {
+ if (copied_[i]) cudaEventDestroy((cudaEvent_t) copied_[i]);
+ if (used_[i]) cudaEventDestroy((cudaEvent_t) used_[i]);
+ copied_[i] = used_[i] = nullptr;
+ }
+ if (blas_) cublasDestroy((cublasHandle_t) blas_);
+ if (st_) cudaStreamDestroy((cudaStream_t) st_);
+ if (cs_) cudaStreamDestroy((cudaStream_t) cs_);
+ for (void* p : dallocs_) cudaFree(p);
+ if (host_) cudaFreeHost(host_);
+ cudaSetDevice(cur);
+ blas_ = st_ = cs_ = nullptr;
+ dallocs_.clear();
+ host_ = nullptr;
+ dev_ = at_ = -1;
+ dev_bytes_ = 0;
+}
+
+bool VisionTower::load(const VisionGeo& g, int device, const Reader& read, std::string& err) {
+ g_ = g;
+ const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = g.heads > 0 ? g.dim / g.heads : 0;
+ if (g.layers <= 0 || hd * g.heads != D || hd % 4 != 0 || P <= 0 || M <= 0 || g.max_tokens < 5) {
+ err = "vision: geometry not covered";
+ return false;
+ }
+ // the largest aligner grid within max_tokens (n_h * (n_w + 1) + 2): the work buffers' size
+ int cells = 0;
+ for (int h = 1; h <= g.max_tokens - 2; ++h) {
+ const int w = (g.max_tokens - 2) / h - 1;
+ if (w >= 1) cells = std::max(cells, h * w);
+ }
+ max_n_ = cells * M * M;
+ if (max_n_ > kSmT * kSmPer) {
+ err = "vision: " + std::to_string(max_n_) + " patches are more than the attention covers";
+ return false;
+ }
+ cudaSetDevice(device);
+ at_ = device;
+ dev_bytes_ = 0;
+ // the checkpoint's tensors, checked before anything is read
+ struct Want {
+ std::string name;
+ std::vector<int64_t> shape;
+ };
+ std::vector<Want> want = {{"vision.patch_embed.proj.weight", {D, 3 * P * P}},
+ {"vision.patch_embed.proj.bias", {D}},
+ {"vision.norm.weight", {D}},
+ {"aligner.w1.weight", {g.out, (int64_t) D * M * M}},
+ {"aligner.w1.bias", {g.out}},
+ {"aligner.w2.weight", {g.out, g.out}},
+ {"aligner.w2.bias", {g.out}}};
+ for (int l = 0; l < g.layers; ++l) {
+ const std::string B = "vision.blocks." + std::to_string(l) + ".";
+ want.push_back({B + "norm1.weight", {D}});
+ want.push_back({B + "norm2.weight", {D}});
+ want.push_back({B + "attn.wqkv.weight", {3 * D, D}});
+ want.push_back({B + "attn.wqkv.bias", {3 * D}});
+ want.push_back({B + "attn.wo.weight", {D, D}});
+ want.push_back({B + "attn.wo.bias", {D}});
+ want.push_back({B + "mlp.w1.weight", {2 * I, D}});
+ want.push_back({B + "mlp.w2.weight", {D, I}});
+ }
+ for (const Want& w : want) {
+ std::vector<int64_t> s;
+ if (!read(w.name, s, nullptr)) {
+ err = "vision: " + w.name + " is not in the checkpoint";
+ return false;
+ }
+ if (s != w.shape) {
+ err = "vision: " + w.name + " has another shape than the config's";
+ return false;
+ }
+ }
+ // the streamed weights, pinned
+ block_n_ = (size_t) 3 * D * D + (size_t) D * D + (size_t) 2 * I * D + (size_t) D * I;
+ al1_off_ = block_n_ * g.layers;
+ al2_off_ = al1_off_ + (size_t) g.out * D * M * M;
+ const size_t host_n = al2_off_ + (size_t) g.out * g.out;
+ if (cudaHostAlloc((void**) &host_, host_n * 2, cudaHostAllocDefault) != cudaSuccess) {
+ host_ = nullptr;
+ err = "vision: pinning " + std::to_string((host_n * 2) >> 20) + " MiB of host memory failed";
+ return false;
+ }
+ std::vector<int64_t> s;
+ for (int l = 0; l < g.layers; ++l) {
+ const std::string B = "vision.blocks." + std::to_string(l) + ".";
+ uint16_t* b = host_ + (size_t) l * block_n_;
+ if (!read(B + "attn.wqkv.weight", s, b) || !read(B + "attn.wo.weight", s, b + (size_t) 3 * D * D) ||
+ !read(B + "mlp.w1.weight", s, b + (size_t) 4 * D * D) ||
+ !read(B + "mlp.w2.weight", s, b + (size_t) 4 * D * D + (size_t) 2 * I * D)) {
+ err = "vision: block " + std::to_string(l) + ": read failed";
+ return false;
+ }
+ }
+ if (!read("aligner.w1.weight", s, host_ + al1_off_) || !read("aligner.w2.weight", s, host_ + al2_off_)) {
+ err = "vision: the aligner: read failed";
+ return false;
+ }
+ // the small ones on the device
+ auto up = [&](const std::string& name, size_t n, uint16_t*& d) -> bool {
+ std::vector<uint16_t> h(n);
+ std::vector<int64_t> sh;
+ if (!read(name, sh, h.data())) {
+ err = "vision: " + name + ": read failed";
+ return false;
+ }
+ if (!dalloc(d, n, err)) return false;
+ VK(cudaMemcpy(d, h.data(), n * 2, cudaMemcpyHostToDevice));
+ return true;
+ };
+ bool ok = up("vision.patch_embed.proj.weight", (size_t) D * 3 * P * P, pe_w_) &&
+ up("vision.patch_embed.proj.bias", D, pe_b_) && up("vision.norm.weight", D, norm_) &&
+ up("aligner.w1.bias", g.out, al1_b_) && up("aligner.w2.bias", g.out, al2_b_);
+ n1_.assign(g.layers, nullptr);
+ n2_.assign(g.layers, nullptr);
+ bqkv_.assign(g.layers, nullptr);
+ bo_.assign(g.layers, nullptr);
+ for (int l = 0; l < g.layers && ok; ++l) {
+ const std::string B = "vision.blocks." + std::to_string(l) + ".";
+ ok = up(B + "norm1.weight", D, n1_[l]) && up(B + "norm2.weight", D, n2_[l]) &&
+ up(B + "attn.wqkv.bias", 3 * D, bqkv_[l]) && up(B + "attn.wo.bias", D, bo_[l]);
+ }
+ if (!ok) return false;
+ { // the rotary's frequencies: theta ^ -(2i / (hd / 2)), hd / 4 of them
+ const int nf = hd / 4;
+ std::vector<float> f(nf);
+ for (int i = 0; i < nf; ++i) f[i] = 1.0f / std::pow(g.theta, (float) (2 * i) / (float) (hd / 2));
+ if (!dalloc(inv_freq_, nf, err)) return false;
+ VK(cudaMemcpy(inv_freq_, f.data(), nf * 4, cudaMemcpyHostToDevice));
+ }
+ // the work buffers: the patch grid's rows whole (the residual, its norm, q / k / v), the GEMM outputs a piece of
+ // rows at a time, the attention's scores a piece of queries at a time (~48 MB)
+ const int L = cells;
+ rb_ = 2048;
+ qb_ = std::max(16, (int) ((48u << 20) / ((size_t) max_n_ * 4)) / 16 * 16);
+ slot_n_ = std::max(block_n_, (size_t) 1 << 22);
+ const size_t y_n = (size_t) rb_ * std::max({3 * D, 2 * I, D});
+ const size_t qkv_n = std::max({(size_t) max_n_ * 3 * D,
+ (size_t) max_n_ * 3 * P * P + ((size_t) max_n_ * 3 * P * P + 1) / 2, // patches, pixels
+ (size_t) L * D * M * M + (size_t) 2 * L * g.out}); // the aligner's
+ if ((size_t) L * g.out > y_n || (size_t) max_n_ * D > y_n) {
+ err = "vision: work buffer sizes";
+ return false;
+ }
+ ok = dalloc(x_, (size_t) max_n_ * D, err) && dalloc(xn_, (size_t) max_n_ * D, err) && dalloc(qkv_, qkv_n, err) &&
+ dalloc(y_, y_n, err) && dalloc(o_, (size_t) max_n_ * D, err) && dalloc(mid_, (size_t) rb_ * I, err) &&
+ dalloc(s_, (size_t) qb_ * max_n_, err) && dalloc(p_, (size_t) qb_ * max_n_, err) &&
+ dalloc(slot_[0], slot_n_, err) && dalloc(slot_[1], slot_n_, err) && dalloc(ws_, kWs, err);
+ if (!ok) return false;
+ patch_ = qkv_; // (before block 0's qkv)
+ rgb_ = (uint8_t*) (qkv_ + (size_t) max_n_ * 3 * P * P);
+ un_ = qkv_; // (after the last block's attention)
+ a16_ = un_ + (size_t) L * D * M * M;
+ outb_ = a16_ + (size_t) L * g.out;
+ cublasHandle_t h = nullptr;
+ VB(cublasCreate(&h));
+ blas_ = h;
+ cudaStream_t st = nullptr, cs = nullptr;
+ VK(cudaStreamCreateWithFlags(&st, cudaStreamNonBlocking));
+ VK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
+ st_ = st;
+ cs_ = cs;
+ VB(cublasSetStream(h, st));
+ VB(cublasSetWorkspace(h, ws_, kWs));
+ for (int i = 0; i < 2; ++i) {
+ cudaEvent_t a = nullptr, b = nullptr;
+ VK(cudaEventCreateWithFlags(&a, cudaEventDisableTiming));
+ VK(cudaEventCreateWithFlags(&b, cudaEventDisableTiming));
+ copied_[i] = a;
+ used_[i] = b;
+ }
+ dev_ = device;
+ return true;
+}
+
+bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,
+ std::string& err) {
+ if (dev_ < 0) {
+ err = "vision: not loaded";
+ return false;
+ }
+ const VisionGeo& g = g_;
+ const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = D / g.heads, H3 = 3 * D;
+ if (width <= 0 || height <= 0 || width % P != 0 || height % P != 0) {
+ err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) +
+ " pixels (both sides must be multiples of " + std::to_string(P) + ")";
+ return false;
+ }
+ const int gh = height / P, gw = width / P, N = gh * gw;
+ n_h = cdiv(gh, M);
+ n_w = cdiv(gw, M);
+ const int L = n_h * n_w;
+ if (N > max_n_ || n_h * (n_w + 1) + 2 > g.max_tokens) {
+ err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) + " pixels is more than " +
+ std::to_string(g.max_tokens) + " image tokens";
+ return false;
+ }
+ cudaSetDevice(dev_);
+ cublasHandle_t h = (cublasHandle_t) blas_;
+ cudaStream_t s = (cudaStream_t) st_, cs = (cudaStream_t) cs_;
+ // the weights in the order the encode takes them: the blocks, then the aligner's in pieces of a slot
+ std::vector<Piece> pcs;
+ for (int l = 0; l < g.layers; ++l) pcs.push_back({host_ + (size_t) l * block_n_, block_n_});
+ const int in1 = D * M * M, r1 = (int) std::min<size_t>(g.out, slot_n_ / in1), r2 = (int) std::min<size_t>(g.out, slot_n_ / g.out);
+ for (int o0 = 0; o0 < g.out; o0 += r1)
+ pcs.push_back({host_ + al1_off_ + (size_t) o0 * in1, (size_t) std::min(r1, g.out - o0) * in1});
+ for (int o0 = 0; o0 < g.out; o0 += r2)
+ pcs.push_back({host_ + al2_off_ + (size_t) o0 * g.out, (size_t) std::min(r2, g.out - o0) * g.out});
+ auto fetch = [&](size_t i) -> bool {
+ const int k = (int) (i % 2);
+ VK(cudaStreamWaitEvent(cs, (cudaEvent_t) used_[k], 0));
+ VK(cudaMemcpyAsync(slot_[k], pcs[i].host, pcs[i].n * 2, cudaMemcpyHostToDevice, cs));
+ VK(cudaEventRecord((cudaEvent_t) copied_[k], cs));
+ return true;
+ };
+ size_t pi = 0; // the piece being used
+ auto take = [&]() -> const uint16_t* { // the next piece's slot, once its copy is in (the one after it started)
+ const int k = (int) (pi % 2);
+ if (pi + 1 < pcs.size() && !fetch(pi + 1)) return nullptr;
+ if (cudaStreamWaitEvent(s, (cudaEvent_t) copied_[k], 0) != cudaSuccess) {
+ err = "vision: waiting for a weight copy failed";
+ return nullptr;
+ }
+ return slot_[k];
+ };
+ auto done = [&]() { // the piece's work is queued: its slot is free once that ran
+ cudaEventRecord((cudaEvent_t) used_[pi % 2], s);
+ ++pi;
+ };
+ if (!fetch(0)) return false;
+
+ // the patches
+ VK(cudaMemcpyAsync(rgb_, rgb, (size_t) N * 3 * P * P, cudaMemcpyHostToDevice, s));
+ patchify_kernel<<<N, 3 * P * P, 0, s>>>(rgb_, width, gw, P, patch_);
+ if (!gemm(h, pe_w_, 3 * P * P, patch_, 3 * P * P, x_, CUDA_R_32F, D, N, 3 * P * P, D, 1.0f, err)) return false;
+ add_rows_kernel<<<N, kT, 0, s>>>(x_, nullptr, 0, pe_b_, D);
+
+ const float scale = 1.0f / std::sqrt((float) hd);
+ const int qb = (int) std::max<size_t>(1, std::min<size_t>((size_t) N, (size_t) qb_ * max_n_ / (size_t) N));
+ for (int l = 0; l < g.layers; ++l) {
+ const uint16_t* wqkv = take();
+ if (wqkv == nullptr) return false;
+ const uint16_t *wo = wqkv + (size_t) H3 * D, *w1 = wo + (size_t) D * D, *w2 = w1 + (size_t) 2 * I * D;
+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n1_[l], D, g.eps, xn_);
+ for (int r0 = 0; r0 < N; r0 += rb_) {
+ const int nr = std::min(rb_, N - r0);
+ if (!gemm(h, wqkv, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, H3, nr, D, H3, 1.0f, err)) return false;
+ qkv_rope_kernel<<<nr, kT, 0, s>>>(y_, bqkv_[l], r0, gw, inv_freq_, g.heads, hd, qkv_);
+ }
+ for (int q0 = 0; q0 < N; q0 += qb) {
+ const int nq = std::min(qb, N - q0);
+ const float one = 1.0f, zero = 0.0f;
+ for (int hh = 0; hh < g.heads; ++hh) {
+ // S^T (keys x queries, column-major) = K Q^T: row-major S [query][key]
+ VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, N, nq, hd, &scale, qkv_ + D + hh * hd, CUDA_R_16BF, H3,
+ qkv_ + (size_t) q0 * H3 + hh * hd, CUDA_R_16BF, H3, &zero, s_, CUDA_R_32F, N,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ softmax_kernel<<<nq, kSmT, 0, s>>>(s_, N, p_);
+ // O^T (hd x queries) = V^T P^T: row-major O [query][head's hd] at column hh * hd
+ VB(cublasGemmEx(h, CUBLAS_OP_N, CUBLAS_OP_N, hd, nq, N, &one, qkv_ + 2 * D + hh * hd, CUDA_R_16BF, H3,
+ p_, CUDA_R_16BF, N, &zero, o_ + hh * hd, CUDA_R_16BF, D, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT));
+ }
+ if (!gemm(h, wo, D, o_, D, y_, CUDA_R_32F, D, nq, D, D, 1.0f, err)) return false;
+ add_rows_kernel<<<nq, kT, 0, s>>>(x_ + (size_t) q0 * D, y_, D, bo_[l], D);
+ }
+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n2_[l], D, g.eps, xn_);
+ for (int r0 = 0; r0 < N; r0 += rb_) {
+ const int nr = std::min(rb_, N - r0);
+ if (!gemm(h, w1, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, 2 * I, nr, D, 2 * I, 1.0f, err)) return false;
+ silu_mul_kernel<<<dim3(cdiv(I, kT), nr), kT, 0, s>>>(y_, I, mid_);
+ if (!gemm(h, w2, I, mid_, I, y_, CUDA_R_32F, D, nr, I, D, 1.0f, err)) return false;
+ add_rows_kernel<<<nr, kT, 0, s>>>(x_ + (size_t) r0 * D, y_, D, nullptr, D);
+ }
+ done();
+ }
+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, norm_, D, g.eps, xn_);
+
+ // the aligner
+ unfold_kernel<<<L, kT, 0, s>>>(xn_, gh, gw, M, D, n_w, un_);
+ for (int o0 = 0; o0 < g.out; o0 += r1) {
+ const uint16_t* w = take();
+ if (w == nullptr) return false;
+ if (!gemm(h, w, in1, un_, in1, y_ + o0, CUDA_R_32F, g.out, L, in1, std::min(r1, g.out - o0), 1.0f, err))
+ return false;
+ done();
+ }
+ bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al1_b_, g.out, 1, a16_);
+ for (int o0 = 0; o0 < g.out; o0 += r2) {
+ const uint16_t* w = take();
+ if (w == nullptr) return false;
+ if (!gemm(h, w, g.out, a16_, g.out, y_ + o0, CUDA_R_32F, g.out, L, g.out, std::min(r2, g.out - o0), 1.0f, err))
+ return false;
+ done();
+ }
+ bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al2_b_, g.out, 0, outb_);
+ rows.resize((size_t) L * g.out);
+ VK(cudaMemcpyAsync(rows.data(), outb_, rows.size() * 2, cudaMemcpyDeviceToHost, s));
+ VK(cudaStreamSynchronize(s));
+ VK(cudaGetLastError());
+ return true;
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_vision.cuh b/src/dsv41/ds4_vision.cuh
new file mode 100644
index 000000000..e3bcb45e2
--- /dev/null
+++ b/src/dsv41/ds4_vision.cuh
@@ -0,0 +1,77 @@
+// src/ds4/ds4_vision.cuh - DeepSeek-V4.1's vision tower on one GPU (DeepSeek's inference/vision.py):
+// patches (14 x 14 RGB, scaled to [-1, 1]) -> Linear(588 -> 1024)
+// 32 blocks: x += attn(rmsnorm(x)) (16 heads of 64, full bidirectional attention over the image, a 2D rotary on q
+// and k: the first half of each head's frequencies the patch row, the second its column);
+// x += w2(silu(gate) * up) with gate, up = w1(rmsnorm(x))
+// rmsnorm, then the aligner: each 3 x 3 block of patches (zero-padded at the right and bottom) as one 9216 row,
+// Linear(9216 -> 5120), GELU, Linear(5120 -> 5120): one LLM embedding row per block, in reading order.
+// The BF16 weights live in pinned host memory and stream to the GPU a block at a time (two slots, the next block's
+// copy under this one's work), so the tower costs the model ~350 MB of the GPU's memory, not ~1 GB; the GEMMs are
+// cuBLAS (BF16 in, FP32 accumulation), the residual stream FP32.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace strata::ds4 {
+
+struct VisionGeo {
+ int layers = 0, dim = 1024, heads = 16, inter = 2816, patch = 14, merge = 3, out = 5120;
+ float theta = 10000.0f, eps = 1e-6f;
+ int max_tokens = 1024; // an image's span in the prompt at most: n_h * (n_w + 1) + 2 tokens
+};
+
+class VisionTower {
+public:
+ /// read(name, shape, dst): the tensor's shape, and (dst not null) its BF16 data into dst; false: not there
+ using Reader = std::function<bool(const std::string& name, std::vector<int64_t>& shape, void* dst)>;
+ VisionTower() = default;
+ VisionTower(const VisionTower&) = delete;
+ VisionTower& operator=(const VisionTower&) = delete;
+ ~VisionTower();
+ bool load(const VisionGeo& g, int device, const Reader& read, std::string& err);
+ /// Frees everything load took (a load that failed half way too); loaded() is false after it.
+ void release();
+ bool loaded() const { return dev_ >= 0; }
+ int device() const { return dev_; }
+ size_t device_bytes() const { return dev_bytes_; }
+ /// The patches an image may have (its aligner grid within max_tokens).
+ int max_patches() const { return max_n_; }
+ /// rgb: height x width x 3 bytes, both sides multiples of the patch size -> the aligner's rows, BF16
+ /// [n_h * n_w][out] in reading order (n_h x n_w: the patch grid over 3, rounded up).
+ bool encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,
+ std::string& err);
+
+private:
+ struct Piece { // a weight the encode streams in: host bits, its size
+ const uint16_t* host = nullptr;
+ size_t n = 0;
+ };
+ VisionGeo g_;
+ int dev_ = -1, at_ = -1, max_n_ = 0, qb_ = 0, rb_ = 0; // (at_: the device its memory is on)
+ size_t dev_bytes_ = 0, slot_n_ = 0;
+ std::vector<void*> dallocs_;
+ uint16_t* host_ = nullptr; // pinned: every block's [wqkv, wo, w1, w2], then aligner w1, w2
+ size_t block_n_ = 0; // a block's weights (BF16 values)
+ size_t al1_off_ = 0, al2_off_ = 0; // the aligner's weights in host_
+ // resident (small): the patch embedding, the norms and biases
+ uint16_t *pe_w_ = nullptr, *pe_b_ = nullptr, *norm_ = nullptr, *al1_b_ = nullptr, *al2_b_ = nullptr;
+ std::vector<uint16_t*> n1_, n2_, bqkv_, bo_;
+ // work buffers (max_n_ patches)
+ uint8_t* rgb_ = nullptr;
+ uint16_t *patch_ = nullptr, *xn_ = nullptr, *qkv_ = nullptr, *o_ = nullptr, *mid_ = nullptr, *p_ = nullptr,
+ *un_ = nullptr, *a16_ = nullptr, *outb_ = nullptr;
+ float *x_ = nullptr, *y_ = nullptr, *s_ = nullptr, *inv_freq_ = nullptr;
+ uint16_t* slot_[2] = {nullptr, nullptr};
+ void* blas_ = nullptr; // cublasHandle_t
+ uint8_t* ws_ = nullptr;
+ void *st_ = nullptr, *cs_ = nullptr; // cudaStream_t: the work, the weight copies
+ void *copied_[2] = {nullptr, nullptr}, *used_[2] = {nullptr, nullptr}; // cudaEvent_t per slot
+ template <class T>
+ bool dalloc(T*& p, size_t n, std::string& err);
+};
+
+} // namespace strata::ds4
diff --git a/src/dsv41/vendor/exl3.cu b/src/dsv41/vendor/exl3.cu
new file mode 100644
index 000000000..9307e4185
--- /dev/null
+++ b/src/dsv41/vendor/exl3.cu
@@ -0,0 +1,1743 @@
+// src/kernels/cuda/exl3.cu - the EXL3 device kernels (the format: include/strata/kernels/exl3.hpp).
+//
+// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/
+// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh), and so does decoding a tile straight into tensor-core fragments
+// (its small-m GEMV and GEMM); the kernels around them are shaped for this engine's decode:
+//
+// * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;
+// warp w takes tile column w through every 16-row slice, decoding 8 weights per lane (one fragment: 2 columns x
+// 4 rows) - on sm_80 and later straight into one mma.m16n8k16 a tile with up to 8 activations as its columns,
+// F32 accumulation; before sm_80 (and on HIP) F32 FMAs on the CUDA cores;
+// * the input rotation runs per block, 128 inputs at a time (F32 or q8_1 in, FP16 in shared memory), so the
+// engine's activation buffers are used as they are; the next 128 inputs' tile words load while these decode;
+// * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over
+// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in
+// split order (deterministic) and runs the output rotation.
+//
+// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy
+// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The
+// prompt path below is this file's on every GPU.
+#include "strata/kernels/exl3.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <mutex>
+#include <utility>
+
+#if !defined(STRATA_USE_HIP)
+namespace strata::kernels::exl3::legacy { // exl3_legacy.cu: v1.1.8's decode kernels
+int launch_errors();
+bool prepare(cudaStream_t s);
+bool several_ok();
+bool mv(const MvJob* jobs, int n, cudaStream_t s);
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);
+bool win_experts(const WinArgs& a, cudaStream_t s);
+} // namespace strata::kernels::exl3::legacy
+#define EXL3_LEGACY(call) \
+ if (use_legacy()) return legacy::call
+#else
+#define EXL3_LEGACY(call) (void) 0
+#endif
+
+namespace strata::kernels::exl3 {
+namespace {
+
+constexpr int kThreads = 256, kWarps = 8;
+constexpr int kCols = 128; // outputs per block: 8 tiles, one rotation block
+constexpr int kMaxKR = 4096; // inputs per block at most (mv's split)
+// a routed expert's widths at most (moe_* / win_*: a block streams its inputs through block_gemv's two 128-value chunk
+// buffers, so the shared memory does not grow with them; DeepSeek-V4.1's n_embd is 5120)
+constexpr int kMaxMoeK = 8192;
+constexpr float kHad = 0.088388347648f; // 1 / sqrt(128)
+
+struct Q81 { // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8
+ __half2 ds;
+ int8_t qs[32];
+};
+
+std::atomic<int> g_launch_errors{0};
+thread_local const int* g_skip = nullptr; // set_skip: the calling thread's flag (nullptr: none)
+void launch_check(const char* what) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+// ---------------------------------------------------------------- the codebooks
+__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {
+#if defined(STRATA_USE_HIP)
+ return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);
+#else
+ return __dp4a(x, 0x01010101u, acc);
+#endif
+}
+
+// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)
+template <int cb>
+__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {
+ if constexpr (cb == 2) {
+ const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);
+ const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);
+ const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));
+ return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));
+ } else {
+ uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;
+ uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;
+ x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;
+ x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;
+ const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),
+ __ushort_as_half((unsigned short) (x1 & 0xffffu)));
+ const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),
+ __ushort_as_half((unsigned short) (x1 >> 16)));
+ return __hadd2(lo, hi);
+ }
+}
+template <int cb>
+__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {
+ return __half22float2(decode2h<cb>(s0, s1));
+}
+
+__device__ __forceinline__ uint32_t h2u(__half2 h) { return *reinterpret_cast<uint32_t*>(&h); }
+__device__ __forceinline__ __half2 u2h(uint32_t u) { return *reinterpret_cast<__half2*>(&u); }
+
+// (hi : lo) >> s, the low 32 bits
+__device__ __forceinline__ uint32_t fsh(uint32_t lo, uint32_t hi, int s) {
+ return (uint32_t) ((((uint64_t) hi << 32) | (uint64_t) lo) >> s);
+}
+
+// Step p's state is the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),
+// wrapping around the tile. A lane decodes its 8 consecutive steps lane * 8 .. + 7 together, the windows of a group
+// of steps cut from one or two words with funnel shifts - exllamav3's grouping per K (exl3_dq.cuh: dq8_aligned_*bits,
+// dq8, dq4, dq2x2), which the steps' fixed offsets keep within two words. f[i]: steps 2i, 2i + 1 as FP16 values.
+// Step lane * 8 + j sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column (lane >> 2) + 8 (j >> 2).
+template <int cb>
+__device__ __forceinline__ void decode_states(const uint32_t (&s)[8], uint32_t (&f)[4]) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) f[i] = h2u(decode2h<cb>(s[2 * i] & 0xffffu, s[2 * i + 1] & 0xffffu));
+}
+// K = 4: the lane's steps end in its own word b (word lane), the first three reach into a (word lane - 1)
+__device__ __forceinline__ void states4(uint32_t a, uint32_t b, uint32_t (&s)[8]) {
+ const uint32_t x = fsh(b, a, 20);
+ s[0] = x >> 8;
+ s[1] = x >> 4;
+ s[2] = x;
+ s[3] = b >> 16;
+ s[4] = b >> 12;
+ s[5] = b >> 8;
+ s[6] = b >> 4;
+ s[7] = b;
+}
+// step p .. p + G - 1's windows (G = 4 or 2) from the two words that hold them
+template <int K, int G>
+__device__ __forceinline__ void states_group(const uint32_t* w, int p, uint32_t* s) {
+ constexpr int NW = 8 * K;
+ const int b0 = (p + 257) * K - 16, b2 = b0 + (G - 1) * K + 16;
+ const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;
+ const uint32_t a = w[i0 % NW], b = w[i2 % NW];
+#pragma unroll
+ for (int j = 0; j < G; ++j) s[G - 1 - j] = fsh(b, a, s2 + j * K);
+}
+template <int K, int cb>
+__device__ __forceinline__ void frag8(const uint32_t* w, int lane, uint32_t (&f)[4]) {
+ const int p = lane * 8;
+ uint32_t s[8];
+ if constexpr (K == 4) {
+ states4(w[(lane + 31) & 31], w[lane], s);
+ } else if constexpr (K == 1 || K == 2) {
+ // the 8 windows overlap in one 32-bit window: steps p + j end 8 - 1 - j K bits apart
+ constexpr int NW = 8 * K, per = 32 / (8 * K); // lanes per word
+ const int i1 = lane / per, i0 = (i1 + NW - 1) % NW;
+ const uint32_t b = fsh(w[i1], w[i0], (per - 1 - lane % per) * 8 * K);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) s[j] = b >> ((7 - j) * K);
+ } else if constexpr (K == 3) {
+ // all 8 windows within two words: two funnel shifts, the rest by shifting
+ constexpr int NW = 24;
+ const int b1 = (p + 257) * 3, b0 = b1 - 16, b2 = b1 + 21;
+ const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;
+ const uint32_t a = w[i0 % NW], b = w[i2 % NW];
+ const uint32_t hi = fsh(b, a, s2), lo = fsh(b, a, s2 + 12);
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ s[7 - j] = hi >> (3 * j);
+ s[3 - j] = lo >> (3 * j);
+ }
+ } else if constexpr (K == 7) {
+#pragma unroll
+ for (int g = 0; g < 4; ++g) states_group<K, 2>(w, p + 2 * g, s + 2 * g);
+ } else { // 5, 6, 8
+ states_group<K, 4>(w, p, s);
+ states_group<K, 4>(w, p + 4, s + 4);
+ }
+ decode_states<cb>(s, f);
+}
+
+// the lane's 8 steps as F32: v[j] = step lane * 8 + j
+template <int K, int cb>
+__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {
+ uint32_t f[4];
+ frag8<K, cb>(w, lane, f);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 d = __half22float2(u2h(f[i]));
+ v[2 * i] = d.x;
+ v[2 * i + 1] = d.y;
+ }
+}
+
+// ---------------------------------------------------------------- the rotation
+// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)
+__device__ __forceinline__ void had128(float* v, int lane) {
+ const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];
+ v[0] = s0 + s1;
+ v[1] = d0 + d1;
+ v[2] = s0 - s1;
+ v[3] = d0 - d1;
+#pragma unroll
+ for (int m = 1; m < 32; m <<= 1) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float o = __shfl_xor_sync(0xffffffffu, v[i], m);
+ v[i] = (lane & m) ? o - v[i] : v[i] + o;
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] *= kHad;
+}
+
+__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }
+
+// the activation a warp rotates into its block's FP16 chunks: F32 (f) or the engine's q8_1 blocks (q)
+struct Xin {
+ const float* f = nullptr;
+ const Q81* q = nullptr;
+};
+
+// the 128-input chunk of x from k0 on: the lane's 4 values and their suh ...
+__device__ __forceinline__ void chunk_load(const Xin& x, const uint16_t* __restrict__ suh, int k0, int lane,
+ float (&v)[4], float (&s)[4]) {
+ const int base = k0 + lane * 4;
+ if (x.f != nullptr) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = x.f[base + i];
+ } else {
+ const Q81* b = x.q + base / 32;
+ const float d = __low2float(b->ds);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) s[i] = h2f(suh[base + i]);
+}
+// ... and H(x * suh) of the chunk, FP16, into dst[0..127]
+__device__ __forceinline__ void chunk_store(float (&v)[4], const float (&s)[4], int lane, __half* dst) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] *= s[i];
+ had128(v, lane);
+ __half2* d = reinterpret_cast<__half2*>(dst + lane * 4);
+ d[0] = __floats2half2_rn(v[0], v[1]);
+ d[1] = __floats2half2_rn(v[2], v[3]);
+}
+
+// ---------------------------------------------------------------- the block GEMV
+// A block owns 128 outputs - 8 tiles, warp w the tile column tc0 + w through all of the block's inputs [k0, k1) - for
+// nc <= kTok activations at once. Per 128-input chunk, warp c (< nc) scales activation c's chunk by suh and rotates
+// it into FP16 (xb, two buffers), then each warp decodes its tile of each of the chunk's 8 slices and multiplies it
+// with every activation: on sm_80 and later as one mma.m16n8k16 a tile (the decoded tile is the A operand - a lane's 8
+// steps are its A fragment - and the activations are the B operand's columns, as exllamav3's small-m GEMV and its
+// tensor-core GEMM decode tiles straight into fragments), F32 accumulation; elsewhere on the CUDA cores, in F32. An
+// activation's sums do not depend on the other columns, so a window's token gets the token path's bits. The next
+// chunk's tile words are loaded while this one is decoded. out[c * 128 + i]: activation c's sum for output i. Ends
+// with a __syncthreads (out complete).
+constexpr int kTok = 8; // activations a block takes at once (the MMA's n)
+
+#if !defined(STRATA_USE_HIP) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+#define EXL3_TC 1
+#endif
+
+#if !defined(EXL3_TC)
+// a lane's 4 weights of one column times the activation's 4 inputs, added to a (the CUDA-core product)
+__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {
+ return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));
+}
+#else
+__device__ __forceinline__ void mma_f32(float (&c)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
+ uint32_t b0, uint32_t b1) {
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+ "{%0,%1,%2,%3};\n"
+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+ : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));
+}
+#endif
+
+// a tile's words, and a lane's share of them (word lane + 32 r)
+template <int K>
+struct Words {
+ static constexpr int W = 8 * K, R = (W + 31) / 32;
+};
+
+// the warp's tile of each of a chunk's 8 slices (slice s at tile0 + s sstride)
+template <int K>
+__device__ __forceinline__ void load_tiles(const uint32_t* __restrict__ tile0, size_t sstride, int lane,
+ uint32_t (&w)[8][Words<K>::R]) {
+ constexpr int W = Words<K>::W, R = Words<K>::R;
+#pragma unroll
+ for (int s = 0; s < 8; ++s)
+#pragma unroll
+ for (int r = 0; r < R; ++r) {
+ const int i = lane + 32 * r;
+ w[s][r] = (W % 32 == 0 || i < W) ? __ldg(tile0 + (size_t) s * sstride + i) : 0u;
+ }
+}
+
+// one slice's fragment from the lane's words: K = 4 straight from the registers (the lane's steps end in its own word,
+// the first three reach into the previous lane's), other K through the warp's stage (8 K words)
+template <int K, int cb>
+__device__ __forceinline__ void slice_frags(const uint32_t (&w)[Words<K>::R], uint32_t* st, int lane, uint32_t (&f)[4]) {
+ if constexpr (K == 4) {
+ uint32_t s[8];
+ states4(__shfl_sync(0xffffffffu, w[0], (lane + 31) & 31), w[0], s);
+ decode_states<cb>(s, f);
+ } else {
+ constexpr int W = Words<K>::W, R = Words<K>::R;
+ __syncwarp();
+#pragma unroll
+ for (int r = 0; r < R; ++r)
+ if (W % 32 == 0 || lane + 32 * r < W) st[lane + 32 * r] = w[r];
+ __syncwarp();
+ frag8<K, cb>(st, lane, f);
+ }
+}
+
+// the blocks an SM keeps of the block_gemv kernels (their registers), as measured on an RTX 3090 and an RTX 5060 Ti:
+// the routed experts four at K <= 4 (a token's 8 experts, 256 blocks, are one wave on 82 SMs), two at wider codes;
+// mv two, three at K > 4 on sm_80 .. sm_90 (sm_120 spills at the 80 registers that takes). Before sm_80 (the CUDA
+// cores' product, more registers) two; HIP reads the second bound differently and keeps its default.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+template <int K>
+constexpr int kMinBlocksX = K <= 4 ? 4 : 2;
+#else
+template <int K>
+constexpr int kMinBlocksX = 2;
+#endif
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 1000
+template <int K>
+constexpr int kMinBlocksMv = K <= 4 ? 2 : 3;
+#else
+template <int K>
+constexpr int kMinBlocksMv = 2;
+#endif
+#if defined(STRATA_USE_HIP)
+#define EXL3_BOUNDS(n) __launch_bounds__(kThreads)
+#else
+#define EXL3_BOUNDS(n) __launch_bounds__(kThreads, n)
+#endif
+
+// the shared memory of a block_gemv block
+template <int K>
+struct BlockSmem {
+ __half xb[2 * kTok * kCols]; // two chunk buffers, kTok activations each
+ uint32_t stage[kWarps * 8 * K]; // a tile per warp (K != 4)
+ float out[kTok * kCols];
+};
+
+template <int K, int cb>
+__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int k0, int k1, const Xin& x,
+ const uint16_t* __restrict__ suh, int nc, BlockSmem<K>& sm) {
+ constexpr int W = Words<K>::W, R = Words<K>::R;
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
+ const size_t sstride = (size_t) ld * W;
+ const uint32_t* tile = tr + ((size_t) (k0 / 16) * ld + tc0 + warp) * W;
+ uint32_t* st = sm.stage + warp * W;
+ const bool rot = warp < nc;
+ const int nq = (k1 - k0) / kCols;
+ uint32_t wc[8][R], wn[8][R];
+ float xv[4], xs4[4];
+ load_tiles<K>(tile, sstride, lane, wc);
+ if (rot) chunk_load(x, suh, k0, lane, xv, xs4);
+#if defined(EXL3_TC)
+ float c[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+#else
+ float acc[kTok][2];
+#pragma unroll
+ for (int cc = 0; cc < kTok; ++cc) acc[cc][0] = acc[cc][1] = 0.0f;
+#endif
+ for (int q = 0; q < nq; ++q) {
+ __half* xq = sm.xb + (q & 1) * (kTok * kCols);
+ if (rot) chunk_store(xv, xs4, lane, xq + warp * kCols);
+ __syncthreads();
+ if (q + 1 < nq) {
+ load_tiles<K>(tile + (size_t) (q + 1) * 8 * sstride, sstride, lane, wn);
+ if (rot) chunk_load(x, suh, k0 + (q + 1) * kCols, lane, xv, xs4);
+ }
+#pragma unroll
+ for (int s = 0; s < 8; ++s) {
+ uint32_t f[4];
+ slice_frags<K, cb>(wc[s], st, lane, f);
+#if defined(EXL3_TC)
+ uint32_t b0 = 0u, b1 = 0u;
+ if (g < nc) {
+ const __half* xr = xq + g * kCols + s * 16 + 2 * t;
+ b0 = *reinterpret_cast<const uint32_t*>(xr);
+ b1 = *reinterpret_cast<const uint32_t*>(xr + 8);
+ }
+ // A: rows (outputs) g / g + 8, columns (inputs) 2t.. / 2t + 8..: steps 0,1 / 4,5 / 2,3 / 6,7
+ mma_f32(c, f[0], f[2], f[1], f[3], b0, b1);
+#else
+ float v[8];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float2 d = __half22float2(u2h(f[i]));
+ v[2 * i] = d.x;
+ v[2 * i + 1] = d.y;
+ }
+#pragma unroll
+ for (int cc = 0; cc < kTok; ++cc)
+ if (cc < nc) {
+ const __half* xr = xq + cc * kCols + s * 16 + 2 * t;
+ const float x0 = __half2float(xr[0]), x1 = __half2float(xr[1]);
+ const float x2 = __half2float(xr[8]), x3 = __half2float(xr[9]);
+ acc[cc][0] = dot4_acc(acc[cc][0], v, x0, x1, x2, x3);
+ acc[cc][1] = dot4_acc(acc[cc][1], v + 4, x0, x1, x2, x3);
+ }
+#endif
+ }
+#pragma unroll
+ for (int s = 0; s < 8; ++s)
+#pragma unroll
+ for (int r = 0; r < R; ++r) wc[s][r] = wn[s][r];
+ }
+ const int o = warp * 16 + g;
+#if defined(EXL3_TC)
+ // c: the tile's outputs g (c[0], c[1]) and g + 8 (c[2], c[3]) for activations 2t (c[0], c[2]) and 2t + 1
+ if (2 * t < nc) {
+ sm.out[(2 * t) * kCols + o] = c[0];
+ sm.out[(2 * t) * kCols + o + 8] = c[2];
+ }
+ if (2 * t + 1 < nc) {
+ sm.out[(2 * t + 1) * kCols + o] = c[1];
+ sm.out[(2 * t + 1) * kCols + o + 8] = c[3];
+ }
+#else
+ // the 4 lanes of one g hold parts of the same two outputs
+#pragma unroll
+ for (int cc = 0; cc < kTok; ++cc)
+ if (cc < nc) {
+#pragma unroll
+ for (int h = 0; h < 2; ++h) {
+ acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 1);
+ acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 2);
+ }
+ if (t == 0) {
+ sm.out[cc * kCols + o] = acc[cc][0];
+ sm.out[cc * kCols + o + 8] = acc[cc][1];
+ }
+ }
+#endif
+ __syncthreads();
+}
+
+// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs
+__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,
+ float* y) {
+ const int lane = threadIdx.x & 31;
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int c = lane * 4 + i;
+ y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- mv
+struct KJob {
+ const uint32_t* tr;
+ const uint16_t* suh;
+ const uint16_t* svh;
+ const float* xf;
+ const Q81* xq;
+ float* y;
+ const float* bias;
+ float* part; // nc x S x n partial sums (S > 1)
+ unsigned* cnt; // n / 128 counters, zero between launches
+ float alpha;
+ int k, n, ld, S, kr;
+ int nc; // the activations (1..kTok), xf / xq / y advanced by these per activation
+ size_t xf_cs, xq_cs, y_cs; // (floats, bytes, floats)
+};
+struct KBatch {
+ KJob j[kMaxMvJobs];
+ int blk_end[kMaxMvJobs];
+ int n;
+ const int* skip; // set_skip's flag: non-zero = a pipelined window's token past a wrong draft (nullptr: none)
+};
+
+// a launch's blocks: its jobs' (output block, input split) pairs, each for all of the job's nc activations at once
+// (block_gemv). S > 1: the partial sums go to the scratch and the last block of an output block adds them up in split
+// order (deterministic) and runs the epilogue
+template <int K, int cb>
+__global__ void EXL3_BOUNDS(kMinBlocksMv<K>) mv_kernel(const KBatch b) {
+ __shared__ __align__(16) BlockSmem<K> sm;
+ __shared__ int last;
+ int ji = 0;
+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
+ const KJob& J = b.j[ji];
+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
+ const int g = local / J.S, sp = local % J.S;
+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
+ const int warp = threadIdx.x >> 5;
+ // a skipped token (set_skip): no work. The flag changes only between launches (skip_gate, on the same stream), so
+ // every block of the launch returns and no split counter moves
+ if (b.skip != nullptr && *b.skip) return;
+ Xin x;
+ if (warp < J.nc) {
+ if (J.xf != nullptr) x.f = J.xf + warp * J.xf_cs;
+ else x.q = (const Q81*) ((const uint8_t*) J.xq + warp * J.xq_cs);
+ }
+ block_gemv<K, cb>(J.tr, J.ld, g * 8, k0, k1, x, J.suh, J.nc, sm);
+ const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;
+ if (J.S == 1) {
+ if (warp < J.nc)
+ epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+ return;
+ }
+ if (threadIdx.x < kCols)
+ for (int c = 0; c < J.nc; ++c)
+ J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.out[c * kCols + threadIdx.x];
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < kCols)
+ for (int c = 0; c < J.nc; ++c) {
+ float s = 0.0f;
+ for (int i = 0; i < J.S; ++i)
+ s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];
+ sm.out[c * kCols + threadIdx.x] = s;
+ }
+ __syncthreads();
+ if (warp < J.nc)
+ epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+ if (threadIdx.x == 0) J.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- the routed experts
+struct KMoe {
+ const unsigned long long* plan_ptr;
+ const float* plan_w;
+ const float* cpu_part;
+ const int* cpu_flag;
+ const float* x;
+ float* h; // [k][n_ff]: the gate's values, then the swiglu
+ float* u; // [k][n_ff]: the up's values
+ const float* sh_out;
+ float* out;
+ float* part; // [k][n_embd]
+ unsigned* cnt; // n_embd / 128
+ unsigned* cnt_gu; // k x n_ff / 128: the gate / up pairs
+ size_t suh[3], svh[3], tr[3];
+ int k, n_embd, n_ff;
+ float limit;
+ const int* skip; // set_skip's flag (nullptr: none)
+};
+
+// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window
+// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...
+__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {
+ const int lane = threadIdx.x & 31;
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));
+}
+// ... the clamped swiglu ...
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
+ if (limit > 0.0f) {
+ gv = fminf(gv, limit);
+ uv = fminf(fmaxf(uv, -limit), limit);
+ }
+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+// ... the second block of a (gate, up) pair to finish (its counter back at zero for the next launch) ...
+__device__ __forceinline__ bool pair_last(unsigned* cnt, int& last) {
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(cnt, 1u) == 1u;
+ __syncthreads();
+ if (!last) return false;
+ __threadfence();
+ if (threadIdx.x == 0) *cnt = 0u;
+ return true;
+}
+// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU
+// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)
+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
+ float m = 0.0f;
+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
+ return __fadd_rn(m, sh);
+}
+
+// grid (n_ff / 128, k, 2): expert blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values; the pair's
+// second block to finish turns them into the clamped swiglu, in h
+template <int K, int cb>
+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_gate_up_kernel(const KMoe a) {
+ __shared__ __align__(16) BlockSmem<K> sm;
+ __shared__ int last;
+ const int ei = blockIdx.y, g = blockIdx.x, r = blockIdx.z;
+ const unsigned long long ptr = a.plan_ptr[ei];
+ if (ptr == 0ull) return;
+ if (a.skip != nullptr && *a.skip) return; // a skipped token (set_skip, as mv_kernel)
+ const uint8_t* blob = (const uint8_t*) ptr;
+ const int warp = threadIdx.x >> 5;
+ Xin x;
+ x.f = a.x;
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,
+ (const uint16_t*) (blob + a.suh[r]), 1, sm);
+ const size_t row = (size_t) ei * a.n_ff + g * kCols;
+ if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[r]) + g * kCols, (r == 0 ? a.h : a.u) + row);
+ if (!pair_last(a.cnt_gu + ei * (a.n_ff / kCols) + g, last)) return;
+ if (threadIdx.x < kCols)
+ a.h[row + threadIdx.x] = swiglu(*(volatile const float*) &a.h[row + threadIdx.x],
+ *(volatile const float*) &a.u[row + threadIdx.x], a.limit);
+}
+
+// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds
+// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert
+template <int K, int cb>
+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_down_kernel(const KMoe a) {
+ __shared__ __align__(16) BlockSmem<K> sm;
+ __shared__ int last;
+ const int ei = blockIdx.y, g = blockIdx.x;
+ const unsigned long long ptr = a.plan_ptr[ei];
+ const int warp = threadIdx.x >> 5;
+ float* part = a.part + (size_t) ei * a.n_embd + g * kCols;
+ if (a.skip != nullptr && *a.skip) return; // a skipped token (set_skip, as mv_kernel)
+ if (ptr != 0ull) {
+ const uint8_t* blob = (const uint8_t*) ptr;
+ Xin x;
+ x.f = a.h + (size_t) ei * a.n_ff;
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,
+ (const uint16_t*) (blob + a.suh[2]), 1, sm);
+ if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);
+ } else if (threadIdx.x < kCols) {
+ part[threadIdx.x] = 0.0f;
+ }
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < kCols) {
+ const int r = g * kCols + threadIdx.x;
+ float p[8];
+ for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];
+ a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,
+ a.sh_out != nullptr ? a.sh_out[r] : 0.0f);
+ }
+ if (threadIdx.x == 0) a.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- a verify window's routed experts
+// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at
+// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kTok of its tokens at a time (as
+// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token
+// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit
+struct KWin {
+ const unsigned long long* grp_ptr;
+ const int *grp_start, *grp_ent, *n_groups;
+ const unsigned long long* plan_ptr;
+ const float* plan_w;
+ const float* cpu_part;
+ size_t cpu_stride;
+ const int* cpu_flag;
+ const float* x;
+ size_t x_stride;
+ float* h; // [T * 8][n_ff]: the gate's values, then the swiglu
+ float* u; // [T * 8][n_ff]: the up's values
+ float* part; // [T * 8][n_embd]
+ const float* sh_out; // [T][n_embd]
+ float* out;
+ size_t out_stride;
+ unsigned* cnt_gu; // groups x n_ff / 128: the gate / up pairs
+ size_t suh[3], svh[3], tr[3];
+ int k, n_embd, n_ff;
+ float limit;
+};
+
+// grid (n_ff / 128, groups at most, 2): group blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values of
+// each of its entries, kTok entries a pass; the pair's second block to finish turns them into the clamped swiglu
+template <int K, int cb>
+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_gu_kernel(const KWin a) {
+ __shared__ __align__(16) BlockSmem<K> sm;
+ __shared__ int last;
+ const int grp = blockIdx.y, g = blockIdx.x, r = blockIdx.z;
+ if (grp >= *a.n_groups) return;
+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+ const int warp = threadIdx.x >> 5;
+ const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];
+ float* dst = r == 0 ? a.h : a.u;
+ for (int mm = m0; mm < m1; mm += kTok) {
+ const int nc = min(kTok, m1 - mm);
+ Xin x;
+ if (warp < nc) x.f = a.x + (size_t) (a.grp_ent[mm + warp] >> 3) * a.x_stride;
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,
+ (const uint16_t*) (blob + a.suh[r]), nc, sm);
+ if (warp < nc)
+ block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,
+ dst + (size_t) a.grp_ent[mm + warp] * a.n_ff + g * kCols);
+ __syncthreads();
+ }
+ if (!pair_last(a.cnt_gu + grp * (a.n_ff / kCols) + g, last)) return;
+ if (threadIdx.x < kCols)
+ for (int m = m0; m < m1; ++m) {
+ const size_t row = (size_t) a.grp_ent[m] * a.n_ff + g * kCols + threadIdx.x;
+ a.h[row] = swiglu(*(volatile const float*) &a.h[row], *(volatile const float*) &a.u[row], a.limit);
+ }
+}
+
+// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows
+template <int K, int cb>
+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_down_kernel(const KWin a) {
+ __shared__ __align__(16) BlockSmem<K> sm;
+ const int grp = blockIdx.y, g = blockIdx.x;
+ if (grp >= *a.n_groups) return;
+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+ const int warp = threadIdx.x >> 5;
+ const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];
+ for (int mm = m0; mm < m1; mm += kTok) {
+ const int nc = min(kTok, m1 - mm);
+ Xin x;
+ if (warp < nc) x.f = a.h + (size_t) a.grp_ent[mm + warp] * a.n_ff;
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,
+ (const uint16_t*) (blob + a.suh[2]), nc, sm);
+ if (warp < nc)
+ block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,
+ a.part + (size_t) a.grp_ent[mm + warp] * a.n_embd + g * kCols);
+ __syncthreads();
+ }
+}
+
+// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)
+__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {
+ const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;
+ float p[8];
+ for (int i = 0; i < a.k; ++i) {
+ const int e = t * 8 + i;
+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
+ }
+ a.out[(size_t) t * a.out_stride + r] =
+ combine(a.plan_w + t * 8, p, a.k,
+ a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,
+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
+}
+
+__global__ void win_wait_kernel(const unsigned* p, unsigned v) {
+ if (threadIdx.x != 0) return;
+ while (*(volatile const unsigned*) p != v) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+ __nanosleep(200);
+#endif
+ }
+ __threadfence_system();
+}
+
+// ---------------------------------------------------------------- reconstruction
+// grid (n / 128, k / 128): the 128 x 128 block of W = diag(suh) H W' H diag(svh), written transposed (dst[c][r])
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) recon_rows_kernel(const uint32_t* __restrict__ tr, int ld,
+ const uint16_t* __restrict__ suh,
+ const uint16_t* __restrict__ svh, int k, uint16_t* dst,
+ bool bf16) {
+ constexpr int W = 8 * K;
+ constexpr int P = kCols + 2; // padded row (halves)
+ __shared__ __half tile[kCols * P];
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ const int nb = blockIdx.x, kb = blockIdx.y;
+ // decode: warp w takes the 16-row slices w (8 tiles each)
+ for (int ks = warp; ks < 8; ks += kWarps) {
+ const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;
+ for (int t = 0; t < 8; ++t) {
+ float v[8];
+ decode8<K, cb>(src + t * W, lane, v);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);
+ const int c = t * 16 + (lane >> 2) + 8 * (j >> 2);
+ tile[r * P + c] = __float2half_rn(v[j]);
+ }
+ }
+ }
+ __syncthreads();
+ // H along the outputs (each row of 128 is one block)
+ for (int r = warp; r < kCols; r += kWarps) {
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[r * P + lane * 4 + i]);
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) tile[r * P + lane * 4 + i] = __float2half_rn(v[i]);
+ }
+ __syncthreads();
+ // H along the inputs, the scales, the transposed store
+ for (int c = warp; c < kCols; c += kWarps) {
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[(lane * 4 + i) * P + c]);
+ had128(v, lane);
+ const float sv = h2f(svh[nb * kCols + c]);
+ uint16_t* d = dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float w = v[i] * sv * h2f(suh[kb * kCols + lane * 4 + i]);
+ if (bf16) {
+ const uint32_t u = __float_as_uint(w);
+ d[i] = (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16); // round to nearest even
+ } else {
+ d[i] = __half_as_ushort(__float2half_rn(w));
+ }
+ }
+ }
+}
+
+// grid (n / 128, k / 16): W' row-major, one tile per warp
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) recon_inner_kernel(const uint32_t* __restrict__ tr, int ld, int n,
+ uint16_t* dst) {
+ constexpr int W = 8 * K;
+ const int lane = threadIdx.x & 31, t = threadIdx.x >> 5;
+ const int tc = blockIdx.x * 8 + t, ks = blockIdx.y;
+ float v[8];
+ decode8<K, cb>(tr + ((size_t) ks * ld + tc) * W, lane, v);
+#pragma unroll
+ for (int j = 0; j < 8; ++j) {
+ const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);
+ const int c = tc * 16 + (lane >> 2) + 8 * (j >> 2);
+ dst[(size_t) r * n + c] = __half_as_ushort(__float2half_rn(v[j]));
+ }
+}
+
+// grid (n / 128, k / 128): W' of the 128 x 128 block, transposed (dst[c][r]) - the decoded values alone. Each warp
+// decodes its slice's 8 tiles into the block's tile (half2 pairs of rows, conflict-free), then writes 256-byte rows.
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) inner_rows_kernel(const uint32_t* __restrict__ tr, int ld, int k,
+ uint16_t* __restrict__ dst) {
+ constexpr int W = 8 * K;
+ constexpr int P = kCols + 8; // a column's 128 rows, padded (halves; 16-byte rows)
+ __shared__ __align__(16) __half tile[kCols * P]; // tile[c * P + r]
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
+ const int nb = blockIdx.x, kb = blockIdx.y;
+ for (int ks = warp; ks < 8; ks += kWarps) {
+ const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;
+ const int r = ks * 16 + 2 * t;
+#pragma unroll
+ for (int tc = 0; tc < 8; ++tc) {
+ uint32_t f[4];
+ frag8<K, cb>(src + tc * W, lane, f);
+ const int c = tc * 16 + g;
+ *reinterpret_cast<uint32_t*>(&tile[c * P + r]) = f[0];
+ *reinterpret_cast<uint32_t*>(&tile[c * P + r + 8]) = f[1];
+ *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r]) = f[2];
+ *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r + 8]) = f[3];
+ }
+ }
+ __syncthreads();
+ for (int c = warp; c < kCols; c += kWarps)
+ *reinterpret_cast<uint2*>(dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4) =
+ *reinterpret_cast<const uint2*>(&tile[c * P + lane * 4]);
+}
+
+// one warp per 128 values of a row: the rotated FP16 input of an EXL3 matrix (rows_rot_f16) ...
+__global__ void __launch_bounds__(kThreads) rows_rot_f16_kernel(const float* __restrict__ x, int ldx,
+ const int* __restrict__ idx, int nrows, int ncols,
+ const uint16_t* __restrict__ suh, uint16_t* out) {
+ const int lane = threadIdx.x & 31, nb = ncols / kCols;
+ const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);
+ if (w >= nrows * nb) return;
+ const int r = w / nb, b = w % nb;
+ Xin xin;
+ xin.f = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;
+ float v[4], sc[4];
+ chunk_load(xin, suh, b * kCols, lane, v, sc);
+ chunk_store(v, sc, lane, reinterpret_cast<__half*>(out) + (size_t) r * ncols + b * kCols);
+}
+// ... and its output rotation in place (block_out's arithmetic)
+__global__ void __launch_bounds__(kThreads) rows_rot_out_kernel(float* y, int ldy, int nrows, int ncols,
+ const uint16_t* __restrict__ svh) {
+ const int lane = threadIdx.x & 31, nb = ncols / kCols;
+ const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);
+ if (w >= nrows * nb) return;
+ const int r = w / nb, b = w % nb;
+ float* p = y + (size_t) r * ldy + b * kCols;
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = p[lane * 4 + i];
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) p[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[b * kCols + lane * 4 + i]));
+}
+
+__global__ void rows_f16_kernel(const float* __restrict__ x, int ldx, const int* __restrict__ idx, int ncols,
+ uint16_t* __restrict__ out) {
+ const int r = blockIdx.y;
+ const float* src = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;
+ uint16_t* dst = out + (size_t) r * ncols;
+ for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < ncols; c += gridDim.x * blockDim.x)
+ dst[c] = __half_as_ushort(__float2half_rn(src[c]));
+}
+
+// ---------------------------------------------------------------- the prompt path's tensor-core GEMMs (sm_80+)
+// A block: BM rows x 128 outputs (one output rotation block). Per 128 inputs the block's rows are scaled by suh,
+// rotated and stored FP16 in shared memory (the A operands); per 16-row slice each warp decodes its 16 x 16 tile
+// straight into mma.m16n8k16's B fragments - a lane's 8 steps ARE the fragment order (steps 0,1 / 2,3 rows 2t+{0,1} /
+// 2t+8+{0,1} of column g, steps 4..7 the same of column g + 8) - and multiplies every m-tile of the block with them.
+// The output rotation and svh run on the F32 tile in shared memory.
+#if !defined(STRATA_USE_HIP)
+namespace tc {
+
+constexpr int BM = 32, MT = BM / 16, AP = 136; // rows a block, its m-tiles, the A rows' pitch (halves)
+
+__device__ __forceinline__ void mma16816(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "
+ "{%0,%1,%2,%3};\n"
+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1));
+#endif
+}
+// the lane's 4 B registers of tile t (words staged in st): b[0..1] columns 0..7, b[2..3] columns 8..15
+template <int K, int cb>
+__device__ __forceinline__ void tile_frags(const uint32_t* st, int lane, uint32_t* b) {
+ uint32_t f[4];
+ frag8<K, cb>(st, lane, f);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) b[i] = f[i];
+}
+
+// A[r][j] = FP16(H(x_r * suh)[j]), j < 128, for the block's rows (zero past nrows). x_r: F32 row idx[r] of xf (ldx)
+// when xf is given, else FP16 row r of xh (ldx). Every warp, BM / 8 rows each.
+__device__ __forceinline__ void load_a(const float* __restrict__ xf, const int* __restrict__ idx,
+ const __half* __restrict__ xh, int ldx, int r0, int nrows, int k0,
+ const uint16_t* __restrict__ suh, __half* A) {
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ for (int rr = warp; rr < BM; rr += kWarps) {
+ float v[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ const int r = r0 + rr;
+ if (rr < nrows) {
+ const int c = k0 + lane * 4;
+ if (xf != nullptr) {
+ const float* src = xf + (size_t) idx[r] * ldx + c;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = src[i];
+ } else {
+ const __half* src = xh + (size_t) r * ldx + c;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = __half2float(src[i]);
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[c + i]);
+ had128(v, lane);
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) A[rr * AP + lane * 4 + i] = __float2half_rn(v[i]);
+ }
+}
+
+// the m-tile's A registers for the 16-column slice s of A
+__device__ __forceinline__ void a_frags(const __half* A, int mt, int s, int lane, uint32_t* a) {
+ const int g = lane >> 2, t = lane & 3;
+ const __half* p = A + (mt * 16 + g) * AP + s * 16 + 2 * t;
+ a[0] = *reinterpret_cast<const uint32_t*>(p);
+ a[1] = *reinterpret_cast<const uint32_t*>(p + 8 * AP);
+ a[2] = *reinterpret_cast<const uint32_t*>(p + 8);
+ a[3] = *reinterpret_cast<const uint32_t*>(p + 8 * AP + 8);
+}
+
+// acc (this warp's 16 columns of the block's BM rows) -> T[BM][128] F32
+__device__ __forceinline__ void store_acc(const float (&acc)[MT][2][4], float* T) {
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;
+#pragma unroll
+ for (int mt = 0; mt < MT; ++mt)
+#pragma unroll
+ for (int ns = 0; ns < 2; ++ns) {
+ const int c = warp * 16 + ns * 8 + 2 * t, r = mt * 16 + g;
+ T[r * 128 + c] = acc[mt][ns][0];
+ T[r * 128 + c + 1] = acc[mt][ns][1];
+ T[(r + 8) * 128 + c] = acc[mt][ns][2];
+ T[(r + 8) * 128 + c + 1] = acc[mt][ns][3];
+ }
+}
+
+struct GArgs {
+ const uint8_t* wbase; // expert e's blob at wbase + e * stride (dense: 0, the offsets are pointers)
+ size_t stride;
+ size_t tr[3], suh[3], svh[3];
+ int ld[3]; // tiles per 16-row slice of each matrix
+ const int* bounds; // device: expert e's rows [bounds[e], bounds[e + 1]) (null: one matrix, rows [0, m))
+ int row0, m; // local row of a sorted row: bounds[e] - row0 + r
+ int k, n; // of the matrix (gate/up: n_embd, n_ff)
+ const float* xf; // F32 input rows by index (idx[local row]) ...
+ const int* idx;
+ const __half* xh; // ... or FP16 rows by local row
+ int ldx;
+ float* y; // F32 output rows (ldy), y = beta y + out ...
+ int ldy;
+ float beta;
+ __half* h; // ... or (gate/up) the swiglu as FP16 rows (ldh)
+ int ldh;
+ float limit;
+};
+
+// one matrix: y = beta y + H(H(x * suh) W') * svh. grid (row blocks, n / 128, experts)
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) gemm_kernel(const GArgs a) {
+ constexpr int W = 8 * K;
+ __shared__ __align__(16) unsigned char sm[BM * 128 * 4]; // A (BM x AP halves), then the F32 tile
+ __shared__ uint32_t stage[kWarps][64];
+ __half* A = reinterpret_cast<__half*>(sm);
+ float* T = reinterpret_cast<float*>(sm);
+ const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;
+ const int e0 = a.bounds != nullptr ? a.bounds[e] - a.row0 : 0;
+ const int ne = a.bounds != nullptr ? a.bounds[e + 1] - a.bounds[e] : a.m;
+ const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);
+ if (nrows <= 0) return;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const uint32_t* tr = (const uint32_t*) (blob + a.tr[0]);
+ const uint16_t* suh = (const uint16_t*) (blob + a.suh[0]);
+ const uint16_t* svh = (const uint16_t*) (blob + a.svh[0]);
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ float acc[MT][2][4] = {};
+ for (int k0 = 0; k0 < a.k; k0 += 128) {
+ __syncthreads();
+ load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, suh, A);
+ __syncthreads();
+#pragma unroll 1
+ for (int s = 0; s < 8; ++s) {
+ const int ks = k0 / 16 + s;
+ const uint32_t* src = tr + ((size_t) ks * a.ld[0] + g * 8 + warp) * W;
+ __syncwarp();
+ for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];
+ __syncwarp();
+ uint32_t b[4];
+ tile_frags<K, cb>(stage[warp], lane, b);
+#pragma unroll
+ for (int mt = 0; mt < MT; ++mt) {
+ uint32_t af[4];
+ a_frags(A, mt, s, lane, af);
+ mma16816(acc[mt][0], af, b[0], b[1]);
+ mma16816(acc[mt][1], af, b[2], b[3]);
+ }
+ }
+ }
+ __syncthreads();
+ store_acc(acc, T);
+ __syncthreads();
+ for (int rr = warp; rr < nrows; rr += kWarps) {
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = T[rr * 128 + lane * 4 + i];
+ had128(v, lane);
+ float* yr = a.y + (size_t) (r0 + rr) * a.ldy + g * 128 + lane * 4;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float o = v[i] * h2f(svh[g * 128 + lane * 4 + i]);
+ yr[i] = a.beta != 0.0f ? a.beta * yr[i] + o : o;
+ }
+ }
+}
+
+// gate and up (matrices 0 and 1) over the same rows, the clamped swiglu, FP16 h. grid (row blocks, n / 128, experts)
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const GArgs a) {
+ constexpr int W = 8 * K;
+ __shared__ __align__(16) unsigned char sm[2 * BM * 128 * 4]; // A_g, A_u, then the two F32 tiles
+ __shared__ uint32_t stage[kWarps][64];
+ __half* A[2] = {reinterpret_cast<__half*>(sm), reinterpret_cast<__half*>(sm) + BM * AP};
+ float* T[2] = {reinterpret_cast<float*>(sm), reinterpret_cast<float*>(sm) + BM * 128};
+ const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;
+ const int e0 = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];
+ const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);
+ if (nrows <= 0) return;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ float acc[2][MT][2][4] = {};
+ for (int k0 = 0; k0 < a.k; k0 += 128) {
+ __syncthreads();
+ for (int m = 0; m < 2; ++m)
+ load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, (const uint16_t*) (blob + a.suh[m]), A[m]);
+ __syncthreads();
+#pragma unroll 1
+ for (int s = 0; s < 8; ++s) {
+ const int ks = k0 / 16 + s;
+#pragma unroll
+ for (int m = 0; m < 2; ++m) {
+ const uint32_t* src = (const uint32_t*) (blob + a.tr[m]) + ((size_t) ks * a.ld[m] + g * 8 + warp) * W;
+ __syncwarp();
+ for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];
+ __syncwarp();
+ uint32_t b[4];
+ tile_frags<K, cb>(stage[warp], lane, b);
+#pragma unroll
+ for (int mt = 0; mt < MT; ++mt) {
+ uint32_t af[4];
+ a_frags(A[m], mt, s, lane, af);
+ mma16816(acc[m][mt][0], af, b[0], b[1]);
+ mma16816(acc[m][mt][1], af, b[2], b[3]);
+ }
+ }
+ }
+ }
+ __syncthreads();
+ store_acc(acc[0], T[0]);
+ store_acc(acc[1], T[1]);
+ __syncthreads();
+ const uint16_t* svg = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[0]) + g * 128;
+ const uint16_t* svu = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[1]) + g * 128;
+ for (int rr = warp; rr < nrows; rr += kWarps) {
+ float vg[4], vu[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ vg[i] = T[0][rr * 128 + lane * 4 + i];
+ vu[i] = T[1][rr * 128 + lane * 4 + i];
+ }
+ had128(vg, lane);
+ had128(vu, lane);
+ __half* hr = a.h + (size_t) (r0 + rr) * a.ldh + g * 128 + lane * 4;
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ float gv = vg[i] * h2f(svg[lane * 4 + i]), uv = vu[i] * h2f(svu[lane * 4 + i]);
+ if (a.limit > 0.0f) {
+ gv = fminf(gv, a.limit);
+ uv = fminf(fmaxf(uv, -a.limit), a.limit);
+ }
+ hr[i] = __float2half_rn((gv / (1.0f + expf(-gv))) * uv);
+ }
+ }
+}
+
+} // namespace tc
+#endif
+
+// ---------------------------------------------------------------- dispatch
+// F(K, cb) for K 1..8 and the three codebooks
+template <template <int, int> class F, typename... A>
+bool dispatch(int K, int cb, A&&... args) {
+#define EXL3_CASE(KK) \
+ case KK: \
+ if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...); \
+ else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...); \
+ else F<KK, 0>::run(std::forward<A>(args)...); \
+ return true;
+ switch (K) {
+ EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)
+ default: return false;
+ }
+#undef EXL3_CASE
+}
+
+template <int K, int cb> struct RunMv {
+ static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }
+};
+template <int K, int cb> struct RunWin {
+ static void run(const KWin& a, int groups, cudaStream_t s) {
+ win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups, 2u), kThreads, 0, s>>>(a);
+ }
+};
+template <int K, int cb> struct RunWinDown {
+ static void run(const KWin& a, int groups, cudaStream_t s) {
+ win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, 0, s>>>(a);
+ }
+};
+template <int K, int cb> struct RunGU {
+ static void run(const KMoe& a, cudaStream_t s) {
+ moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k, 2u), kThreads, 0, s>>>(a);
+ }
+};
+template <int K, int cb> struct RunDown {
+ static void run(const KMoe& a, cudaStream_t s) {
+ moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);
+ }
+};
+template <int K, int cb> struct RunRows {
+ static void run(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
+ recon_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(
+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.suh, m.svh, m.k, dst, bf16);
+ }
+};
+#if !defined(STRATA_USE_HIP)
+template <int K, int cb> struct RunGemm {
+ static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) { tc::gemm_kernel<K, cb><<<grid, kThreads, 0, s>>>(a); }
+};
+template <int K, int cb> struct RunGateUp {
+ static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) {
+ tc::gate_up_kernel<K, cb><<<grid, kThreads, 0, s>>>(a);
+ }
+};
+#endif
+template <int K, int cb> struct RunInnerRows {
+ static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {
+ inner_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(
+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.k, dst);
+ }
+};
+template <int K, int cb> struct RunInner {
+ static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {
+ recon_inner_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / 16)), kThreads, 0, s>>>(
+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.n, dst);
+ }
+};
+
+// ---------------------------------------------------------------- per-stream scratch
+constexpr size_t kPartFloats = (size_t) 2 << 20; // split partials: 8 MB
+constexpr size_t kCounters = 16384;
+constexpr size_t kCntGU = 8192; // the counters from here on: the experts' gate / up pairs
+constexpr size_t kHFloats = (size_t) 8 * 8192; // the experts' h
+constexpr size_t kUFloats = (size_t) 128 * 4096; // the experts' up values: a window's 16 tokens x 8, n_ff <= 4096
+struct Scratch {
+ float* part = nullptr;
+ unsigned* cnt = nullptr;
+ float* h = nullptr;
+ float* u = nullptr;
+};
+std::mutex g_mu;
+std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;
+
+Scratch* scratch(cudaStream_t s) {
+ int dev = 0;
+ cudaGetDevice(&dev);
+ // (the last stream this thread used: an engine thread per device launches without the lock)
+ thread_local int t_dev = -1;
+ thread_local cudaStream_t t_s = nullptr;
+ thread_local Scratch* t_sc = nullptr;
+ if (t_sc != nullptr && t_dev == dev && t_s == s) return t_sc;
+ std::lock_guard<std::mutex> lk(g_mu);
+ auto it = g_scratch.find({dev, s});
+ if (it != g_scratch.end()) {
+ t_dev = dev, t_s = s, t_sc = &it->second;
+ return &it->second;
+ }
+ cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;
+ if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {
+ std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");
+ return nullptr;
+ }
+ Scratch sc;
+ if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||
+ cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||
+ cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||
+ cudaMalloc(&sc.u, kUFloats * sizeof(float)) != cudaSuccess ||
+ cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {
+ std::fprintf(stderr, "exl3: scratch did not allocate\n");
+ return nullptr;
+ }
+ cudaStreamSynchronize(s);
+ return &(g_scratch[{dev, s}] = sc);
+}
+
+int sm_count() {
+ int dev = 0, sms = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
+ return std::max(1, sms);
+}
+
+// Whether the current device decodes with v1.1.8's kernels (exl3_legacy.cu): NVIDIA before sm_80, where this file's
+// tiles run on the CUDA cores and were slower; STRATA_GLM_EXL3_DECODE=legacy | new picks one on any GPU (A/B). The
+// token path and the verify windows take the same family on a device, so a window stays bit for bit its token path.
+// Asked once per device.
+bool use_legacy() {
+#if defined(STRATA_USE_HIP)
+ return false;
+#else
+ static std::atomic<int> cache[64]; // 0 not asked, 1 v1.1.8's kernels, 2 this file's
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 64) return false;
+ int c = cache[dev].load(std::memory_order_acquire);
+ if (c == 0) {
+ int major = 0;
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ const char* v = std::getenv("STRATA_GLM_EXL3_DECODE");
+ c = v != nullptr && std::strcmp(v, "legacy") == 0 ? 1 : v != nullptr && std::strcmp(v, "new") == 0 ? 2
+ : major < 8 ? 1
+ : 2;
+ if (c == 1) std::fprintf(stderr, "exl3: CUDA%d decodes with v1.1.8's kernels\n", dev);
+ cache[dev].store(c, std::memory_order_release);
+ }
+ return c == 1;
+#endif
+}
+
+// the blocks of mv_kernel<K> an SM of the current device keeps at once (its registers and shared memory), asked once
+// per device and K
+template <int K, int cb> struct OccMv {
+ static void run(int& n) {
+ if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&n, mv_kernel<K, cb>, kThreads, 0) != cudaSuccess) {
+ cudaGetLastError();
+ n = 2;
+ }
+ }
+};
+int mv_occ(int K) {
+ static std::atomic<int> cache[64][9]; // 0: not asked
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 64 || K < 1 || K > 8) return 2;
+ int n = cache[dev][K].load(std::memory_order_acquire);
+ if (n == 0) {
+ n = 2;
+ dispatch<OccMv>(K, 2, n);
+ n = std::max(1, n);
+ cache[dev][K].store(n, std::memory_order_release);
+ }
+ return n;
+}
+
+// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block (the same for every
+// activation count, so a job of several sums each activation as a job of one). About two blocks per SM; where an SM
+// keeps three or more (`occ`: the RTX 3090's at K > 4) four a SM, down to 128 inputs a block - its decode sets the pace
+// there, and the more, shorter blocks balance it over the SMs (RTX 3090, K6 / K5, us: a KDA layer's 8192 x 4096
+// output 47.6 -> 45.5, the dense layers' 4096 x 12288 gate 58.9 -> 52.5 and 12288 x 4096 down 59.3 -> 52.2, the
+// shared expert's 4096 x 2048 17.4 -> 16.5); with two a SM, a launch that would spill a short second wave (more blocks
+// than the SMs keep, less than one and a half times) takes twice the splits (RTX 5060 Ti, K5: that gate 99.4 -> 84.8
+// us, that down 100.6 -> 84.9)
+// call_groups: the output blocks the call's jobs of this code count together against the target, not the job's own -
+// where an SM keeps three or more, all of them (RTX 3090, K6, a KDA layer's q, k and v, three 4096 x 8192 jobs: 1536
+// blocks 119.6 us, 384 blocks 105.9); at two a SM only jobs of the same shape (RTX 5060 Ti: those q, k and v 191 ->
+// 185 us, a shared expert's gate and up 35.9 -> 35.0; the MLA layer's q_a and kv_a together would lose 8%). Never the
+// activations: a window's token gets the token path's split
+void mv_split(const Mat& m, int sms, int occ, int& S, int& kr, int call_groups = 0) {
+ const bool many = occ >= 3;
+ const int groups = std::max(m.n / kCols, call_groups);
+ const int kr_min = many ? 128 : 256, target = (many ? 4 : 2) * sms, P = occ * sms;
+ S = (m.k + kMaxKR - 1) / kMaxKR;
+ while (groups * S < target && m.k / (S * 2) >= kr_min) S *= 2;
+ if (!many && groups * S > P && 2 * groups * S < 3 * P && m.k / (S * 2) >= kr_min) S *= 2;
+ kr = ((m.k + S - 1) / S + 127) / 128 * 128;
+ S = (m.k + kr - 1) / kr;
+}
+
+} // namespace
+
+Mat view(const Mat& m, int c0, int nc) {
+ Mat v = m;
+ v.ld = m.ld > 0 ? m.ld : m.n / 16;
+ v.trellis = m.trellis + (size_t) (c0 / 16) * 8 * m.K;
+ v.svh = m.svh + c0;
+ v.n = nc;
+ return v;
+}
+
+bool supported(const Mat& m) {
+ return m.trellis != nullptr && m.suh != nullptr && m.svh != nullptr && m.k > 0 && m.n > 0 && m.k % 128 == 0 &&
+ m.n % 128 == 0 && m.K >= 1 && m.K <= 8 && m.cb >= 0 && m.cb <= 2;
+}
+
+void set_skip(const int* flag) { g_skip = flag; }
+
+int mv_splits(const Mat& m) {
+ if (!supported(m)) return 0;
+ int S = 1, kr = 0;
+ mv_split(m, sm_count(), mv_occ(m.K), S, kr);
+ return S;
+}
+
+int mv_cols(const Mat& m) {
+ if (!supported(m)) return 0;
+#if !defined(STRATA_USE_HIP)
+ if (use_legacy()) return 4; // (v1.1.8's kernels: the runs of four their callers have always used)
+#endif
+ const int S = mv_splits(m);
+ if (S <= 0) return 0;
+ if (S == 1) return kTok;
+ return (int) std::max<size_t>(1, std::min<size_t>(kTok, kPartFloats / ((size_t) S * m.n)));
+}
+
+int launch_errors() {
+#if !defined(STRATA_USE_HIP)
+ return g_launch_errors.load(std::memory_order_relaxed) + legacy::launch_errors();
+#else
+ return g_launch_errors.load(std::memory_order_relaxed);
+#endif
+}
+
+bool prepare(cudaStream_t s) {
+ EXL3_LEGACY(prepare(s));
+ return scratch(s) != nullptr;
+}
+
+bool several_ok() {
+ EXL3_LEGACY(several_ok());
+ return true;
+}
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+ EXL3_LEGACY(mv(jobs, n, s));
+ if (n <= 0 || n > kMaxMvJobs) return false;
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ const int sms = sm_count();
+ // the launch units: a job's activations in runs of up to kTok (one pass over its weights each)
+ struct Unit {
+ int j, c0, nc;
+ };
+ Unit units[kMaxMvJobs * kMaxMvCols];
+ int nu = 0;
+ const int run = kTok;
+ for (int j = 0; j < n; ++j) {
+ const Mat& m = *jobs[j].m;
+ if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||
+ jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {
+ std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,
+ m.K, m.cb, jobs[j].ncol);
+ return false;
+ }
+ for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};
+ }
+ // each job's call_groups (mv_split): the output blocks of the call's jobs of its code - of its shape too where an SM
+ // keeps fewer than three
+ int cg[kMaxMvJobs] = {};
+ for (int j = 0; j < n; ++j) {
+ const Mat& a = *jobs[j].m;
+ const bool many = mv_occ(a.K) >= 3;
+ for (int q = 0; q < n; ++q) {
+ const Mat& b = *jobs[q].m;
+ if (b.K == a.K && b.cb == a.cb && (many || (b.k == a.k && b.n == a.n))) cg[j] += b.n / kCols;
+ }
+ }
+ bool done[sizeof(units) / sizeof(units[0])] = {};
+ for (int i = 0; i < nu; ++i) {
+ if (done[i]) continue;
+ const Mat& m0 = *jobs[units[i].j].m;
+ // a launch: the units of one (K, codebook), as many as KBatch and the split scratch take (the next launch on
+ // the stream may use the scratch again: this one's last blocks are done with it by then)
+ KBatch b{};
+ int blocks = 0, nb = 0;
+ size_t part_at = 0, cnt_at = 0;
+ for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {
+ const MvJob& job = jobs[units[u].j];
+ const Mat& m = *job.m;
+ if (done[u] || m.K != m0.K || m.cb != m0.cb) continue;
+ const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;
+ int S = 1, kr = 0;
+ mv_split(m, sms, mv_occ(m.K), S, kr, cg[units[u].j]);
+ const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;
+ if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {
+ if (nb > 0) continue; // the next launch
+ std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);
+ return false;
+ }
+ done[u] = true;
+ KJob& J = b.j[nb];
+ J.tr = m.trellis;
+ J.suh = m.suh;
+ J.svh = m.svh;
+ J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;
+ J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;
+ J.y = job.y + c0 * job.y_cs;
+ J.bias = job.bias;
+ J.alpha = job.alpha;
+ J.k = m.k;
+ J.n = m.n;
+ J.ld = m.ld > 0 ? m.ld : m.n / 16;
+ J.S = S;
+ J.kr = kr;
+ J.part = sc->part + part_at;
+ J.cnt = sc->cnt + cnt_at;
+ J.nc = nc;
+ J.xf_cs = job.xf_cs;
+ J.xq_cs = job.xq_cs;
+ J.y_cs = job.y_cs;
+ part_at += need;
+ if (S > 1) cnt_at += (size_t) groups;
+ blocks += groups * S;
+ b.blk_end[nb++] = blocks;
+ }
+ b.n = nb;
+ b.skip = g_skip;
+ if (!dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)) return false;
+ }
+ launch_check("mv");
+ return true;
+}
+
+static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||
+ a.n_ff > kMaxMoeK || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||
+ (size_t) (a.n_embd / kCols) > kCntGU || kCntGU + (size_t) a.k * (a.n_ff / kCols) > kCounters) {
+ std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);
+ return false;
+ }
+ m = KMoe{};
+ m.plan_ptr = a.plan_ptr;
+ m.plan_w = a.plan_w;
+ m.cpu_part = a.cpu_part;
+ m.cpu_flag = a.cpu_flag;
+ m.h = sc->h;
+ m.u = sc->u;
+ m.part = sc->part;
+ m.cnt = sc->cnt;
+ m.cnt_gu = sc->cnt + kCntGU;
+ for (int r = 0; r < 3; ++r) {
+ m.suh[r] = a.lay.suh[r];
+ m.svh[r] = a.lay.svh[r];
+ m.tr[r] = a.lay.trellis[r];
+ }
+ m.k = a.k;
+ m.n_embd = a.n_embd;
+ m.n_ff = a.n_ff;
+ m.limit = a.limit;
+ m.skip = g_skip;
+ return true;
+}
+
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {
+ EXL3_LEGACY(moe_gate_up(a, x, h, s));
+ KMoe m;
+ if (!moe_args(a, m, s)) return;
+ if (a.lay.K[0] != a.lay.K[1]) {
+ std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);
+ return;
+ }
+ m.x = x;
+ if (h != nullptr) m.h = h;
+ if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);
+ launch_check("moe_gate_up");
+}
+
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
+ EXL3_LEGACY(moe_down(a, h, sh_out, out, s));
+ KMoe m;
+ if (!moe_args(a, m, s)) return;
+ if (h != nullptr) m.h = const_cast<float*>(h);
+ m.sh_out = sh_out;
+ m.out = out;
+ if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);
+ launch_check("moe_down");
+}
+
+bool win_experts(const WinArgs& a, cudaStream_t s) {
+ EXL3_LEGACY(win_experts(a, s));
+ if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||
+ a.n_ff > kMaxMoeK || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {
+ std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,
+ a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);
+ return false;
+ }
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)
+ if ((size_t) a.T * 8 * a.n_ff > kUFloats || kCntGU + (size_t) groups * (a.n_ff / kCols) > kCounters) {
+ std::fprintf(stderr, "exl3 win_experts: %d tokens are more than the scratch takes\n", a.T);
+ return false;
+ }
+ KWin w{};
+ w.grp_ptr = a.grp_ptr;
+ w.grp_start = a.grp_start;
+ w.grp_ent = a.grp_ent;
+ w.n_groups = a.n_groups;
+ w.plan_ptr = a.plan_ptr;
+ w.plan_w = a.plan_w;
+ w.cpu_part = a.cpu_part;
+ w.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;
+ w.cpu_flag = a.cpu_flag;
+ w.x = a.x;
+ w.x_stride = a.x_stride;
+ w.h = a.h;
+ w.u = sc->u;
+ w.cnt_gu = sc->cnt + kCntGU;
+ w.part = a.part;
+ w.sh_out = a.sh_out;
+ w.out = a.out;
+ w.out_stride = a.out_stride;
+ for (int r = 0; r < 3; ++r) {
+ w.suh[r] = a.lay.suh[r];
+ w.svh[r] = a.lay.svh[r];
+ w.tr[r] = a.lay.trellis[r];
+ }
+ w.k = a.k;
+ w.n_embd = a.n_embd;
+ w.n_ff = a.n_ff;
+ w.limit = a.limit;
+ if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||
+ !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))
+ return false;
+ if (a.wait_ptr != nullptr) win_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);
+ win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);
+ launch_check("win_experts");
+ return true;
+}
+
+bool mma_supported() {
+#if defined(STRATA_USE_HIP)
+ return false;
+#else
+ int dev = 0, major = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);
+ return major >= 8;
+#endif
+}
+
+void set_gate_up(const SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,
+ cudaStream_t s) {
+#if !defined(STRATA_USE_HIP)
+ if (sa.lay.K[0] != sa.lay.K[1]) {
+ std::fprintf(stderr, "exl3 set_gate_up: gate K %d != up K %d\n", sa.lay.K[0], sa.lay.K[1]);
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
+ tc::GArgs a{};
+ a.wbase = sa.wbase;
+ a.stride = sa.stride;
+ for (int r = 0; r < 2; ++r) {
+ a.tr[r] = sa.lay.trellis[r];
+ a.suh[r] = sa.lay.suh[r];
+ a.svh[r] = sa.lay.svh[r];
+ a.ld[r] = sa.n_ff / 16;
+ }
+ a.bounds = sa.bounds;
+ a.row0 = sa.row0;
+ a.k = sa.n_embd;
+ a.n = sa.n_ff;
+ a.xf = x;
+ a.idx = row_idx;
+ a.ldx = ldx;
+ a.h = (__half*) h16;
+ a.ldh = ldh;
+ a.limit = sa.limit;
+ const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_ff / kCols), (unsigned) sa.n_exp);
+ if (!dispatch<RunGateUp>(sa.lay.K[0], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_gate_up: K\n");
+ launch_check("set_gate_up");
+#else
+ (void) sa; (void) x; (void) ldx; (void) row_idx; (void) h16; (void) ldh; (void) s;
+#endif
+}
+
+void set_down(const SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {
+#if !defined(STRATA_USE_HIP)
+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;
+ tc::GArgs a{};
+ a.wbase = sa.wbase;
+ a.stride = sa.stride;
+ a.tr[0] = sa.lay.trellis[2];
+ a.suh[0] = sa.lay.suh[2];
+ a.svh[0] = sa.lay.svh[2];
+ a.ld[0] = sa.n_embd / 16;
+ a.bounds = sa.bounds;
+ a.row0 = sa.row0;
+ a.k = sa.n_ff;
+ a.n = sa.n_embd;
+ a.xh = (const __half*) h16;
+ a.ldx = ldh;
+ a.y = y;
+ a.ldy = ldy;
+ const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_embd / kCols), (unsigned) sa.n_exp);
+ if (!dispatch<RunGemm>(sa.lay.K[2], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_down: K\n");
+ launch_check("set_down");
+#else
+ (void) sa; (void) h16; (void) ldh; (void) y; (void) ldy; (void) s;
+#endif
+}
+
+void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s) {
+#if !defined(STRATA_USE_HIP)
+ if (rows <= 0) return;
+ tc::GArgs a{};
+ a.wbase = nullptr;
+ a.tr[0] = (size_t) m.trellis;
+ a.suh[0] = (size_t) m.suh;
+ a.svh[0] = (size_t) m.svh;
+ a.ld[0] = m.ld > 0 ? m.ld : m.n / 16;
+ a.m = rows;
+ a.k = m.k;
+ a.n = m.n;
+ a.xh = (const __half*) x16;
+ a.ldx = ldx;
+ a.y = y;
+ a.ldy = ldy;
+ a.beta = beta;
+ const dim3 grid((unsigned) ((rows + tc::BM - 1) / tc::BM), (unsigned) (m.n / kCols), 1);
+ if (!supported(m) || !dispatch<RunGemm>(m.K, m.cb, a, grid, s)) {
+ std::fprintf(stderr, "exl3 gemm_rows: unsupported matrix\n");
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ launch_check("gemm_rows");
+#else
+ (void) m; (void) x16; (void) ldx; (void) rows; (void) y; (void) ldy; (void) beta; (void) s;
+#endif
+}
+
+void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {
+ if (!supported(m) || !dispatch<RunRows>(m.K, m.cb, m, dst, bf16, s)) {
+ std::fprintf(stderr, "exl3 reconstruct_rows: unsupported matrix\n");
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ launch_check("reconstruct_rows");
+}
+
+void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s) {
+ if (!supported(m) || !dispatch<RunInnerRows>(m.K, m.cb, m, dst, s)) {
+ std::fprintf(stderr, "exl3 reconstruct_inner_rows: unsupported matrix\n");
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ launch_check("reconstruct_inner_rows");
+}
+
+void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,
+ cudaStream_t s) {
+ if (nrows <= 0) return;
+ const long long warps = (long long) nrows * (ncols / kCols);
+ rows_rot_f16_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(x, ldx, idx, nrows, ncols, suh,
+ out);
+ launch_check("rows_rot_f16");
+}
+
+void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s) {
+ if (nrows <= 0) return;
+ const long long warps = (long long) nrows * (ncols / kCols);
+ rows_rot_out_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(y, ldy, nrows, ncols, svh);
+ launch_check("rows_rot_out");
+}
+
+void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {
+ if (nrows <= 0) return;
+ rows_f16_kernel<<<dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s>>>(x, ldx, idx,
+ ncols, out);
+ launch_check("rows_f16");
+}
+
+void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s) {
+ if (!supported(m) || !dispatch<RunInner>(m.K, m.cb, m, dst, s)) {
+ std::fprintf(stderr, "exl3 reconstruct_inner: unsupported matrix\n");
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ return;
+ }
+ launch_check("reconstruct_inner");
+}
+
+} // namespace strata::kernels::exl3
diff --git a/src/dsv41/vendor/exl3_cpu.cpp b/src/dsv41/vendor/exl3_cpu.cpp
new file mode 100644
index 000000000..11eca4ebe
--- /dev/null
+++ b/src/dsv41/vendor/exl3_cpu.cpp
@@ -0,0 +1,1269 @@
+// src/kernels/cpu/exl3_cpu.cpp - EXL3 routed experts on the CPU (see include/strata/kernels/cpu/exl3_cpu.hpp).
+//
+// The vector kernels follow exllamav3's CPU expert kernel (MIT, turboderp - exllamav3_ext/cpu/moe_mul1.cpp, its
+// AVX-512 BW and AVX2 tiers): a tile row's 16 states come out of the tile's words with cross-lane permutes and a
+// per-lane funnel shift from compile-time tables, the codebook product's bytes are added with vpmaddubsw against ones
+// and vpmaddwd against ones. Unlike there, the activations stay F32 (no int8 rows) and the affine offset is the
+// device's FP16 constant, so the CPU lane computes what the device kernels compute up to the FP16 rounding of each
+// weight. The AVX-512 and AVX2 tiers do the same FMAs in the same order: they give the same bits (the bands,
+// the split-K pieces, the GEMM's sums and the vectorised transforms alike).
+#include "strata/kernels/cpu/exl3_cpu.hpp"
+
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/f16_bits.hpp"
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <utility>
+#include <vector>
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define EXL3_X86 1
+#include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+
+#if defined(EXL3_X86) && (defined(__GNUC__) || defined(__clang__))
+#define EXL3_AVX512 __attribute__((target("avx512f,avx512bw,fma")))
+#define EXL3_AVX2 __attribute__((target("avx2,fma,f16c")))
+#define EXL3_AVX2_INLINE __attribute__((target("avx2,fma,f16c"), always_inline)) inline
+#elif defined(_MSC_VER)
+#define EXL3_AVX512
+#define EXL3_AVX2
+#define EXL3_AVX2_INLINE __forceinline
+#else
+#define EXL3_AVX512
+#define EXL3_AVX2
+#define EXL3_AVX2_INLINE inline
+#endif
+
+namespace strata::kernels::cpu {
+namespace {
+
+constexpr uint32_t kMul1 = 0x83DCD12Du;
+
+float k_inv() { return f32_from_f16(0x1eee); }
+float k_zero() { return 1024.0f * f32_from_f16(0x1eee) + f32_from_f16(0xc931); } // w = s * k_inv + k_zero
+
+// the 128-point Sylvester transform over sqrt(128), in place
+void had128(float* v) {
+ for (int h = 1; h < 128; h <<= 1)
+ for (int i = 0; i < 128; i += 2 * h)
+ for (int j = i; j < i + h; ++j) {
+ const float a = v[j], b = v[j + h];
+ v[j] = a + b;
+ v[j + h] = a - b;
+ }
+ const float s = 0.088388347648f;
+ for (int i = 0; i < 128; ++i) v[i] *= s;
+}
+
+// step p of a tile: row (p/8 % 4) * 2 + (p & 1) + 8 (p/2 & 1), column p/32 + 8 (p/4 & 1); inv[row * 16 + col] = p
+constexpr std::array<uint16_t, 256> make_inv() {
+ std::array<uint16_t, 256> inv{};
+ for (int p = 0; p < 256; ++p) {
+ const int lane = p >> 3, j = p & 7;
+ const int r = (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);
+ const int c = (lane >> 2) + 8 * (j >> 2);
+ inv[(size_t) (r * 16 + c)] = (uint16_t) p;
+ }
+ return inv;
+}
+
+// a tile row's 16 states: column c's 16-bit window ends at bit b1 = (p + 1 + 256) K of the tile's MSB-first stream;
+// its words i0 (the window's first bit) and i1 (its last), and the right shift of (w[i0] : w[i1]) that aligns it
+template <int K>
+struct RowTab {
+ int32_t i0[16][16] = {}, i1[16][16] = {}, sr[16][16] = {}, sl[16][16] = {};
+ uint16_t hi0[16] = {}, hi1[16] = {}; // columns whose word is in the second register pair (>= 32)
+};
+template <int K>
+constexpr RowTab<K> make_rowtab() {
+ RowTab<K> t{};
+ constexpr int W = 8 * K;
+ const auto inv = make_inv();
+ for (int r = 0; r < 16; ++r)
+ for (int c = 0; c < 16; ++c) {
+ const int p = inv[(size_t) (r * 16 + c)];
+ const int b1 = (p + 1 + 256) * K;
+ const int w1 = ((b1 - 1) >> 5) % W, w0 = ((b1 - 16) >> 5) % W;
+ const int sh = (((b1 - 1) >> 5) + 1) * 32 - b1;
+ t.i0[r][c] = w0;
+ t.i1[r][c] = w1;
+ t.sr[r][c] = sh;
+ t.sl[r][c] = 32 - sh; // a shift by 32 clears the lane: a window inside one word
+ if (w0 >= 32) t.hi0[r] = (uint16_t) (t.hi0[r] | (1u << c));
+ if (w1 >= 32) t.hi1[r] = (uint16_t) (t.hi1[r] | (1u << c));
+ }
+ return t;
+}
+
+// ---------------------------------------------------------------- scalar
+template <int K>
+void band_scalar(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* acc /* 128 */) {
+ constexpr int W = 8 * K;
+ static constexpr RowTab<K> T = make_rowtab<K>();
+ const int ld = m.n / 16;
+ for (int i = 0; i < 128; ++i) acc[i] = 0.0f;
+ for (int ks = ks0; ks < ks1; ++ks)
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t* w = m.trellis + ((size_t) ks * ld + tc0 + t) * W;
+ for (int r = 0; r < 16; ++r) {
+ const float xv = xh[ks * 16 + r];
+ for (int c = 0; c < 16; ++c) {
+ const uint64_t a = w[T.i0[r][c]], b = w[T.i1[r][c]];
+ const uint32_t st = (uint32_t) (((a << 32) | b) >> T.sr[r][c]) & 0xffffu;
+ const uint32_t x = st * kMul1;
+ const int s = (int) ((x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24));
+ acc[t * 16 + c] += (float) s * xv;
+ }
+ }
+ }
+}
+
+// ---------------------------------------------------------------- AVX-512
+#if defined(EXL3_X86)
+template <int K>
+EXL3_AVX512 inline __m512i gather_words(const RowTab<K>& T, const int32_t (&idx)[16][16], const uint16_t* hi, int r,
+ __m512i p0, __m512i p1, __m512i p2, __m512i p3) {
+ const __m512i ix = _mm512_loadu_si512((const void*) idx[r]);
+ __m512i v = _mm512_permutex2var_epi32(p0, ix, p1);
+ if constexpr (8 * K > 32) {
+ if (hi[r] != 0) v = _mm512_mask_blend_epi32((__mmask16) hi[r], v, _mm512_permutex2var_epi32(p2, ix, p3));
+ }
+ (void) T;
+ return v;
+}
+
+template <int K>
+EXL3_AVX512 void band_avx512(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* out /* 128 */) {
+ constexpr int W = 8 * K;
+ static constexpr RowTab<K> T = make_rowtab<K>();
+ constexpr auto ld_mask = [](int n) -> __mmask16 {
+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);
+ };
+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const __m512i mult = _mm512_set1_epi32((int) kMul1);
+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);
+ const __m512i low16 = _mm512_set1_epi32(0xffff);
+ __m512 acc[8];
+ for (int t = 0; t < 8; ++t) acc[t] = _mm512_setzero_ps();
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ // the band's 8 tiles of this slice are contiguous; the next slices' are one row of tiles on
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ const float* xk = xh + ks * 16;
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t* w = row + t * W;
+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);
+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();
+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();
+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();
+ __m512 a = acc[t];
+#pragma GCC unroll 16
+ for (int r = 0; r < 16; ++r) {
+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);
+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);
+ const __m512i st = _mm512_and_si512(
+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),
+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),
+ low16);
+ const __m512i bs = _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16);
+ a = _mm512_fmadd_ps(_mm512_cvtepi32_ps(bs), _mm512_set1_ps(xk[r]), a);
+ }
+ acc[t] = a;
+ }
+ }
+ for (int t = 0; t < 8; ++t) _mm512_storeu_ps(out + t * 16, acc[t]);
+}
+
+// band_avx512 for N activations at once (xh[i] -> out[i]), as exllamav3's CPU kernel takes up to 4 token rows: each
+// tile row's 16 states are pulled out and byte-summed once, then one FMA per activation - activation i's FMAs are
+// band_avx512's for it alone, in its order, so out[i] is the same bit for bit. An extra activation costs one FMA of
+// the row's ~12 instructions. (The tiles' accumulators live in memory, loaded once a tile and slice as there: only
+// the N of the tile in hand take registers.)
+template <int K, int N>
+EXL3_AVX512 void band_avx512_n(const Exl3Part& m, const float* const* xh, int tc0, int ks0, int ks1,
+ float* const* out /* N x 128 */) {
+ constexpr int W = 8 * K;
+ static constexpr RowTab<K> T = make_rowtab<K>();
+ constexpr auto ld_mask = [](int n) -> __mmask16 {
+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);
+ };
+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const __m512i mult = _mm512_set1_epi32((int) kMul1);
+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);
+ const __m512i low16 = _mm512_set1_epi32(0xffff);
+ __m512 acc[8][N];
+ for (int t = 0; t < 8; ++t)
+ for (int i = 0; i < N; ++i) acc[t][i] = _mm512_setzero_ps();
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ const float* xk[N];
+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t* w = row + t * W;
+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);
+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();
+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();
+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();
+ __m512 a[N];
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) a[i] = acc[t][i];
+#pragma GCC unroll 16
+ for (int r = 0; r < 16; ++r) {
+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);
+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);
+ const __m512i st = _mm512_and_si512(
+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),
+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),
+ low16);
+ const __m512 f = _mm512_cvtepi32_ps(
+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16));
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) a[i] = _mm512_fmadd_ps(f, _mm512_set1_ps(xk[i][r]), a[i]);
+ }
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) acc[t][i] = a[i];
+ }
+ }
+ for (int i = 0; i < N; ++i)
+ for (int t = 0; t < 8; ++t) _mm512_storeu_ps(out[i] + t * 16, acc[t][i]);
+}
+
+// rows_avx512_n: slices [ks0, ks1) of the tile columns [t0, t1) - a split-K piece, or a run of its columns - for N
+// activations (xh[i] -> acc[i], indexed by column; columns 16 t0 .. 16 t1 overwritten). The tile rows are read in
+// memory order, where a band's tiles are 1 KB runs a whole tile row apart: 40 threads of a 2-socket Xeon 6152 read a K4
+// expert from DRAM in 0.14 ms in pieces, 0.21 in bands (~90 vs ~60 GB/s), the kernel the same. A tile's 16 sums per
+// activation live in acc between slices; each row's decode and FMAs are band_avx512's, so acc[i] is the N = 1 call's
+// for xh[i] alone, bit for bit, whatever the columns' run. The prefetch runs 16 tiles ahead in the job's own order
+// (into the next slice past the run's end, not the next job's columns)
+template <int K, int N>
+EXL3_AVX512 void rows_avx512_n(const Exl3Part& m, const float* const* xh, int ks0, int ks1, int t0, int t1,
+ float* const* acc) {
+ constexpr int W = 8 * K;
+ static constexpr RowTab<K> T = make_rowtab<K>();
+ constexpr auto ld_mask = [](int n) -> __mmask16 {
+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);
+ };
+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const __m512i mult = _mm512_set1_epi32((int) kMul1);
+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);
+ const __m512i low16 = _mm512_set1_epi32(0xffff);
+ for (int i = 0; i < N; ++i)
+ for (int c = 16 * t0; c < 16 * t1; c += 16) _mm512_storeu_ps(acc[i] + c, _mm512_setzero_ps());
+ const int span = t1 - t0;
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ float xk[N][16];
+ for (int i = 0; i < N; ++i)
+ for (int r = 0; r < 16; ++r) xk[i][r] = xh[i][ks * 16 + r];
+ for (int t = t0; t < t1; ++t) {
+ const uint32_t* w = row + (size_t) t * W;
+ const uint32_t* pf = t + 16 < t1 || span == ld ? w + 16 * W : row + row_words + (size_t) (t + 16 - span) * W;
+ for (int l = 0; l < W * 4; l += 64) _mm_prefetch((const char*) pf + l, _MM_HINT_T0);
+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);
+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();
+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();
+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();
+ __m512 a[N];
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) a[i] = _mm512_loadu_ps(acc[i] + t * 16);
+#pragma GCC unroll 16
+ for (int r = 0; r < 16; ++r) {
+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);
+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);
+ const __m512i st = _mm512_and_si512(
+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),
+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),
+ low16);
+ const __m512 f = _mm512_cvtepi32_ps(
+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16));
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) a[i] = _mm512_fmadd_ps(f, _mm512_set1_ps(xk[i][r]), a[i]);
+ }
+#pragma GCC unroll 8
+ for (int i = 0; i < N; ++i) _mm512_storeu_ps(acc[i] + t * 16, a[i]);
+ }
+ }
+}
+
+// band_gemm: a band's byte-sum products for many activations - each chunk of kGemmK inputs decoded ONCE into F32
+// (wc, in L2), then multiplied as a GEMM: tiles of TB activations x 64 columns, the sums in registers, two FMAs a
+// load. The prompt path's CPU rows, where an expert serves tens to hundreds of tokens: one decode (~6 instructions a
+// 16-weight row) for all of them instead of one per kExl3CpuMaxN.
+constexpr int kGemmK = 256;
+
+template <int K>
+EXL3_AVX512 void decode_chunk(const Exl3Part& m, int tc0, int ks0, int ks1, float* wc /* (ks1 - ks0) 16 x 128 */) {
+ constexpr int W = 8 * K;
+ static constexpr RowTab<K> T = make_rowtab<K>();
+ constexpr auto ld_mask = [](int n) -> __mmask16 {
+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);
+ };
+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);
+ const size_t row_words = (size_t) (m.n / 16) * W;
+ const __m512i mult = _mm512_set1_epi32((int) kMul1);
+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);
+ const __m512i low16 = _mm512_set1_epi32(0xffff);
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ float* wr = wc + (size_t) (ks - ks0) * 16 * 128;
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t* w = row + t * W;
+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);
+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();
+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();
+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();
+#pragma GCC unroll 16
+ for (int r = 0; r < 16; ++r) {
+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);
+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);
+ const __m512i st = _mm512_and_si512(
+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),
+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),
+ low16);
+ const __m512i bs =
+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16);
+ _mm512_store_ps(wr + r * 128 + t * 16, _mm512_cvtepi32_ps(bs));
+ }
+ }
+ }
+}
+
+// acc[t][0..128) += sum_{k < kr} x[t][k] wc[k][..] for TB activations
+template <int TB>
+EXL3_AVX512 void gemm_tile(const float* wc, int kr, const float* const* x, float* const* acc) {
+ for (int cb = 0; cb < 128; cb += 64) {
+ __m512 a[TB][4];
+ for (int t = 0; t < TB; ++t)
+ for (int j = 0; j < 4; ++j) a[t][j] = _mm512_loadu_ps(acc[t] + cb + j * 16);
+ const float* w = wc + cb;
+ for (int k = 0; k < kr; ++k, w += 128) {
+ const __m512 w0 = _mm512_load_ps(w), w1 = _mm512_load_ps(w + 16), w2 = _mm512_load_ps(w + 32),
+ w3 = _mm512_load_ps(w + 48);
+#pragma GCC unroll 8
+ for (int t = 0; t < TB; ++t) {
+ const __m512 xv = _mm512_set1_ps(x[t][k]);
+ a[t][0] = _mm512_fmadd_ps(xv, w0, a[t][0]);
+ a[t][1] = _mm512_fmadd_ps(xv, w1, a[t][1]);
+ a[t][2] = _mm512_fmadd_ps(xv, w2, a[t][2]);
+ a[t][3] = _mm512_fmadd_ps(xv, w3, a[t][3]);
+ }
+ }
+ for (int t = 0; t < TB; ++t)
+ for (int j = 0; j < 4; ++j) _mm512_storeu_ps(acc[t] + cb + j * 16, a[t][j]);
+ }
+}
+
+// nb bands from tile column tc0 (acc[t]: the activation's 128 nb outputs from there): each chunk of the inputs is
+// taken for every band in turn, so the activations' chunk (n x kGemmK floats) comes from L2 for the nb bands - an
+// expert's rows are its own tokens, and with a job a band they came from DRAM again for every band (~16x the
+// expert's weights in bytes)
+template <int K>
+EXL3_AVX512 void band_gemm_avx512(const Exl3Part& m, const float* const* xh, int n, int tc0, int nb,
+ float* const* acc, float* wc) {
+ constexpr int KS = kGemmK / 16;
+ constexpr int TB = 6; // 24 sums + 4 weights + the activation: 29 of the 32 registers
+ for (int t = 0; t < n; ++t) std::memset(acc[t], 0, (size_t) nb * 128 * sizeof(float));
+ std::vector<float*> ab((size_t) n);
+ for (int ks0 = 0; ks0 < m.k / 16; ks0 += KS) {
+ const int ks1 = std::min(m.k / 16, ks0 + KS), kr = (ks1 - ks0) * 16;
+ for (int bb = 0; bb < nb; ++bb) {
+ decode_chunk<K>(m, tc0 + 8 * bb, ks0, ks1, wc);
+ for (int t = 0; t < n; ++t) ab[(size_t) t] = acc[t] + bb * 128;
+ float* const* ap = ab.data();
+ int t = 0;
+ for (; t + TB <= n; t += TB) {
+ const float* xp[TB];
+ for (int i = 0; i < TB; ++i) xp[i] = xh[t + i] + ks0 * 16;
+ gemm_tile<TB>(wc, kr, xp, ap + t);
+ }
+ if (t < n) { // the last 1..TB-1 activations in one pass too
+ const float* xp[TB];
+ for (int i = 0; i < n - t; ++i) xp[i] = xh[t + i] + ks0 * 16;
+ switch (n - t) {
+ case 1: gemm_tile<1>(wc, kr, xp, ap + t); break;
+ case 2: gemm_tile<2>(wc, kr, xp, ap + t); break;
+ case 3: gemm_tile<3>(wc, kr, xp, ap + t); break;
+ case 4: gemm_tile<4>(wc, kr, xp, ap + t); break;
+ default: gemm_tile<5>(wc, kr, xp, ap + t); break;
+ }
+ }
+ }
+ }
+}
+
+// had128 in 8 registers: every butterfly takes the operands the scalar loop takes (a lane's partner by a permute; the
+// lower lane a + b, the upper b's lane t - r = a - b), so the result is had128's bit for bit
+EXL3_AVX512 inline __m512 bfly(__m512 r, __m512 t, __mmask16 upper) {
+ return _mm512_mask_sub_ps(_mm512_add_ps(r, t), upper, t, r);
+}
+EXL3_AVX512 void had128_avx512(float* v) {
+ __m512 r[8];
+ for (int i = 0; i < 8; ++i) r[i] = _mm512_loadu_ps(v + 16 * i);
+ for (int i = 0; i < 8; ++i) {
+ r[i] = bfly(r[i], _mm512_permute_ps(r[i], 0b10110001), (__mmask16) 0xAAAA); // h = 1
+ r[i] = bfly(r[i], _mm512_permute_ps(r[i], 0b01001110), (__mmask16) 0xCCCC); // h = 2
+ r[i] = bfly(r[i], _mm512_shuffle_f32x4(r[i], r[i], 0b10110001), (__mmask16) 0xF0F0); // h = 4
+ r[i] = bfly(r[i], _mm512_shuffle_f32x4(r[i], r[i], 0b01001110), (__mmask16) 0xFF00); // h = 8
+ }
+ for (int h = 1; h < 8; h <<= 1) // h = 16, 32, 64: whole registers
+ for (int i = 0; i < 8; i += 2 * h)
+ for (int j = i; j < i + h; ++j) {
+ const __m512 a = r[j], b = r[j + h];
+ r[j] = _mm512_add_ps(a, b);
+ r[j + h] = _mm512_sub_ps(a, b);
+ }
+ const __m512 s = _mm512_set1_ps(0.088388347648f);
+ for (int i = 0; i < 8; ++i) _mm512_storeu_ps(v + 16 * i, _mm512_mul_ps(r[i], s));
+}
+
+// exl3_cpu_prepare_block / exl3_cpu_band_out with the FP16 scales converted 16 at a time (vcvtph2ps: exact, as
+// f32_from_f16) and the block's sum in eight double lanes
+EXL3_AVX512 float prepare_block_avx512(const uint16_t* suh, const float* x, float* xh) {
+ for (int i = 0; i < 128; i += 16) {
+ const __m512 sc = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) (suh + i)));
+ _mm512_storeu_ps(xh + i, _mm512_mul_ps(_mm512_loadu_ps(x + i), sc));
+ }
+ had128_avx512(xh);
+ __m512d s0 = _mm512_setzero_pd(), s1 = _mm512_setzero_pd();
+ for (int i = 0; i < 128; i += 16) {
+ s0 = _mm512_add_pd(s0, _mm512_cvtps_pd(_mm256_loadu_ps(xh + i)));
+ s1 = _mm512_add_pd(s1, _mm512_cvtps_pd(_mm256_loadu_ps(xh + i + 8)));
+ }
+ return (float) _mm512_reduce_add_pd(_mm512_add_pd(s0, s1));
+}
+EXL3_AVX512 void band_out_avx512(const float* acc, float ki, float kz, const uint16_t* svh, float* y) {
+ const __m512 vk = _mm512_set1_ps(ki), vz = _mm512_set1_ps(kz);
+ for (int i = 0; i < 128; i += 16) _mm512_storeu_ps(y + i, _mm512_fmadd_ps(_mm512_loadu_ps(acc + i), vk, vz));
+ had128_avx512(y);
+ for (int i = 0; i < 128; i += 16) {
+ const __m512 sc = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) (svh + i)));
+ _mm512_storeu_ps(y + i, _mm512_mul_ps(_mm512_loadu_ps(y + i), sc));
+ }
+}
+
+// ---------------------------------------------------------------- AVX2
+// The AVX-512 tier on 8-lane registers, a tile row's 16 columns as two halves of 8 (after exllamav3's AVX2 tier).
+// Column c' of a half reads the tile's words K c' + base (mod 8K) - its 8 words lie in one residue class of the word
+// index mod K, and its funnel shift is the same for all 8 (the layout's row-shift invariant). So each tile's K
+// registers are regrouped once into its K classes, and every half's gather is then ONE vpermd of a class register
+// (none where the lanes are in order already) instead of a permute and a blend per register the half reads: odd K
+// takes each class lane from exactly one register (blends only), K 2 / 4 / 8 are shuffles / a 4x4 / an 8x8 transpose,
+// K 6 a permute and a blend per register. exllamav3 gets the same single permute from a planar order repacked at load
+// (`_copy_repacked`); here the regrouping costs a few register ops a tile and the RAM tier and the GPUs keep the
+// tiles as they are. The shifts are immediates, and the byte sum and the FMA are the AVX-512 tier's, lane for lane
+// in the same (slice, row) order: the two tiers give the same bits.
+
+// the lane of tile word w in its class register (class w % K; avx2_classes builds them)
+constexpr int avx2_slot(int K, int w) {
+ return (K & 1) ? (w & 7)
+ : K == 2 ? (w & 4) | ((w >> 2) & 2) | ((w >> 1) & 1)
+ : K == 4 ? (w >> 3) + 4 * ((w >> 2) & 1)
+ : K == 8 ? (w >> 3)
+ : w / K;
+}
+
+// per row and half: the right shift, whether the 16-bit windows sit in one word (no second gather), and for the
+// window's first and last word the class register and the vpermd lanes that put its 8 words in column order
+template <int K>
+struct Avx2Tab {
+ int sh[16][2] = {};
+ bool one[16][2] = {};
+ int cls[16][2][2] = {};
+ bool ident[16][2][2] = {};
+ int32_t idx[16][2][2][8] = {};
+ bool ok = true; // the invariants above hold for every row and half
+};
+template <int K>
+constexpr Avx2Tab<K> make_avx2tab() {
+ Avx2Tab<K> t{};
+ const RowTab<K> rt = make_rowtab<K>();
+ for (int r = 0; r < 16; ++r)
+ for (int h = 0; h < 2; ++h) {
+ const int c0 = 8 * h;
+ t.sh[r][h] = rt.sr[r][c0];
+ t.one[r][h] = true;
+ for (int c = c0; c < c0 + 8; ++c) {
+ if (rt.sr[r][c] != t.sh[r][h]) t.ok = false;
+ if (rt.i0[r][c] != rt.i1[r][c]) t.one[r][h] = false;
+ }
+ if (t.one[r][h] != (t.sh[r][h] <= 16)) t.ok = false;
+ for (int wd = 0; wd < 2; ++wd) {
+ const auto& ix = wd == 0 ? rt.i0 : rt.i1;
+ t.cls[r][h][wd] = ix[r][c0] % K;
+ t.ident[r][h][wd] = true;
+ for (int c = 0; c < 8; ++c) {
+ if (ix[r][c0 + c] % K != t.cls[r][h][wd]) t.ok = false;
+ t.idx[r][h][wd][c] = avx2_slot(K, ix[r][c0 + c]);
+ if (t.idx[r][h][wd][c] != c) t.ident[r][h][wd] = false;
+ }
+ }
+ }
+ return t;
+}
+template <int K>
+inline constexpr Avx2Tab<K> kAvx2Tab = make_avx2tab<K>();
+
+// odd K: lane l of class rho is register j's, the one j with (8 j + l) % K == rho
+template <int K, int RHO, int J>
+constexpr int avx2_odd_mask() {
+ int m = 0;
+ for (int l = 0; l < 8; ++l)
+ if ((8 * J + l) % K == RHO) m |= 1 << l;
+ return m;
+}
+template <int K, int RHO, int J = 1>
+EXL3_AVX2_INLINE __m256i avx2_odd_class(const __m256i (&R)[K], __m256i acc) {
+ if constexpr (J == K) {
+ return acc;
+ } else {
+ constexpr int m = avx2_odd_mask<K, RHO, J>(); // (clang's _mm256_blend_epi32 is a macro: no template commas)
+ return avx2_odd_class<K, RHO, J + 1>(R, _mm256_blend_epi32(acc, R[J], m));
+ }
+}
+template <int K, int... RHO>
+EXL3_AVX2_INLINE void avx2_odd_classes(const __m256i (&R)[K], __m256i (&C)[K], std::integer_sequence<int, RHO...>) {
+ ((C[RHO] = avx2_odd_class<K, RHO>(R, R[0])), ...);
+}
+// any K (used for K 6): class rho's word rho + K s at lane s - each register's words permuted there, then blended
+template <int K, int RHO, int J>
+constexpr int avx2_gen_mask() {
+ int m = 0;
+ for (int s = 0; s < 8; ++s)
+ if ((RHO + K * s) / 8 == J) m |= 1 << s;
+ return m;
+}
+template <int K, int RHO, int J, int S>
+constexpr int avx2_gen_lane() {
+ return (RHO + K * S) / 8 == J ? (RHO + K * S) % 8 : 0;
+}
+template <int K, int RHO, int J = 0>
+EXL3_AVX2_INLINE __m256i avx2_gen_class(const __m256i (&R)[K], __m256i acc) {
+ if constexpr (J == K) {
+ return acc;
+ } else if constexpr (avx2_gen_mask<K, RHO, J>() == 0) {
+ return avx2_gen_class<K, RHO, J + 1>(R, acc);
+ } else {
+ const __m256i p = _mm256_permutevar8x32_epi32(
+ R[J], _mm256_setr_epi32(avx2_gen_lane<K, RHO, J, 0>(), avx2_gen_lane<K, RHO, J, 1>(),
+ avx2_gen_lane<K, RHO, J, 2>(), avx2_gen_lane<K, RHO, J, 3>(),
+ avx2_gen_lane<K, RHO, J, 4>(), avx2_gen_lane<K, RHO, J, 5>(),
+ avx2_gen_lane<K, RHO, J, 6>(), avx2_gen_lane<K, RHO, J, 7>()));
+ constexpr int m = avx2_gen_mask<K, RHO, J>(); // (a macro under clang, as above)
+ return avx2_gen_class<K, RHO, J + 1>(R, _mm256_blend_epi32(acc, p, m));
+ }
+}
+template <int K, int... RHO>
+EXL3_AVX2_INLINE void avx2_gen_classes(const __m256i (&R)[K], __m256i (&C)[K], std::integer_sequence<int, RHO...>) {
+ ((C[RHO] = avx2_gen_class<K, RHO>(R, _mm256_setzero_si256())), ...);
+}
+// a 4x4 transpose in each 128-bit half: o[q] = lanes q and q + 4 of r0..r3
+EXL3_AVX2_INLINE void avx2_t4(__m256i r0, __m256i r1, __m256i r2, __m256i r3, __m256i (&o)[4]) {
+ const __m256i t0 = _mm256_unpacklo_epi32(r0, r1), t1 = _mm256_unpackhi_epi32(r0, r1);
+ const __m256i t2 = _mm256_unpacklo_epi32(r2, r3), t3 = _mm256_unpackhi_epi32(r2, r3);
+ o[0] = _mm256_unpacklo_epi64(t0, t2);
+ o[1] = _mm256_unpackhi_epi64(t0, t2);
+ o[2] = _mm256_unpacklo_epi64(t1, t3);
+ o[3] = _mm256_unpackhi_epi64(t1, t3);
+}
+// a tile's 8K words as its K class registers (word w in C[w % K] at lane avx2_slot(K, w))
+template <int K>
+EXL3_AVX2_INLINE void avx2_classes(const uint32_t* w, __m256i (&C)[K]) {
+ __m256i R[K];
+ for (int j = 0; j < K; ++j) R[j] = _mm256_loadu_si256((const __m256i*) (w + 8 * j));
+ if constexpr (K == 1) {
+ C[0] = R[0];
+ } else if constexpr (K % 2 == 1) {
+ avx2_odd_classes<K>(R, C, std::make_integer_sequence<int, K>{});
+ } else if constexpr (K == 2) {
+ const __m256 a = _mm256_castsi256_ps(R[0]), b = _mm256_castsi256_ps(R[1]);
+ C[0] = _mm256_castps_si256(_mm256_shuffle_ps(a, b, 0x88)); // words 0 2 8 10 | 4 6 12 14
+ C[1] = _mm256_castps_si256(_mm256_shuffle_ps(a, b, 0xDD)); // words 1 3 9 11 | 5 7 13 15
+ } else if constexpr (K == 4) {
+ avx2_t4(R[0], R[1], R[2], R[3], C);
+ } else if constexpr (K == 8) {
+ __m256i A[4], B[4];
+ avx2_t4(R[0], R[1], R[2], R[3], A);
+ avx2_t4(R[4], R[5], R[6], R[7], B);
+ for (int q = 0; q < 4; ++q) {
+ C[q] = _mm256_permute2x128_si256(A[q], B[q], 0x20);
+ C[q + 4] = _mm256_permute2x128_si256(A[q], B[q], 0x31);
+ }
+ } else {
+ avx2_gen_classes<K>(R, C, std::make_integer_sequence<int, K>{});
+ }
+}
+
+// the 8 words (first or last of each window) of row R's half H, in column order
+template <int K, int R, int H, int WD>
+EXL3_AVX2_INLINE __m256i avx2_gather(const __m256i (&C)[K]) {
+ constexpr const Avx2Tab<K>& T = kAvx2Tab<K>;
+ if constexpr (T.ident[R][H][WD]) {
+ return C[T.cls[R][H][WD]];
+ } else {
+ return _mm256_permutevar8x32_epi32(C[T.cls[R][H][WD]],
+ _mm256_loadu_si256((const __m256i*) T.idx[R][H][WD]));
+ }
+}
+
+struct Avx2Consts {
+ __m256i mult, ones8, ones16, low16;
+};
+EXL3_AVX2_INLINE Avx2Consts avx2_consts() {
+ return {_mm256_set1_epi32((int) kMul1), _mm256_set1_epi8(1), _mm256_set1_epi16(1), _mm256_set1_epi32(0xffff)};
+}
+
+// row R's half H: its 8 states, and their codebook byte sums as F32
+template <int K, int R, int H>
+EXL3_AVX2_INLINE __m256 avx2_sums(const __m256i (&C)[K], const Avx2Consts& k) {
+ constexpr const Avx2Tab<K>& T = kAvx2Tab<K>;
+ constexpr int sh = T.sh[R][H];
+ const __m256i b = avx2_gather<K, R, H, 1>(C);
+ __m256i st;
+ if constexpr (T.one[R][H] && sh == 16) {
+ st = _mm256_srli_epi32(b, 16);
+ } else if constexpr (T.one[R][H]) {
+ st = _mm256_and_si256(_mm256_srli_epi32(b, sh), k.low16);
+ } else {
+ const __m256i a = avx2_gather<K, R, H, 0>(C);
+ st = _mm256_and_si256(_mm256_or_si256(_mm256_srli_epi32(b, sh), _mm256_slli_epi32(a, 32 - sh)), k.low16);
+ }
+ return _mm256_cvtepi32_ps(
+ _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_mullo_epi32(st, k.mult), k.ones8), k.ones16));
+}
+
+template <int K, int R>
+EXL3_AVX2_INLINE void avx2_rows(const __m256i (&C)[K], const float* xk, __m256& lo, __m256& hi, const Avx2Consts& k) {
+ if constexpr (R < 16) {
+ const __m256 xv = _mm256_broadcast_ss(xk + R);
+ lo = _mm256_fmadd_ps(avx2_sums<K, R, 0>(C, k), xv, lo);
+ hi = _mm256_fmadd_ps(avx2_sums<K, R, 1>(C, k), xv, hi);
+ avx2_rows<K, R + 1>(C, xk, lo, hi, k);
+ }
+}
+
+template <int K>
+EXL3_AVX2 void band_avx2(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* out /* 128 */) {
+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");
+ constexpr int W = 8 * K;
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const Avx2Consts k = avx2_consts();
+ __m256 acc[8][2]; // the band's 8 tiles (16 registers: in memory, a tile's two taken at a time)
+ for (int t = 0; t < 8; ++t) acc[t][0] = acc[t][1] = _mm256_setzero_ps();
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ // the band's 8 tiles of this slice are contiguous; the next slices' are one row of tiles on
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ const float* xk = xh + ks * 16;
+ for (int t = 0; t < 8; ++t) {
+ __m256i C[K];
+ avx2_classes<K>(row + t * W, C);
+ __m256 lo = acc[t][0], hi = acc[t][1];
+ avx2_rows<K, 0>(C, xk, lo, hi, k);
+ acc[t][0] = lo;
+ acc[t][1] = hi;
+ }
+ }
+ for (int t = 0; t < 8; ++t) {
+ _mm256_storeu_ps(out + t * 16, acc[t][0]);
+ _mm256_storeu_ps(out + t * 16 + 8, acc[t][1]);
+ }
+}
+
+// band_avx2 for N activations at once, as band_avx512_n: a half row's states summed once, then one FMA per
+// activation - activation i's FMAs are band_avx2's for it alone, in its order, so out[i] is the same bit for bit.
+// Up to 3 activations both halves of a tile at once (2N accumulators, one broadcast a row and activation); more, a
+// tile's halves one after the other (N accumulators, not 2N: 16 registers) - 2 activations 1.55 -> 1.34 ms an
+// expert-token on one Zen 3 core, 3: 1.18 -> 1.11
+template <int N, int... I>
+EXL3_AVX2_INLINE void avx2_fma_n(__m256 f, const float* const (&xk)[N], int r, __m256 (&a)[N],
+ std::integer_sequence<int, I...>) {
+ ((a[I] = _mm256_fmadd_ps(f, _mm256_broadcast_ss(xk[I] + r), a[I])), ...);
+}
+template <int K, int N, int H, int R>
+EXL3_AVX2_INLINE void avx2_rows_n(const __m256i (&C)[K], const float* const (&xk)[N], __m256 (&a)[N],
+ const Avx2Consts& k) {
+ if constexpr (R < 16) {
+ avx2_fma_n<N>(avx2_sums<K, R, H>(C, k), xk, R, a, std::make_integer_sequence<int, N>{});
+ avx2_rows_n<K, N, H, R + 1>(C, xk, a, k);
+ }
+}
+template <int N, int... I>
+EXL3_AVX2_INLINE void avx2_fma_n2(__m256 f0, __m256 f1, const float* const (&xk)[N], int r, __m256 (&lo)[N],
+ __m256 (&hi)[N], std::integer_sequence<int, I...>) {
+ ((lo[I] = _mm256_fmadd_ps(f0, _mm256_broadcast_ss(xk[I] + r), lo[I]),
+ hi[I] = _mm256_fmadd_ps(f1, _mm256_broadcast_ss(xk[I] + r), hi[I])),
+ ...);
+}
+template <int K, int N, int R>
+EXL3_AVX2_INLINE void avx2_rows_n2(const __m256i (&C)[K], const float* const (&xk)[N], __m256 (&lo)[N],
+ __m256 (&hi)[N], const Avx2Consts& k) {
+ if constexpr (R < 16) {
+ avx2_fma_n2<N>(avx2_sums<K, R, 0>(C, k), avx2_sums<K, R, 1>(C, k), xk, R, lo, hi,
+ std::make_integer_sequence<int, N>{});
+ avx2_rows_n2<K, N, R + 1>(C, xk, lo, hi, k);
+ }
+}
+template <int K, int N>
+EXL3_AVX2 void band_avx2_n(const Exl3Part& m, const float* const* xh, int tc0, int ks0, int ks1,
+ float* const* out /* N x 128 */) {
+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");
+ constexpr int W = 8 * K;
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const Avx2Consts k = avx2_consts();
+ __m256 acc[8][2][N];
+ for (int t = 0; t < 8; ++t)
+ for (int h = 0; h < 2; ++h)
+ for (int i = 0; i < N; ++i) acc[t][h][i] = _mm256_setzero_ps();
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ const float* xk[N];
+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;
+ for (int t = 0; t < 8; ++t) {
+ __m256i C[K];
+ avx2_classes<K>(row + t * W, C);
+ if constexpr (N <= 3) {
+ __m256 lo[N], hi[N];
+ for (int i = 0; i < N; ++i) lo[i] = acc[t][0][i], hi[i] = acc[t][1][i];
+ avx2_rows_n2<K, N, 0>(C, xk, lo, hi, k);
+ for (int i = 0; i < N; ++i) acc[t][0][i] = lo[i], acc[t][1][i] = hi[i];
+ } else {
+ __m256 a[N];
+ for (int i = 0; i < N; ++i) a[i] = acc[t][0][i];
+ avx2_rows_n<K, N, 0, 0>(C, xk, a, k);
+ for (int i = 0; i < N; ++i) acc[t][0][i] = a[i], a[i] = acc[t][1][i];
+ avx2_rows_n<K, N, 1, 0>(C, xk, a, k);
+ for (int i = 0; i < N; ++i) acc[t][1][i] = a[i];
+ }
+ }
+ }
+ for (int i = 0; i < N; ++i)
+ for (int t = 0; t < 8; ++t) {
+ _mm256_storeu_ps(out[i] + t * 16, acc[t][0][i]);
+ _mm256_storeu_ps(out[i] + t * 16 + 8, acc[t][1][i]);
+ }
+}
+
+// #188's split-K piece (rows_avx512_n) on the AVX2 tier: slices [ks0, ks1) of EVERY column for N activations, the
+// tile rows in memory order, a tile's sums in acc between slices; each row's decode and FMAs are band_avx2's, so acc[i]
+// is the band kernel's for its columns and xh[i] alone, bit for bit (and rows_avx512_n's)
+template <int K, int N>
+EXL3_AVX2 void rows_avx2_n(const Exl3Part& m, const float* const* xh, int ks0, int ks1, int t0, int t1,
+ float* const* acc) {
+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");
+ constexpr int W = 8 * K;
+ const int ld = m.n / 16;
+ const size_t row_words = (size_t) ld * W;
+ const Avx2Consts k = avx2_consts();
+ for (int i = 0; i < N; ++i)
+ for (int c = 16 * t0; c < 16 * t1; c += 8) _mm256_storeu_ps(acc[i] + c, _mm256_setzero_ps());
+ const int span = t1 - t0;
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ const float* xk[N];
+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;
+ for (int t = t0; t < t1; ++t) {
+ const uint32_t* w = row + (size_t) t * W;
+ const uint32_t* pf = t + 16 < t1 || span == ld ? w + 16 * W : row + row_words + (size_t) (t + 16 - span) * W;
+ for (int l = 0; l < W * 4; l += 64) _mm_prefetch((const char*) pf + l, _MM_HINT_T0);
+ __m256i C[K];
+ avx2_classes<K>(w, C);
+ if constexpr (N == 1) {
+ __m256 lo = _mm256_loadu_ps(acc[0] + t * 16), hi = _mm256_loadu_ps(acc[0] + t * 16 + 8);
+ avx2_rows<K, 0>(C, xk[0], lo, hi, k);
+ _mm256_storeu_ps(acc[0] + t * 16, lo);
+ _mm256_storeu_ps(acc[0] + t * 16 + 8, hi);
+ } else if constexpr (N <= 3) {
+ __m256 lo[N], hi[N];
+ for (int i = 0; i < N; ++i)
+ lo[i] = _mm256_loadu_ps(acc[i] + t * 16), hi[i] = _mm256_loadu_ps(acc[i] + t * 16 + 8);
+ avx2_rows_n2<K, N, 0>(C, xk, lo, hi, k);
+ for (int i = 0; i < N; ++i)
+ _mm256_storeu_ps(acc[i] + t * 16, lo[i]), _mm256_storeu_ps(acc[i] + t * 16 + 8, hi[i]);
+ } else {
+ __m256 a[N];
+ for (int i = 0; i < N; ++i) a[i] = _mm256_loadu_ps(acc[i] + t * 16);
+ avx2_rows_n<K, N, 0, 0>(C, xk, a, k);
+ for (int i = 0; i < N; ++i) _mm256_storeu_ps(acc[i] + t * 16, a[i]), a[i] = _mm256_loadu_ps(acc[i] + t * 16 + 8);
+ avx2_rows_n<K, N, 1, 0>(C, xk, a, k);
+ for (int i = 0; i < N; ++i) _mm256_storeu_ps(acc[i] + t * 16 + 8, a[i]);
+ }
+ }
+ }
+}
+
+// #188's band GEMM (decode_chunk + gemm_tile) on the AVX2 tier: a chunk of a band's byte sums decoded once into F32,
+// then TB activations x 32 columns a pass, the sums in registers (12 of the 16 with TB 3) - each column's FMAs in the
+// band kernel's order (k ascending; x w + a as band_avx2's s x + a), so the same bits as gemm_tile on AVX-512
+template <int K, int R>
+EXL3_AVX2_INLINE void avx2_decode_rows(const __m256i (&C)[K], float* wr, int t, const Avx2Consts& k) {
+ if constexpr (R < 16) {
+ _mm256_store_ps(wr + R * 128 + t * 16, avx2_sums<K, R, 0>(C, k));
+ _mm256_store_ps(wr + R * 128 + t * 16 + 8, avx2_sums<K, R, 1>(C, k));
+ avx2_decode_rows<K, R + 1>(C, wr, t, k);
+ }
+}
+template <int K>
+EXL3_AVX2 void decode_chunk_avx2(const Exl3Part& m, int tc0, int ks0, int ks1, float* wc /* (ks1 - ks0) 16 x 128 */) {
+ constexpr int W = 8 * K;
+ const size_t row_words = (size_t) (m.n / 16) * W;
+ const Avx2Consts k = avx2_consts();
+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;
+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {
+ const char* pf = (const char*) (row + 4 * row_words);
+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);
+ float* wr = wc + (size_t) (ks - ks0) * 16 * 128;
+ for (int t = 0; t < 8; ++t) {
+ __m256i C[K];
+ avx2_classes<K>(row + t * W, C);
+ avx2_decode_rows<K, 0>(C, wr, t, k);
+ }
+ }
+}
+// acc[t][0..128) += sum_{k < kr} x[t][k] wc[k][..] for TB activations
+template <int TB>
+EXL3_AVX2 void gemm_tile_avx2(const float* wc, int kr, const float* const* x, float* const* acc) {
+ for (int cb = 0; cb < 128; cb += 32) {
+ __m256 a[TB][4];
+ for (int t = 0; t < TB; ++t)
+ for (int j = 0; j < 4; ++j) a[t][j] = _mm256_loadu_ps(acc[t] + cb + j * 8);
+ const float* w = wc + cb;
+ for (int q = 0; q < kr; ++q, w += 128) {
+ const __m256 w0 = _mm256_load_ps(w), w1 = _mm256_load_ps(w + 8), w2 = _mm256_load_ps(w + 16),
+ w3 = _mm256_load_ps(w + 24);
+ for (int t = 0; t < TB; ++t) {
+ const __m256 xv = _mm256_broadcast_ss(x[t] + q);
+ a[t][0] = _mm256_fmadd_ps(xv, w0, a[t][0]);
+ a[t][1] = _mm256_fmadd_ps(xv, w1, a[t][1]);
+ a[t][2] = _mm256_fmadd_ps(xv, w2, a[t][2]);
+ a[t][3] = _mm256_fmadd_ps(xv, w3, a[t][3]);
+ }
+ }
+ for (int t = 0; t < TB; ++t)
+ for (int j = 0; j < 4; ++j) _mm256_storeu_ps(acc[t] + cb + j * 8, a[t][j]);
+ }
+}
+template <int K>
+EXL3_AVX2 void band_gemm_avx2(const Exl3Part& m, const float* const* xh, int n, int tc0, int nb, float* const* acc,
+ float* wc) {
+ constexpr int KS = kGemmK / 16;
+ constexpr int TB = 3; // 12 sums + 4 weights of the 16 registers
+ for (int t = 0; t < n; ++t) std::memset(acc[t], 0, (size_t) nb * 128 * sizeof(float));
+ std::vector<float*> ab((size_t) n);
+ for (int ks0 = 0; ks0 < m.k / 16; ks0 += KS) {
+ const int ks1 = std::min(m.k / 16, ks0 + KS), kr = (ks1 - ks0) * 16;
+ for (int bb = 0; bb < nb; ++bb) {
+ decode_chunk_avx2<K>(m, tc0 + 8 * bb, ks0, ks1, wc);
+ for (int t = 0; t < n; ++t) ab[(size_t) t] = acc[t] + bb * 128;
+ float* const* ap = ab.data();
+ int t = 0;
+ for (; t + TB <= n; t += TB) {
+ const float* xp[TB];
+ for (int i = 0; i < TB; ++i) xp[i] = xh[t + i] + ks0 * 16;
+ gemm_tile_avx2<TB>(wc, kr, xp, ap + t);
+ }
+ if (t < n) {
+ const float* xp[TB];
+ for (int i = 0; i < n - t; ++i) xp[i] = xh[t + i] + ks0 * 16;
+ if (n - t == 1) gemm_tile_avx2<1>(wc, kr, xp, ap + t);
+ else gemm_tile_avx2<2>(wc, kr, xp, ap + t);
+ }
+ }
+ }
+}
+
+// had128 on 16 registers (exllamav3's hadamard_128_avx2): every butterfly takes the scalar loop's operands (a lane's
+// partner by a permute; the lower lane a + b, the upper t - r = a - b), so the result is had128's bit for bit
+EXL3_AVX2 void had128_avx2(float* v) {
+ __m256 r[16];
+ for (int i = 0; i < 16; ++i) r[i] = _mm256_loadu_ps(v + 8 * i);
+ for (int i = 0; i < 16; ++i) {
+ __m256 t = _mm256_permute_ps(r[i], 0b10110001); // h = 1
+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b10101010);
+ t = _mm256_permute_ps(r[i], 0b01001110); // h = 2
+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b11001100);
+ t = _mm256_permute2f128_ps(r[i], r[i], 0x01); // h = 4
+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b11110000);
+ }
+ for (int h = 1; h < 16; h <<= 1) // h = 8 .. 64: whole registers
+ for (int i = 0; i < 16; i += 2 * h)
+ for (int j = i; j < i + h; ++j) {
+ const __m256 a = r[j], b = r[j + h];
+ r[j] = _mm256_add_ps(a, b);
+ r[j + h] = _mm256_sub_ps(a, b);
+ }
+ const __m256 s = _mm256_set1_ps(0.088388347648f);
+ for (int i = 0; i < 16; ++i) _mm256_storeu_ps(v + 8 * i, _mm256_mul_ps(r[i], s));
+}
+
+// prepare_block_avx512 / band_out_avx512 on the AVX2 tier, the same operations in the same order: the FP16 scales by
+// vcvtph2ps (exact), the block's sum in the same eight double lanes (lanes 0-7 of the 16-float steps' first half, 8-15
+// of the second, added) and reduced as GCC's _mm512_reduce_add_pd does (upper four + lower four, then the two pairs,
+// then the last two), band_out's FMA - the same bits as the AVX-512 tier
+EXL3_AVX2 float prepare_block_avx2(const uint16_t* suh, const float* x, float* xh) {
+ for (int i = 0; i < 128; i += 8) {
+ const __m256 sc = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (suh + i)));
+ _mm256_storeu_ps(xh + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), sc));
+ }
+ had128_avx2(xh);
+ __m256d s0a = _mm256_setzero_pd(), s0b = _mm256_setzero_pd(), s1a = _mm256_setzero_pd(), s1b = _mm256_setzero_pd();
+ for (int i = 0; i < 128; i += 16) {
+ s0a = _mm256_add_pd(s0a, _mm256_cvtps_pd(_mm_loadu_ps(xh + i)));
+ s0b = _mm256_add_pd(s0b, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 4)));
+ s1a = _mm256_add_pd(s1a, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 8)));
+ s1b = _mm256_add_pd(s1b, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 12)));
+ }
+ const __m256d lo = _mm256_add_pd(s0a, s1a), hi = _mm256_add_pd(s0b, s1b); // the 8 lanes of s0 + s1
+ const __m256d t3 = _mm256_add_pd(hi, lo);
+ const __m128d t6 = _mm_add_pd(_mm256_extractf128_pd(t3, 1), _mm256_castpd256_pd128(t3));
+ return (float) (_mm_cvtsd_f64(t6) + _mm_cvtsd_f64(_mm_unpackhi_pd(t6, t6)));
+}
+EXL3_AVX2 void band_out_avx2(const float* acc, float ki, float kz, const uint16_t* svh, float* y) {
+ const __m256 vk = _mm256_set1_ps(ki), vz = _mm256_set1_ps(kz);
+ for (int i = 0; i < 128; i += 8) _mm256_storeu_ps(y + i, _mm256_fmadd_ps(_mm256_loadu_ps(acc + i), vk, vz));
+ had128_avx2(y);
+ for (int i = 0; i < 128; i += 8) {
+ const __m256 sc = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (svh + i)));
+ _mm256_storeu_ps(y + i, _mm256_mul_ps(_mm256_loadu_ps(y + i), sc));
+ }
+}
+
+// AVX-512 F and BW with FMA, the OS saving the state (STRATA_FORCE_AVX2 is the tier choice's, below)
+bool cpu_avx512_exl3() {
+ unsigned r[4] = {0, 0, 0, 0};
+ auto cpuid = [&](unsigned leaf, unsigned sub) {
+#if defined(_MSC_VER)
+ int x[4];
+ __cpuidex(x, (int) leaf, (int) sub);
+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+#endif
+ };
+ cpuid(0, 0);
+ if (r[0] < 7) return false;
+ cpuid(1, 0);
+ if (!((r[2] >> 27) & 1u) || !((r[2] >> 12) & 1u)) return false; // OSXSAVE, FMA
+#if defined(_MSC_VER)
+ const unsigned long long xcr0 = _xgetbv(0);
+#else
+ unsigned lo = 0, hi = 0;
+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;
+#endif
+ if ((xcr0 & 0xE6) != 0xE6) return false; // the OS saves the AVX-512 state
+ cpuid(7, 0);
+ return ((r[1] >> 16) & 1u) && ((r[1] >> 30) & 1u); // AVX-512F, AVX-512BW
+}
+// F16C (the AVX2 tier's FP16 scale conversions; every AVX2 CPU has it, asked anyway)
+bool cpu_f16c() {
+ unsigned r[4] = {0, 0, 0, 0};
+#if defined(_MSC_VER)
+ int x[4];
+ __cpuidex(x, 1, 0);
+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];
+#else
+ __cpuid_count(1, 0, r[0], r[1], r[2], r[3]);
+#endif
+ return ((r[2] >> 29) & 1u) != 0;
+}
+#endif
+
+// the tiers this CPU runs (bit t: tier t - 0 scalar, 1 AVX2 + FMA + F16C, 2 AVX-512 F/BW), and the one the lane takes:
+// the widest, but AVX2 on an AVX-512 CPU with STRATA_FORCE_AVX2=1 (to compare the two vector tiers)
+int tier_mask() {
+ static const int mask = [] {
+ int t = 1;
+#if defined(EXL3_X86)
+ if (cpu_avx2_ok() && cpu_f16c()) t |= 2;
+ if (cpu_avx512_exl3()) t |= 4;
+#endif
+ return t;
+ }();
+ return mask;
+}
+int detect_tier() {
+ const int t = tier_mask();
+ const char* f = std::getenv("STRATA_FORCE_AVX2");
+ if ((t & 4) && !(f != nullptr && f[0] == '1')) return 2;
+ return (t & 2) ? 1 : 0;
+}
+
+template <int K>
+void band_on(int tier, const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* acc) {
+#if defined(EXL3_X86)
+ if (tier == 2) {
+ band_avx512<K>(m, xh, tc0, ks0, ks1, acc);
+ return;
+ }
+ if (tier == 1) {
+ band_avx2<K>(m, xh, tc0, ks0, ks1, acc);
+ return;
+ }
+#endif
+ band_scalar<K>(m, xh, tc0, ks0, ks1, acc);
+}
+
+// n activations (1..kExl3CpuMaxN) on one band
+template <int K>
+void band_on_n(int tier, const Exl3Part& m, const float* const* xh, int n, int tc0, int ks0, int ks1,
+ float* const* acc) {
+#if defined(EXL3_X86)
+ if (tier == 2 && n > 1) {
+ switch (n) {
+ case 2: band_avx512_n<K, 2>(m, xh, tc0, ks0, ks1, acc); return;
+ case 3: band_avx512_n<K, 3>(m, xh, tc0, ks0, ks1, acc); return;
+ case 4: band_avx512_n<K, 4>(m, xh, tc0, ks0, ks1, acc); return;
+ case 5: band_avx512_n<K, 5>(m, xh, tc0, ks0, ks1, acc); return;
+ case 6: band_avx512_n<K, 6>(m, xh, tc0, ks0, ks1, acc); return;
+ case 7: band_avx512_n<K, 7>(m, xh, tc0, ks0, ks1, acc); return;
+ default: band_avx512_n<K, 8>(m, xh, tc0, ks0, ks1, acc); return;
+ }
+ }
+ if (tier == 1 && n > 1) {
+ switch (n) {
+ case 2: band_avx2_n<K, 2>(m, xh, tc0, ks0, ks1, acc); return;
+ case 3: band_avx2_n<K, 3>(m, xh, tc0, ks0, ks1, acc); return;
+ case 4: band_avx2_n<K, 4>(m, xh, tc0, ks0, ks1, acc); return;
+ case 5: band_avx2_n<K, 5>(m, xh, tc0, ks0, ks1, acc); return;
+ case 6: band_avx2_n<K, 6>(m, xh, tc0, ks0, ks1, acc); return;
+ case 7: band_avx2_n<K, 7>(m, xh, tc0, ks0, ks1, acc); return;
+ default: band_avx2_n<K, 8>(m, xh, tc0, ks0, ks1, acc); return;
+ }
+ }
+#endif
+ for (int i = 0; i < n; ++i) band_on<K>(tier, m, xh[i], tc0, ks0, ks1, acc[i]);
+}
+
+// a split-K piece's tile columns [t0, t1) for n activations (1..kExl3CpuMaxN); the scalar loop: each band's
+// band_scalar (t0, t1 multiples of 8 there)
+template <int K>
+void rows_on_n(int tier, const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int t0, int t1,
+ float* const* acc) {
+#if defined(EXL3_X86)
+ if (tier == 2) {
+ switch (n) {
+ case 1: rows_avx512_n<K, 1>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 2: rows_avx512_n<K, 2>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 3: rows_avx512_n<K, 3>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 4: rows_avx512_n<K, 4>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 5: rows_avx512_n<K, 5>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 6: rows_avx512_n<K, 6>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 7: rows_avx512_n<K, 7>(m, xh, ks0, ks1, t0, t1, acc); return;
+ default: rows_avx512_n<K, 8>(m, xh, ks0, ks1, t0, t1, acc); return;
+ }
+ }
+ if (tier == 1) {
+ switch (n) {
+ case 1: rows_avx2_n<K, 1>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 2: rows_avx2_n<K, 2>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 3: rows_avx2_n<K, 3>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 4: rows_avx2_n<K, 4>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 5: rows_avx2_n<K, 5>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 6: rows_avx2_n<K, 6>(m, xh, ks0, ks1, t0, t1, acc); return;
+ case 7: rows_avx2_n<K, 7>(m, xh, ks0, ks1, t0, t1, acc); return;
+ default: rows_avx2_n<K, 8>(m, xh, ks0, ks1, t0, t1, acc); return;
+ }
+ }
+#endif
+ for (int i = 0; i < n; ++i)
+ for (int c0 = 16 * t0; c0 < 16 * t1; c0 += 128) band_scalar<K>(m, xh[i], c0 / 16, ks0, ks1, acc[i] + c0);
+}
+
+void band_acc_tier(int tier, const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc) {
+ const int tc0 = c0 / 16;
+ switch (m.K) {
+ case 1: band_on<1>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 2: band_on<2>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 3: band_on<3>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 4: band_on<4>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 5: band_on<5>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 6: band_on<6>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 7: band_on<7>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ case 8: band_on<8>(tier, m, xh, tc0, ks0, ks1, acc); break;
+ default:
+ for (int i = 0; i < 128; ++i) acc[i] = 0.0f;
+ }
+}
+
+void band_acc_n_tier(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+ float* const* acc) {
+ const int tc0 = c0 / 16;
+ for (int i0 = 0; i0 < n; i0 += kExl3CpuMaxN) {
+ const int c = std::min(kExl3CpuMaxN, n - i0);
+ switch (m.K) {
+ case 1: band_on_n<1>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 2: band_on_n<2>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 3: band_on_n<3>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 4: band_on_n<4>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 5: band_on_n<5>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 6: band_on_n<6>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 7: band_on_n<7>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ case 8: band_on_n<8>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;
+ default:
+ for (int i = 0; i < c; ++i)
+ for (int j = 0; j < 128; ++j) acc[i0 + i][j] = 0.0f;
+ }
+ }
+}
+
+} // namespace
+
+int exl3_cpu_tier() {
+ static const int tier = detect_tier();
+ return tier;
+}
+
+bool exl3_cpu_vector() { return exl3_cpu_tier() > 0; }
+
+float exl3_cpu_prepare_block(const uint16_t* suh, const float* x, float* xh) {
+#if defined(EXL3_X86)
+ if (exl3_cpu_tier() == 2) return prepare_block_avx512(suh, x, xh);
+ if (exl3_cpu_tier() == 1) return prepare_block_avx2(suh, x, xh);
+#endif
+ for (int i = 0; i < 128; ++i) xh[i] = x[i] * f32_from_f16(suh[i]);
+ had128(xh);
+ double sum = 0.0;
+ for (int i = 0; i < 128; ++i) sum += xh[i];
+ return (float) sum;
+}
+
+float exl3_cpu_prepare(const Exl3Part& m, const float* x, float* xh) {
+ double sum = 0.0;
+ for (int b = 0; b < m.k; b += 128) sum += exl3_cpu_prepare_block(m.suh + b, x + b, xh + b);
+ return (float) sum;
+}
+
+void exl3_cpu_band_acc(const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc) {
+ band_acc_tier(exl3_cpu_tier(), m, xh, c0, ks0, ks1, acc);
+}
+
+void exl3_cpu_band_acc_n(const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+ float* const* acc) {
+ band_acc_n_tier(exl3_cpu_tier(), m, xh, n, c0, ks0, ks1, acc);
+}
+
+bool exl3_cpu_band_acc_on(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,
+ float* const* acc) {
+ if (tier < 0 || tier > 2 || ((tier_mask() >> tier) & 1) == 0 || n < 1) return false;
+ if (n == 1) band_acc_tier(tier, m, xh[0], c0, ks0, ks1, acc[0]);
+ else band_acc_n_tier(tier, m, xh, n, c0, ks0, ks1, acc);
+ return true;
+}
+
+void exl3_cpu_rows_acc_cols_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int c0, int c1,
+ float* const* acc) {
+ const int tier = exl3_cpu_tier();
+ const int t0 = c0 / 16, t1 = c1 / 16;
+ for (int i0 = 0; i0 < n; i0 += kExl3CpuMaxN) {
+ const int c = std::min(kExl3CpuMaxN, n - i0);
+ switch (m.K) {
+ case 1: rows_on_n<1>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 2: rows_on_n<2>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 3: rows_on_n<3>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 4: rows_on_n<4>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 5: rows_on_n<5>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 6: rows_on_n<6>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 7: rows_on_n<7>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ case 8: rows_on_n<8>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;
+ default:
+ for (int i = 0; i < c; ++i)
+ for (int j = c0; j < c1; ++j) acc[i0 + i][j] = 0.0f;
+ }
+ }
+}
+
+void exl3_cpu_rows_acc_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, float* const* acc) {
+ exl3_cpu_rows_acc_cols_n(m, xh, n, ks0, ks1, 0, m.n, acc);
+}
+
+size_t exl3_cpu_gemm_scratch() { return (size_t) kGemmK * 128; }
+
+void exl3_cpu_band_gemm(const Exl3Part& m, const float* const* xh, int n, int c0, int nb, float* const* acc,
+ float* wc) {
+#if defined(EXL3_X86)
+ if (exl3_cpu_tier() == 2) {
+ switch (m.K) {
+ case 1: band_gemm_avx512<1>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 2: band_gemm_avx512<2>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 3: band_gemm_avx512<3>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 4: band_gemm_avx512<4>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 5: band_gemm_avx512<5>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 6: band_gemm_avx512<6>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 7: band_gemm_avx512<7>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 8: band_gemm_avx512<8>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ default: break;
+ }
+ }
+ if (exl3_cpu_tier() == 1) {
+ switch (m.K) {
+ case 1: band_gemm_avx2<1>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 2: band_gemm_avx2<2>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 3: band_gemm_avx2<3>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 4: band_gemm_avx2<4>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 5: band_gemm_avx2<5>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 6: band_gemm_avx2<6>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 7: band_gemm_avx2<7>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ case 8: band_gemm_avx2<8>(m, xh, n, c0 / 16, nb, acc, wc); return;
+ default: break;
+ }
+ }
+#endif
+ (void) wc;
+ for (int i = 0; i < n; ++i)
+ for (int b = 0; b < nb; ++b) exl3_cpu_band_acc(m, xh[i], c0 + 128 * b, 0, m.k / 16, acc[i] + 128 * b);
+}
+
+void exl3_cpu_rows_acc(const Exl3Part& m, const float* xh, int ks0, int ks1, float* acc) {
+ exl3_cpu_rows_acc_n(m, &xh, 1, ks0, ks1, &acc);
+}
+
+void exl3_cpu_band_out(const Exl3Part& m, const float* acc, float xsum, int c0, float* y) {
+ const float ki = k_inv(), kz = k_zero() * xsum;
+#if defined(EXL3_X86)
+ if (exl3_cpu_tier() == 2) {
+ band_out_avx512(acc, ki, kz, m.svh + c0, y);
+ return;
+ }
+ if (exl3_cpu_tier() == 1) {
+ band_out_avx2(acc, ki, kz, m.svh + c0, y);
+ return;
+ }
+#endif
+ for (int i = 0; i < 128; ++i) y[i] = acc[i] * ki + kz;
+ had128(y);
+ for (int i = 0; i < 128; ++i) y[i] *= f32_from_f16(m.svh[c0 + i]);
+}
+
+void exl3_cpu_band(const Exl3Part& m, const float* xh, float xsum, int c0, float* y) {
+ float acc[128];
+ exl3_cpu_band_acc(m, xh, c0, 0, m.k / 16, acc);
+ exl3_cpu_band_out(m, acc, xsum, c0, y);
+}
+
+} // namespace strata::kernels::cpu
diff --git a/src/dsv41/vendor/exl3_legacy.cu b/src/dsv41/vendor/exl3_legacy.cu
new file mode 100644
index 000000000..64ca50fd8
--- /dev/null
+++ b/src/dsv41/vendor/exl3_legacy.cu
@@ -0,0 +1,965 @@
+// src/kernels/cuda/exl3_legacy.cu - v1.1.8's EXL3 decode kernels, kept for NVIDIA GPUs before sm_80 (V100, T4,
+// RTX 20): #180's decode kernels run the tiles on the tensor cores from sm_80 on, and their CUDA-core fallback was
+// slower here than these (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). exl3.cu's mv / moe_gate_up / moe_down
+// / win_experts / prepare / several_ok call these on such a GPU (STRATA_GLM_EXL3_DECODE=legacy|new picks one on any
+// GPU); the prompt path is exl3.cu's on every GPU. The code below is v1.1.8's as it was, the prompt path's kernels
+// left out.
+//
+// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/
+// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh); the kernels around them are shaped for this engine's decode:
+//
+// * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;
+// its 8 warps take the slice's 16-row tile rows in turn, each staging the row's 8 tiles in shared memory with
+// coalesced loads and decoding 8 weights per lane (one fragment: 2 columns x 4 rows), F32 accumulation;
+// * the input rotation runs per block on the slice it reads (F32 or q8_1 in, FP16 in shared memory), so the
+// engine's activation buffers are used as they are;
+// * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over
+// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in
+// split order (deterministic) and runs the output rotation.
+#if !defined(STRATA_USE_HIP) // (AMD runs exl3.cu's kernels; EXL3 does not run there anyway)
+#include "strata/kernels/exl3.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cstdio>
+#include <map>
+#include <mutex>
+#include <utility>
+
+namespace strata::kernels::exl3::legacy {
+namespace {
+
+constexpr int kThreads = 256, kWarps = 8;
+constexpr int kCols = 128; // outputs per block: 8 tiles, one rotation block
+constexpr int kMaxKR = 4096; // inputs per block (the rotated slice lives in shared memory)
+constexpr float kHad = 0.088388347648f; // 1 / sqrt(128)
+
+struct Q81 { // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8
+ __half2 ds;
+ int8_t qs[32];
+};
+
+std::atomic<int> g_launch_errors{0};
+void launch_check(const char* what) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));
+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+// ---------------------------------------------------------------- the codebooks
+__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {
+#if defined(STRATA_USE_HIP)
+ return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);
+#else
+ return __dp4a(x, 0x01010101u, acc);
+#endif
+}
+
+// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)
+template <int cb>
+__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {
+ if constexpr (cb == 2) {
+ const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);
+ const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);
+ const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));
+ return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));
+ } else {
+ uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;
+ uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;
+ x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;
+ x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;
+ const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),
+ __ushort_as_half((unsigned short) (x1 & 0xffffu)));
+ const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),
+ __ushort_as_half((unsigned short) (x1 >> 16)));
+ return __hadd2(lo, hi);
+ }
+}
+template <int cb>
+__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {
+ return __half22float2(decode2h<cb>(s0, s1));
+}
+
+// step p's state: the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),
+// wrapping around the tile (the + 256 K keeps the start non-negative)
+template <int K>
+__device__ __forceinline__ uint32_t state_at(const uint32_t* w, int p) {
+ constexpr int W = 8 * K;
+ const int b1 = (p + 1 + 256) * K;
+ const int i1 = (b1 - 1) >> 5, i0 = (b1 - 16) >> 5;
+ const int sh = ((i1 + 1) << 5) - b1;
+ const uint64_t m = ((uint64_t) w[i0 % W] << 32) | (uint64_t) w[i1 % W];
+ return (uint32_t) (m >> sh) & 0xffffu;
+}
+
+// lane's 8 steps (lane * 8 + j): v[j] sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column
+// (lane >> 2) + 8 (j >> 2)
+template <int K, int cb>
+__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {
+ const int p0 = lane * 8;
+#pragma unroll
+ for (int j = 0; j < 8; j += 2) {
+ const float2 d = decode2<cb>(state_at<K>(w, p0 + j), state_at<K>(w, p0 + j + 1));
+ v[j] = d.x;
+ v[j + 1] = d.y;
+ }
+}
+
+// ---------------------------------------------------------------- the rotation
+// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)
+__device__ __forceinline__ void had128(float* v, int lane) {
+ const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];
+ v[0] = s0 + s1;
+ v[1] = d0 + d1;
+ v[2] = s0 - s1;
+ v[3] = d0 - d1;
+#pragma unroll
+ for (int m = 1; m < 32; m <<= 1) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const float o = __shfl_xor_sync(0xffffffffu, v[i], m);
+ v[i] = (lane & m) ? o - v[i] : v[i] + o;
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] *= kHad;
+}
+
+__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }
+
+// xs[i] = H(x * suh)[k0 + i] for i < kr (kr % 128 == 0); x from xf (F32) or xq (q8_1); every warp of the block
+__device__ void rotate_in(const float* __restrict__ xf, const Q81* __restrict__ xq, const uint16_t* __restrict__ suh,
+ int k0, int kr, __half* xs) {
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ for (int c = warp; c < kr / 128; c += kWarps) {
+ const int base = k0 + c * 128 + lane * 4;
+ float v[4];
+ if (xf != nullptr) {
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = xf[base + i];
+ } else {
+ const Q81* b = xq + base / 32;
+ const float d = __low2float(b->ds);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];
+ }
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[base + i]);
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) xs[c * 128 + lane * 4 + i] = __float2half_rn(v[i]);
+ }
+}
+
+// ---------------------------------------------------------------- the block GEMV
+// a lane's 4 weights of one column times the activation's 4 inputs, added to a: the rounding spelled out, so the
+// one-activation and the several-activation GEMVs (block_gemv, block_gemv_n) give each activation the same bits
+__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {
+ return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));
+}
+
+// red[c] (c < 128) = sum over the 16-row slices [ks0, ks1) of x' . W'[:, tc0 * 16 + c], x' = xs from slice ks0 on.
+// tr: the matrix's first tile, ld tiles per slice row. stage: kWarps * 64 K words, red: kWarps * 128 floats.
+// Ends with a __syncthreads (red[0..127] complete).
+template <int K, int cb>
+__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,
+ uint32_t* stage, float* red) {
+ constexpr int W = 8 * K, TW = 8 * W;
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ uint32_t* st = stage + warp * TW;
+ float acc[8][2];
+#pragma unroll
+ for (int t = 0; t < 8; ++t) acc[t][0] = acc[t][1] = 0.0f;
+ const int kk = (lane & 3) * 2;
+ for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {
+ const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;
+ __syncwarp();
+#pragma unroll
+ for (int i = lane; i < TW; i += 32) st[i] = src[i];
+ __syncwarp();
+ const __half* xk = xs + (ks - ks0) * 16;
+ const float x0 = __half2float(xk[kk]), x1 = __half2float(xk[kk + 1]);
+ const float x2 = __half2float(xk[kk + 8]), x3 = __half2float(xk[kk + 9]);
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ float v[8];
+ decode8<K, cb>(st + t * W, lane, v);
+ acc[t][0] = dot4_acc(acc[t][0], v, x0, x1, x2, x3);
+ acc[t][1] = dot4_acc(acc[t][1], v + 4, x0, x1, x2, x3);
+ }
+ }
+ // the 4 lanes of one lane / 4 hold the same two columns of every tile
+#pragma unroll
+ for (int t = 0; t < 8; ++t)
+#pragma unroll
+ for (int c = 0; c < 2; ++c) {
+ acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 1);
+ acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 2);
+ }
+ if ((lane & 3) == 0)
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ red[warp * kCols + t * 16 + (lane >> 2)] = acc[t][0];
+ red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[t][1];
+ }
+ __syncthreads();
+ if (threadIdx.x < kCols) {
+ float s = 0.0f;
+#pragma unroll
+ for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];
+ red[threadIdx.x] = s;
+ }
+ __syncthreads();
+}
+
+// block_gemv for nc activations at once (nc <= kNC; activation c's rotated slice at xs + c * kMaxKR): each lane decodes
+// its 8 weights of a tile once and adds them into every activation's sums - as exllamav3's small-m GEMV and fused MoE
+// kernels share a tile's decode among the rows of an MMA, but with block_gemv's own F32 arithmetic, so outc[c * 128 +
+// i] (i < 128) is bit for bit block_gemv's red[i] for activation c alone. Ends with a __syncthreads.
+constexpr int kNC = 4;
+template <int K, int cb>
+__device__ void block_gemv_n(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,
+ int nc, uint32_t* stage, float* red, float* outc) {
+ constexpr int W = 8 * K, TW = 8 * W;
+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;
+ uint32_t* st = stage + warp * TW;
+ float acc[kNC][8][2];
+#pragma unroll
+ for (int c = 0; c < kNC; ++c)
+#pragma unroll
+ for (int t = 0; t < 8; ++t) acc[c][t][0] = acc[c][t][1] = 0.0f;
+ const int kk = (lane & 3) * 2;
+ for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {
+ const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;
+ __syncwarp();
+#pragma unroll
+ for (int i = lane; i < TW; i += 32) st[i] = src[i];
+ __syncwarp();
+ float x[kNC][4];
+#pragma unroll
+ for (int c = 0; c < kNC; ++c) {
+ const __half* xk = xs + c * kMaxKR + (ks - ks0) * 16;
+ if (c < nc) {
+ x[c][0] = __half2float(xk[kk]);
+ x[c][1] = __half2float(xk[kk + 1]);
+ x[c][2] = __half2float(xk[kk + 8]);
+ x[c][3] = __half2float(xk[kk + 9]);
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ float v[8];
+ decode8<K, cb>(st + t * W, lane, v);
+#pragma unroll
+ for (int c = 0; c < kNC; ++c)
+ if (c < nc) {
+ acc[c][t][0] = dot4_acc(acc[c][t][0], v, x[c][0], x[c][1], x[c][2], x[c][3]);
+ acc[c][t][1] = dot4_acc(acc[c][t][1], v + 4, x[c][0], x[c][1], x[c][2], x[c][3]);
+ }
+ }
+ }
+ // per activation, block_gemv's reduction
+#pragma unroll
+ for (int c = 0; c < kNC; ++c) {
+ if (c >= nc) break;
+#pragma unroll
+ for (int t = 0; t < 8; ++t)
+#pragma unroll
+ for (int h = 0; h < 2; ++h) {
+ acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 1);
+ acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 2);
+ }
+ if ((lane & 3) == 0)
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ red[warp * kCols + t * 16 + (lane >> 2)] = acc[c][t][0];
+ red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[c][t][1];
+ }
+ __syncthreads();
+ if (threadIdx.x < kCols) {
+ float s = 0.0f;
+#pragma unroll
+ for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];
+ outc[c * kCols + threadIdx.x] = s;
+ }
+ __syncthreads();
+ }
+}
+
+// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs
+__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,
+ float* y) {
+ const int lane = threadIdx.x & 31;
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) {
+ const int c = lane * 4 + i;
+ y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);
+ }
+}
+
+// ---------------------------------------------------------------- mv
+struct KJob {
+ const uint32_t* tr;
+ const uint16_t* suh;
+ const uint16_t* svh;
+ const float* xf;
+ const Q81* xq;
+ float* y;
+ const float* bias;
+ float* part; // S x n partial sums (S > 1); mv_n_kernel: nc x S x n
+ unsigned* cnt; // n / 128 counters, zero between launches
+ float alpha;
+ int k, n, ld, S, kr;
+ int nc; // mv_n_kernel: the activations (1..kNC), xf / xq / y advanced by these per activation
+ size_t xf_cs, xq_cs, y_cs; // (floats, bytes, floats)
+};
+struct KBatch {
+ KJob j[kMaxMvJobs];
+ int blk_end[kMaxMvJobs];
+ int n;
+};
+
+// the dynamic shared memory of the several-activation kernels (mv_n_kernel, win_gu_kernel, win_down_kernel):
+// red[kWarps][128] outc[kNC][128] gu[2][kNC][128] (F32), stage[kWarps * 64 K] (words), xs[kNC][kMaxKR] (FP16) - up to
+// 58 KB, past the 48 KB a launch gets without asking (set_smem)
+constexpr size_t smem_n_bytes(int K) {
+ return (size_t) (kWarps * kCols + kNC * kCols + 2 * kNC * kCols) * 4 + (size_t) kWarps * 64 * K * 4 +
+ (size_t) kNC * kMaxKR * 2;
+}
+struct SmemN {
+ float *red, *outc, *gu;
+ uint32_t* stage;
+ __half* xs;
+};
+template <int K>
+__device__ __forceinline__ SmemN smem_n(unsigned char* p) {
+ SmemN s;
+ s.red = (float*) p;
+ s.outc = s.red + kWarps * kCols;
+ s.gu = s.outc + kNC * kCols;
+ s.stage = (uint32_t*) (s.gu + 2 * kNC * kCols);
+ s.xs = (__half*) (s.stage + kWarps * 64 * K);
+ return s;
+}
+
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) mv_kernel(const KBatch b) {
+ __shared__ __half xs[kMaxKR];
+ __shared__ uint32_t stage[kWarps * 64 * K];
+ __shared__ float red[kWarps * kCols];
+ __shared__ int last;
+ int ji = 0;
+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
+ const KJob& J = b.j[ji];
+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
+ const int g = local / J.S, sp = local % J.S;
+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
+ rotate_in(J.xf, J.xq, J.suh, k0, k1 - k0, xs);
+ __syncthreads();
+ block_gemv<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, xs, stage, red);
+ const int warp = threadIdx.x >> 5;
+ if (J.S == 1) {
+ if (warp == 0)
+ epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr,
+ J.y + g * kCols);
+ return;
+ }
+ if (threadIdx.x < kCols) J.part[(size_t) sp * J.n + g * kCols + threadIdx.x] = red[threadIdx.x];
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < kCols) {
+ float s = 0.0f;
+ for (int i = 0; i < J.S; ++i) s += *(volatile const float*) &J.part[(size_t) i * J.n + g * kCols + threadIdx.x];
+ red[threadIdx.x] = s;
+ }
+ __syncthreads();
+ if (warp == 0)
+ epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr, J.y + g * kCols);
+ if (threadIdx.x == 0) J.cnt[g] = 0u;
+}
+
+// mv_kernel for a job's nc activations at once (a verify window's tokens): the same input split, the weights read and
+// decoded once for all of them (block_gemv_n), each activation's partials and sums in mv_kernel's order - every
+// activation's y is mv_kernel's for it alone, bit for bit
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) mv_n_kernel(const KBatch b) {
+ extern __shared__ __align__(16) unsigned char smem_raw[];
+ const SmemN sm = smem_n<K>(smem_raw);
+ __shared__ int last;
+ int ji = 0;
+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;
+ const KJob& J = b.j[ji];
+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);
+ const int g = local / J.S, sp = local % J.S;
+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);
+ for (int c = 0; c < J.nc; ++c)
+ rotate_in(J.xf != nullptr ? J.xf + c * J.xf_cs : nullptr,
+ J.xq != nullptr ? (const Q81*) ((const uint8_t*) J.xq + c * J.xq_cs) : nullptr, J.suh, k0, k1 - k0,
+ sm.xs + c * kMaxKR);
+ __syncthreads();
+ block_gemv_n<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, sm.xs, J.nc, sm.stage, sm.red, sm.outc);
+ const int warp = threadIdx.x >> 5;
+ const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;
+ if (J.S == 1) {
+ if (warp < J.nc)
+ epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+ return;
+ }
+ if (threadIdx.x < kCols)
+ for (int c = 0; c < J.nc; ++c)
+ J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.outc[c * kCols + threadIdx.x];
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < kCols)
+ for (int c = 0; c < J.nc; ++c) {
+ float s = 0.0f;
+ for (int i = 0; i < J.S; ++i)
+ s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];
+ sm.outc[c * kCols + threadIdx.x] = s;
+ }
+ __syncthreads();
+ if (warp < J.nc) epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);
+ if (threadIdx.x == 0) J.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- the routed experts
+struct KMoe {
+ const unsigned long long* plan_ptr;
+ const float* plan_w;
+ const float* cpu_part;
+ const int* cpu_flag;
+ const float* x;
+ float* h; // [k][n_ff]
+ const float* sh_out;
+ float* out;
+ float* part; // [k][n_embd]
+ unsigned* cnt; // n_embd / 128
+ size_t suh[3], svh[3], tr[3];
+ int k, n_embd, n_ff;
+ float limit;
+};
+
+// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window
+// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...
+__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {
+ const int lane = threadIdx.x & 31;
+ float v[4];
+#pragma unroll
+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];
+ had128(v, lane);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));
+}
+// ... the clamped swiglu ...
+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {
+ if (limit > 0.0f) {
+ gv = fminf(gv, limit);
+ uv = fminf(fmaxf(uv, -limit), limit);
+ }
+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);
+}
+// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU
+// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)
+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {
+ float m = 0.0f;
+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);
+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);
+ return __fadd_rn(m, sh);
+}
+
+// grid (n_ff / 128, k): expert blockIdx.y's gate and up for 128 h values, then the clamped swiglu
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) moe_gate_up_kernel(const KMoe a) {
+ __shared__ __half xs[2][kMaxKR];
+ __shared__ uint32_t stage[kWarps * 64 * K];
+ __shared__ float red[kWarps * kCols];
+ __shared__ float gu[2][kCols];
+ const int ei = blockIdx.y, g = blockIdx.x;
+ const unsigned long long ptr = a.plan_ptr[ei];
+ if (ptr == 0ull) return;
+ const uint8_t* blob = (const uint8_t*) ptr;
+ const int warp = threadIdx.x >> 5;
+ for (int r = 0; r < 2; ++r) rotate_in(a.x, nullptr, (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, xs[r]);
+ __syncthreads();
+ for (int r = 0; r < 2; ++r) {
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, xs[r], stage, red);
+ if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[r]) + g * kCols, gu[r]);
+ __syncthreads();
+ }
+ if (threadIdx.x < kCols)
+ a.h[(size_t) ei * a.n_ff + g * kCols + threadIdx.x] = swiglu(gu[0][threadIdx.x], gu[1][threadIdx.x], a.limit);
+}
+
+// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds
+// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) moe_down_kernel(const KMoe a) {
+ __shared__ __half xs[kMaxKR];
+ __shared__ uint32_t stage[kWarps * 64 * K];
+ __shared__ float red[kWarps * kCols];
+ __shared__ int last;
+ const int ei = blockIdx.y, g = blockIdx.x;
+ const unsigned long long ptr = a.plan_ptr[ei];
+ const int warp = threadIdx.x >> 5;
+ float* part = a.part + (size_t) ei * a.n_embd + g * kCols;
+ if (ptr != 0ull) {
+ const uint8_t* blob = (const uint8_t*) ptr;
+ rotate_in(a.h + (size_t) ei * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0, a.n_ff, xs);
+ __syncthreads();
+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, xs, stage, red);
+ if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);
+ } else if (threadIdx.x < kCols) {
+ part[threadIdx.x] = 0.0f;
+ }
+ __threadfence();
+ __syncthreads();
+ if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);
+ __syncthreads();
+ if (!last) return;
+ __threadfence();
+ if (threadIdx.x < kCols) {
+ const int r = g * kCols + threadIdx.x;
+ float p[8];
+ for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];
+ a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,
+ a.sh_out != nullptr ? a.sh_out[r] : 0.0f);
+ }
+ if (threadIdx.x == 0) a.cnt[g] = 0u;
+}
+
+// ---------------------------------------------------------------- a verify window's routed experts
+// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at
+// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kNC of its tokens at a time (as
+// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token
+// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit
+struct KWin {
+ const unsigned long long* grp_ptr;
+ const int *grp_start, *grp_ent, *n_groups;
+ const unsigned long long* plan_ptr;
+ const float* plan_w;
+ const float* cpu_part;
+ const int* cpu_flag;
+ const float* x;
+ size_t x_stride;
+ float* h; // [T * 8][n_ff]
+ float* part; // [T * 8][n_embd]
+ const float* sh_out; // [T][n_embd]
+ float* out;
+ size_t out_stride;
+ size_t suh[3], svh[3], tr[3];
+ int k, n_embd, n_ff;
+ float limit;
+};
+
+// grid (n_ff / 128, groups at most): group blockIdx.y's gate and up for 128 h values of each entry, the clamped swiglu
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) win_gu_kernel(const KWin a) {
+ extern __shared__ __align__(16) unsigned char smem_raw[];
+ const SmemN sm = smem_n<K>(smem_raw);
+ const int grp = blockIdx.y, g = blockIdx.x;
+ if (grp >= *a.n_groups) return;
+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+ const int warp = threadIdx.x >> 5;
+ const int m1 = a.grp_start[grp + 1];
+ for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {
+ const int nc = min(kNC, m1 - m0);
+ for (int r = 0; r < 2; ++r) {
+ for (int c = 0; c < nc; ++c)
+ rotate_in(a.x + (size_t) (a.grp_ent[m0 + c] >> 3) * a.x_stride, nullptr,
+ (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, sm.xs + c * kMaxKR);
+ __syncthreads();
+ block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, sm.xs, nc,
+ sm.stage, sm.red, sm.outc);
+ if (warp < nc)
+ block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,
+ sm.gu + (r * kNC + warp) * kCols);
+ __syncthreads();
+ }
+ if (threadIdx.x < kCols)
+ for (int c = 0; c < nc; ++c)
+ a.h[(size_t) a.grp_ent[m0 + c] * a.n_ff + g * kCols + threadIdx.x] =
+ swiglu(sm.gu[c * kCols + threadIdx.x], sm.gu[(kNC + c) * kCols + threadIdx.x], a.limit);
+ __syncthreads();
+ }
+}
+
+// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows
+template <int K, int cb>
+__global__ void __launch_bounds__(kThreads) win_down_kernel(const KWin a) {
+ extern __shared__ __align__(16) unsigned char smem_raw[];
+ const SmemN sm = smem_n<K>(smem_raw);
+ const int grp = blockIdx.y, g = blockIdx.x;
+ if (grp >= *a.n_groups) return;
+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];
+ const int warp = threadIdx.x >> 5;
+ const int m1 = a.grp_start[grp + 1];
+ for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {
+ const int nc = min(kNC, m1 - m0);
+ for (int c = 0; c < nc; ++c)
+ rotate_in(a.h + (size_t) a.grp_ent[m0 + c] * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0,
+ a.n_ff, sm.xs + c * kMaxKR);
+ __syncthreads();
+ block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, sm.xs, nc,
+ sm.stage, sm.red, sm.outc);
+ if (warp < nc)
+ block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,
+ a.part + (size_t) a.grp_ent[m0 + warp] * a.n_embd + g * kCols);
+ __syncthreads();
+ }
+}
+
+// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)
+__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {
+ const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;
+ float p[8];
+ for (int i = 0; i < a.k; ++i) {
+ const int e = t * 8 + i;
+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;
+ }
+ a.out[(size_t) t * a.out_stride + r] =
+ combine(a.plan_w + t * 8, p, a.k, a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.n_embd + r : nullptr,
+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);
+}
+
+// ---------------------------------------------------------------- dispatch
+// F(K, cb) for K 1..8 and the three codebooks
+template <template <int, int> class F, typename... A>
+bool dispatch(int K, int cb, A&&... args) {
+#define EXL3_CASE(KK) \
+ case KK: \
+ if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...); \
+ else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...); \
+ else F<KK, 0>::run(std::forward<A>(args)...); \
+ return true;
+ switch (K) {
+ EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)
+ default: return false;
+ }
+#undef EXL3_CASE
+}
+
+template <int K, int cb> struct RunMv {
+ static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }
+};
+template <int K, int cb> struct RunMvN {
+ static void run(const KBatch& b, int blocks, cudaStream_t s) {
+ mv_n_kernel<K, cb><<<blocks, kThreads, smem_n_bytes(K), s>>>(b);
+ }
+};
+template <int K, int cb> struct RunWin {
+ static void run(const KWin& a, int groups, cudaStream_t s) {
+ win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups), kThreads, smem_n_bytes(K), s>>>(a);
+ }
+};
+template <int K, int cb> struct RunWinDown {
+ static void run(const KWin& a, int groups, cudaStream_t s) {
+ win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, smem_n_bytes(K),
+ s>>>(a);
+ }
+};
+template <int K, int cb> struct AllowSmemN {
+ static void run() {
+ const int b = (int) smem_n_bytes(K);
+ cudaFuncSetAttribute(mv_n_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+ cudaFuncSetAttribute(win_gu_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+ cudaFuncSetAttribute(win_down_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);
+ }
+};
+template <int K, int cb> struct RunGU {
+ static void run(const KMoe& a, cudaStream_t s) {
+ moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);
+ }
+};
+template <int K, int cb> struct RunDown {
+ static void run(const KMoe& a, cudaStream_t s) {
+ moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);
+ }
+};
+// ---------------------------------------------------------------- per-stream scratch
+constexpr size_t kPartFloats = (size_t) 2 << 20; // split partials: 8 MB
+constexpr size_t kCounters = 16384;
+constexpr size_t kHFloats = (size_t) 8 * 8192; // the experts' h
+struct Scratch {
+ float* part = nullptr;
+ unsigned* cnt = nullptr;
+ float* h = nullptr;
+};
+std::mutex g_mu;
+std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;
+
+Scratch* scratch(cudaStream_t s) {
+ int dev = 0;
+ cudaGetDevice(&dev);
+ std::lock_guard<std::mutex> lk(g_mu);
+ auto it = g_scratch.find({dev, s});
+ if (it != g_scratch.end()) return &it->second;
+ cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;
+ if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {
+ std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");
+ return nullptr;
+ }
+ Scratch sc;
+ if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||
+ cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||
+ cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||
+ cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {
+ std::fprintf(stderr, "exl3: scratch did not allocate\n");
+ return nullptr;
+ }
+ cudaStreamSynchronize(s);
+ return &(g_scratch[{dev, s}] = sc);
+}
+
+int sm_count() {
+ int dev = 0, sms = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
+ return std::max(1, sms);
+}
+
+// the several-activation kernels may use smem_n_bytes of shared memory: asked once per device (prepare, before a graph
+// capture; else at the first such launch). A device that cannot give a block that much (48 KB GPUs) runs one
+// activation at a time instead
+constexpr int kMaxDev = 64;
+std::atomic<int> g_smem_n[kMaxDev]; // 0 not asked, 1 allowed, 2 refused
+bool allow_smem_n() {
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= kMaxDev) return false;
+ const int st = g_smem_n[dev].load(std::memory_order_acquire);
+ if (st != 0) return st == 1;
+ std::lock_guard<std::mutex> lk(g_mu);
+ if (g_smem_n[dev].load(std::memory_order_relaxed) != 0) return g_smem_n[dev].load() == 1;
+ int optin = 0;
+ cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
+ bool ok = (size_t) optin >= smem_n_bytes(8);
+ if (ok) {
+ for (int K = 1; K <= 8; ++K)
+ for (int cb = 0; cb <= 2; ++cb) dispatch<AllowSmemN>(K, cb);
+ ok = cudaGetLastError() == cudaSuccess;
+ }
+ if (!ok)
+ std::fprintf(stderr, "exl3: CUDA%d gives a block %d bytes of shared memory, the several-activation kernels "
+ "need %zu - a verify window's tokens run one at a time there\n", dev, optin,
+ smem_n_bytes(8));
+ g_smem_n[dev].store(ok ? 1 : 2, std::memory_order_release);
+ return ok;
+}
+
+// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block and about two blocks
+// per SM (the same for every activation count: mv_kernel and mv_n_kernel sum each activation alike)
+void mv_split(const Mat& m, int sms, int& S, int& kr) {
+ const int groups = m.n / kCols;
+ S = (m.k + kMaxKR - 1) / kMaxKR;
+ while (groups * S < 2 * sms && m.k / (S * 2) >= 256) S *= 2;
+ kr = ((m.k + S - 1) / S + 127) / 128 * 128;
+ S = (m.k + kr - 1) / kr;
+}
+
+} // namespace
+
+int launch_errors() { return g_launch_errors.load(std::memory_order_relaxed); }
+
+bool prepare(cudaStream_t s) {
+ if (scratch(s) == nullptr) return false;
+ (void) allow_smem_n();
+ return true;
+}
+
+bool several_ok() { return allow_smem_n(); }
+
+bool mv(const MvJob* jobs, int n, cudaStream_t s) {
+ if (n <= 0 || n > kMaxMvJobs) return false;
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ const int sms = sm_count();
+ // the launch units: a job of one activation is mv_kernel's, a job of several mv_n_kernel's in runs of up to kNC
+ // (one mv_kernel unit an activation where the device lacks the shared memory)
+ struct Unit {
+ int j, c0, nc;
+ };
+ Unit units[kMaxMvJobs * kMaxMvCols];
+ int nu = 0;
+ bool several = false;
+ for (int j = 0; j < n; ++j) several = several || jobs[j].ncol > 1;
+ const int run = several && allow_smem_n() ? kNC : 1;
+ for (int j = 0; j < n; ++j) {
+ const Mat& m = *jobs[j].m;
+ if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||
+ jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {
+ std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,
+ m.K, m.cb, jobs[j].ncol);
+ return false;
+ }
+ for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};
+ }
+ bool done[sizeof(units) / sizeof(units[0])] = {};
+ for (int i = 0; i < nu; ++i) {
+ if (done[i]) continue;
+ const Mat& m0 = *jobs[units[i].j].m;
+ const bool multi = units[i].nc > 1;
+ // a launch: the units of one (K, codebook, kernel), as many as KBatch and the split scratch take (the next
+ // launch on the stream may use the scratch again: this one's last blocks are done with it by then)
+ KBatch b{};
+ int blocks = 0, nb = 0;
+ size_t part_at = 0, cnt_at = 0;
+ for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {
+ const MvJob& job = jobs[units[u].j];
+ const Mat& m = *job.m;
+ if (done[u] || m.K != m0.K || m.cb != m0.cb || (units[u].nc > 1) != multi) continue;
+ const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;
+ int S = 1, kr = 0;
+ mv_split(m, sms, S, kr);
+ const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;
+ if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {
+ if (nb > 0) continue; // the next launch
+ std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);
+ return false;
+ }
+ done[u] = true;
+ KJob& J = b.j[nb];
+ J.tr = m.trellis;
+ J.suh = m.suh;
+ J.svh = m.svh;
+ J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;
+ J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;
+ J.y = job.y + c0 * job.y_cs;
+ J.bias = job.bias;
+ J.alpha = job.alpha;
+ J.k = m.k;
+ J.n = m.n;
+ J.ld = m.ld > 0 ? m.ld : m.n / 16;
+ J.S = S;
+ J.kr = kr;
+ J.part = sc->part + part_at;
+ J.cnt = sc->cnt + cnt_at;
+ J.nc = nc;
+ J.xf_cs = job.xf_cs;
+ J.xq_cs = job.xq_cs;
+ J.y_cs = job.y_cs;
+ part_at += need;
+ if (S > 1) cnt_at += (size_t) groups;
+ blocks += groups * S;
+ b.blk_end[nb++] = blocks;
+ }
+ b.n = nb;
+ if (!(multi ? dispatch<RunMvN>(m0.K, m0.cb, b, blocks, s) : dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)))
+ return false;
+ }
+ launch_check("mv");
+ return true;
+}
+
+static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {
+ Scratch* sc = scratch(s);
+ if (sc == nullptr) return false;
+ if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||
+ a.n_ff > kMaxKR || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||
+ (size_t) (a.n_embd / kCols) > kCounters) {
+ std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);
+ return false;
+ }
+ m = KMoe{};
+ m.plan_ptr = a.plan_ptr;
+ m.plan_w = a.plan_w;
+ m.cpu_part = a.cpu_part;
+ m.cpu_flag = a.cpu_flag;
+ m.h = sc->h;
+ m.part = sc->part;
+ m.cnt = sc->cnt;
+ for (int r = 0; r < 3; ++r) {
+ m.suh[r] = a.lay.suh[r];
+ m.svh[r] = a.lay.svh[r];
+ m.tr[r] = a.lay.trellis[r];
+ }
+ m.k = a.k;
+ m.n_embd = a.n_embd;
+ m.n_ff = a.n_ff;
+ m.limit = a.limit;
+ return true;
+}
+
+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {
+ KMoe m;
+ if (!moe_args(a, m, s)) return;
+ if (a.lay.K[0] != a.lay.K[1]) {
+ std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);
+ return;
+ }
+ m.x = x;
+ if (h != nullptr) m.h = h;
+ if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);
+ launch_check("moe_gate_up");
+}
+
+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {
+ KMoe m;
+ if (!moe_args(a, m, s)) return;
+ if (h != nullptr) m.h = const_cast<float*>(h);
+ m.sh_out = sh_out;
+ m.out = out;
+ if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);
+ launch_check("moe_down");
+}
+
+bool win_experts(const WinArgs& a, cudaStream_t s) {
+ if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||
+ a.n_ff > kMaxKR || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {
+ std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,
+ a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);
+ return false;
+ }
+ if (!allow_smem_n()) return false;
+ KWin w{};
+ w.grp_ptr = a.grp_ptr;
+ w.grp_start = a.grp_start;
+ w.grp_ent = a.grp_ent;
+ w.n_groups = a.n_groups;
+ w.plan_ptr = a.plan_ptr;
+ w.plan_w = a.plan_w;
+ w.cpu_part = a.cpu_part;
+ w.cpu_flag = a.cpu_flag;
+ w.x = a.x;
+ w.x_stride = a.x_stride;
+ w.h = a.h;
+ w.part = a.part;
+ w.sh_out = a.sh_out;
+ w.out = a.out;
+ w.out_stride = a.out_stride;
+ for (int r = 0; r < 3; ++r) {
+ w.suh[r] = a.lay.suh[r];
+ w.svh[r] = a.lay.svh[r];
+ w.tr[r] = a.lay.trellis[r];
+ }
+ w.k = a.k;
+ w.n_embd = a.n_embd;
+ w.n_ff = a.n_ff;
+ w.limit = a.limit;
+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)
+ if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||
+ !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))
+ return false;
+ win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);
+ launch_check("win_experts");
+ return true;
+}
+
+} // namespace strata::kernels::exl3::legacy
+#endif
diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cu
new file mode 100644
index 000000000..7779231cf
--- /dev/null
+++ b/src/dsv41/vendor/sampler.cu
@@ -0,0 +1,579 @@
+// src/kernels/cuda/sampler.cu - P2.S2: the sampler chain, in llama.cpp's order.
+//
+// penalties -> top_k -> top_p -> min_p -> temperature -> pick
+//
+// THE ORDER IS THE WHOLE CONTENT OF THIS FILE. llama.cpp builds its chain by walking `params.samplers`, whose
+// default is { PENALTIES, DRY, TOP_N_SIGMA, TOP_K, TYPICAL_P, TOP_P, MIN_P, XTC, TEMPERATURE } (`common/common.h`
+// at 3cf03257) - ONE penalties stage, first, and TEMPERATURE AFTER THE TRUNCATION FILTERS. (Issue #53: this file
+// used to apply the penalties a second time after the temperature, and min_p before top_p - both taken from the
+// order of the `case` labels in `common/sampling.cpp`, which is not the order the chain runs.) Every order
+// produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test
+// does that against an independently computed distribution.
+//
+// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,
+// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /
+// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).
+#include "strata/kernels/sampler.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+namespace strata::kernels {
+namespace {
+
+// Philox 4x32-10, the counter-based generator the phase asks for. Counter-based matters because it makes the
+// stream a function of (seed, position) rather than of how many draws came before - so a batch can be sampled
+// in any order and a run is reproducible.
+__device__ __forceinline__ uint32_t philox4x32_round(uint32_t& c0, uint32_t& c1, uint32_t& c2, uint32_t& c3,
+ uint32_t k0, uint32_t k1) {
+ const uint32_t hi0 = __umulhi(0x9E3779B9u, c0);
+ const uint32_t hi1 = __umulhi(0xBB67AE85u, c2);
+ const uint32_t lo0 = 0x9E3779B9u * c0;
+ const uint32_t lo1 = 0xBB67AE85u * c2;
+ const uint32_t n0 = hi1 ^ c1 ^ k0;
+ const uint32_t n1 = lo1;
+ const uint32_t n2 = hi0 ^ c3 ^ k1;
+ const uint32_t n3 = lo0;
+ c0 = n0; c1 = n1; c2 = n2; c3 = n3;
+ return 0;
+}
+
+__device__ __forceinline__ float philox_uniform(uint64_t seed, uint64_t counter) {
+ uint32_t c0 = (uint32_t) counter, c1 = (uint32_t) (counter >> 32);
+ uint32_t c2 = (uint32_t) seed, c3 = (uint32_t) (seed >> 32);
+ for (int i = 0; i < 10; ++i) {
+ philox4x32_round(c0, c1, c2, c3, (uint32_t) i, 0u);
+ }
+ // 24 bits of mantissa, so the value is uniform in [0,1) with no rounding to 1.0
+ return (float) (c0 >> 8) * (1.0f / 16777216.0f);
+}
+
+// `count_in_history` and the penalty application, transcribed from `llama_sampler_penalties_apply`.
+// The repeat penalty MULTIPLIES for non-positive logits and DIVIDES for positive ones - dividing
+// unconditionally is the natural reading of the source paper and it INVERTS the penalty on half the
+// vocabulary. The presence penalty is `float(count > 0)`, a boolean, not the count.
+__device__ __forceinline__ int history_count(const int* __restrict__ h, int n, int v) {
+ int c = 0;
+ for (int i = 0; i < n; ++i) if (h[i] == v) ++c;
+ return c;
+}
+
+__device__ __forceinline__ float apply_penalties(float logit, int count, const SamplerParams& p) {
+ if (count <= 0) return logit;
+ if (logit <= 0.0f) logit *= p.penalty_repeat;
+ else logit /= p.penalty_repeat;
+ logit -= (float) count * p.penalty_freq + (count > 0 ? 1.0f : 0.0f) * p.penalty_present;
+ return logit;
+}
+
+/// **THE GREEDY ARGMAX, ONE BLOCK PER TOKEN, COVERING THE VOCABULARY.**
+///
+/// **WHY THIS IS A SEPARATE KERNEL AND NOT A BRANCH.** `sampler_kernel` is launched as a grid over TOKENS
+/// with 64 threads and a `if (t >= n_tokens) return;` at the top. The decode path has `n_tokens == 1`, so
+/// that launch was `<<<1, 64>>>`, 63 threads exited on the first line, and ONE THREAD walked all 248,320
+/// logits in a dependent loop on one SM of 48. Measured in isolation (`bench/micro/sampler_cost.cu`):
+/// **3.11 ms per token**, 5.7% of an ~54 ms token, and the whole of round 309's `sample` phase - the two
+/// synchronisations around it are 0.03 ms each.
+///
+/// The obvious repair is to parallelise the scan inside `sampler_kernel`, and it is WRONG: with one thread
+/// per token, a block reduction over the vocabulary has nothing to reduce, and the threads that returned
+/// early are not there for `__syncthreads` or `__shfl_down_sync`. The first attempt did exactly that and
+/// produced the token `5120` thirty-two times. The grid has to be over tokens with the BLOCK over the
+/// vocabulary, which is a different launch configuration and therefore a different kernel.
+///
+/// **THE TIE RULE IS UNCHANGED AND THAT IS THE WHOLE CORRECTNESS ARGUMENT.** The serial scan walked `v`
+/// ascending with `if (s > bv)`, so the LOWEST index wins a tie. Each thread keeps that rule over its own
+/// strided subset and the reduction resolves two candidates by taking the larger value and, on equality, the
+/// SMALLER index - the same total order, so `sampler_parity` and C1 see no change.
+__global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vocab,
+ const int* __restrict__ history, int history_len, const SamplerParams p,
+ int pmin, int plen, int* __restrict__ out) {
+ const int t = blockIdx.x;
+ const float* l = logits + (size_t) t * n_vocab;
+ (void) pmin;
+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+ int hlen = 0;
+ if (hrow) {
+ hlen = plen < history_len ? plen : history_len;
+ if (hlen < 0) hlen = 0;
+ hrow += history_len - hlen; // the window is the TAIL
+ }
+
+ // PENALTY MEMBERSHIP AS A BITMAP. The history touches at most `hlen` tokens of a quarter-million
+ // vocabulary, but the naive `history_count` per candidate per argmax round costs O(k x n_vocab x hlen)
+ // integer compares (~318 M per token at k=20, hlen=64 - measured 45 -> 31 tok/s on a real workload).
+ // A shared bitmap gives an O(1) membership test, and only the (at most hlen) hits pay the count scan;
+ // the counts - and therefore every sampled value - are exactly what the per-candidate scan produced.
+ extern __shared__ unsigned int penal_bits[];
+ const int bits_words = (int) ((n_vocab + 31) / 32);
+ // The gate needs a NON-EMPTY WINDOW (`hlen > 0`): the launch sizes the shared bitmap only when penalties
+ // are on, so a caller handing over a history buffer with `penalty_last_n == 0` must not touch it.
+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+ if (use_bits) {
+ for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;
+ __syncthreads();
+ for (int i = threadIdx.x; i < hlen; i += blockDim.x)
+ if (hrow[i] >= 0 && hrow[i] < n_vocab) // an id outside the vocabulary is never a candidate
+ atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));
+ __syncthreads();
+ }
+ auto hit_count = [&](int v) -> int {
+ if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;
+ return history_count(hrow, hlen, v);
+ };
+
+ // `n_vocab` is the "no candidate" index: it loses every comparison to a real one, so a thread with no
+ // elements contributes nothing rather than contributing a bogus zero.
+ float bv = __int_as_float(0xff800000); // -inf
+ int best = n_vocab;
+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+ const float s = apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);
+ if (s > bv) { bv = s; best = v; }
+ }
+ for (int off = 16; off > 0; off >>= 1) {
+ const float ov = __shfl_down_sync(0xFFFFFFFFu, bv, off);
+ const int oi = __shfl_down_sync(0xFFFFFFFFu, best, off);
+ if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }
+ }
+ __shared__ float sv[32];
+ __shared__ int si[32];
+ const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);
+ if (lane == 0) { sv[warp] = bv; si[warp] = best; }
+ __syncthreads();
+ if (warp == 0) {
+ const int nw = (int) ((blockDim.x + 31) >> 5);
+ float wv = lane < nw ? sv[lane] : __int_as_float(0xff800000);
+ int wi = lane < nw ? si[lane] : n_vocab;
+ for (int off = 16; off > 0; off >>= 1) {
+ const float ov = __shfl_down_sync(0xFFFFFFFFu, wv, off);
+ const int oi = __shfl_down_sync(0xFFFFFFFFu, wi, off);
+ if (ov > wv || (ov == wv && oi < wi)) { wv = ov; wi = oi; }
+ }
+ // A tie between two `-inf` candidates leaves `wi == n_vocab`, and the serial version answered 0.
+ if (lane == 0) out[t] = (wi < n_vocab) ? wi : 0;
+ }
+}
+
+/// **THE SAMPLED PATH, ONE BLOCK PER TOKEN.** The kernel below replaced a version that ran the whole chain
+/// in ONE THREAD per token (`<<<ceil(T/64), 64>>>`, so a 4-token window fielded four threads): `top_k` alone
+/// was `k` sequential scans of the vocabulary with an inner sweep over the already-taken list - 20 x 248,320
+/// iterations of dependent work on one SM - and a verify window measured **1.6 s in the sampler**, which made
+/// every temperature-bearing request ~30x slower than a greedy one. The selection is `k` argmax rounds, and
+/// an argmax over the vocabulary parallelises exactly like `sampler_greedy_kernel` (block over the vocab), so
+/// the rounds run back to back inside a block-per-token launch: the per-token cost falls to
+/// `k x n_vocab / 1024` plus `k` block reductions.
+///
+/// THE SEMANTICS ARE THE SERIAL ONES, EXACTLY. Each round's argmax resolves ties to the LOWEST index (the
+/// serial scan's strict `>` keeps the first maximum it meets), so the kept sequence - both its set and its
+/// order - is unchanged; `top_p`'s cut reads that order in double arithmetic as before; temperature and the
+/// Philox draw apply after the cut. `sampler_parity` pins all of it against the host reference.
+// MODE 0 = the draw (sample_tokens, unchanged); 1 = speculative sampling's draft (sample_draft: the draw from the
+// draft block's logits, on a Philox stream of its own, and its kept list in ql[t]); 2 = its verify (sample_verify:
+// the draft draft[t] with probability min(1, p/q), else a draw from max(0, p - q)). The selection is shared.
+constexpr uint64_t kSpecDraftKey = 0x5bd1e9955bd1e995ull, kSpecAcceptKey = 0x9e3779b97f4a7c15ull,
+ kSpecResidKey = 0xc2b2ae3d27d4eb4full;
+// one block of 1024 threads per row (the launches below): the bound keeps every mode at 64 registers a thread, so a
+// block fits the 64K-register file - unbounded, sm_75 (Turing) compiled the draft mode to 68 and its launch failed
+// ("too many resources requested for launch") wherever the NextN block sat on a Turing card
+template <int MODE>
+__global__ void __launch_bounds__(1024) sampler_kernel(const float* __restrict__ logits, int n_vocab, int n_tokens,
+ const int* __restrict__ history, int history_len, const SamplerParams p,
+ int* __restrict__ out, SpecList* __restrict__ ql, const int* __restrict__ draft) {
+ const int t = blockIdx.x;
+ if (t >= n_tokens) return;
+ const float* l = logits + (size_t) t * n_vocab;
+
+ // Temperature is needed by BOTH stages below, so it is computed here; the chain still APPLIES it after
+ // the truncation filters - the survivors are chosen on the raw logits and only then scaled.
+ const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;
+
+ // The penalty window is the last `penalty_last_n` entries of this row's history (disabled at this
+ // launch: `sample_tokens` refuses a non-zero `penalty_last_n` without a history buffer).
+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;
+ int hlen = 0;
+ if (hrow) {
+ hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;
+ if (hlen < 0) hlen = 0;
+ hrow += history_len - hlen; // the window is the TAIL
+ }
+
+ // the membership bitmap, as in `sampler_greedy_kernel` - see the cost note there. The gate needs an
+ // NON-EMPTY WINDOW too: the launch sizes the bitmap only when penalties are on, so a caller that hands over
+ // a stale history buffer with `penalty_last_n == 0` must not touch it.
+ extern __shared__ unsigned int penal_bits[];
+ const int bits_words = (int) ((n_vocab + 31) / 32);
+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+ if (use_bits) {
+ for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;
+ __syncthreads();
+ for (int i = threadIdx.x; i < hlen; i += blockDim.x)
+ if (hrow[i] >= 0 && hrow[i] < n_vocab) // an id outside the vocabulary is never a candidate
+ atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));
+ __syncthreads();
+ }
+ auto hit_count = [&](int v) -> int {
+ if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;
+ return history_count(hrow, hlen, v);
+ };
+
+ // top_k in 1..64 is taken as given; 0 ("off") and anything wider mean the widest shortlist the kernel
+ // keeps, 64. Every row writes out[t]: a verify window reads all of them.
+ const int KMAX = 64;
+ int k = (p.top_k > 0 && p.top_k < KMAX) ? p.top_k : KMAX;
+ if (k > n_vocab) k = n_vocab;
+
+ // ---- top_k: an exact RADIX SELECT, then a rank sort of the k survivors. The order is the serial one:
+ // descending by (penalised) logit, ties to the lower index - i.e. descending by the composite key
+ // (ordered(score) << 32) | ~index, which is distinct for every token. So: find the k-th largest ordered score
+ // T byte by byte (4 histogram passes over the vocabulary), keep every token above T and the lowest-indexed
+ // ones equal to T, then rank the k of them by the composite key. The k block-argmax rounds this replaces
+ // re-scanned the taken list for every token in every round: 1.5 ms a sample at top_k 20 and 11 ms at 64 on an
+ // RTX 4090 (the tail card's critical path in every sampled decode step).
+ // THE COMMON CASE takes two passes: a coarse histogram of the top 11 bits finds the bin that holds the k-th
+ // largest; every token in that bin or above (at most kCap) is copied to shared memory and ranked there by the
+ // composite key. A flat distribution with more than kCap tokens there takes the full radix select below.
+ __shared__ int sel_ids[KMAX];
+ __shared__ float sel_logit[KMAX];
+ __shared__ unsigned int sh_prefix, sh_remaining, sh_ncand, sh_tie_base;
+ // ONE shared pool, its parts reused phase by phase: the penalty bitmap (dynamic shared memory, 31 KB at a Qwen
+ // vocabulary) must still fit beside it under the 48 KB a launch gets without opting in.
+ // [0, 2048) the coarse histogram, then (once its bin is known) the candidates' scores and ids;
+ // in the fallback, the byte histogram and the k candidates
+ // [2048, 3072) the block scan
+ constexpr int kBins = 2048, kCap = 1024; // the coarse histogram: the ordered score's top 11 bits
+ __shared__ unsigned int pool[kBins + 1024];
+ unsigned int* coarse = pool;
+ unsigned int* ccand_u = pool;
+ int* ccand_i = (int*) pool + kCap;
+ unsigned int* hist = pool;
+ unsigned int* cand_u = pool + 256;
+ int* cand_i = (int*) pool + 256 + KMAX;
+ unsigned int* scan = pool + kBins;
+ __shared__ int sh_bin;
+ __shared__ unsigned int sh_ncoarse;
+ // a score's ordered bits: larger float -> larger unsigned; NaN below everything (never kept before a number)
+ auto ordered = [](float x) -> unsigned int {
+ unsigned int b = __float_as_uint(x);
+ if ((b & 0x7fffffffu) > 0x7f800000u) return 0u;
+ if (b == 0x80000000u) b = 0u; // -0 ties with +0, as the float comparison it replaces
+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
+ };
+ auto score = [&](int v) -> float {
+ return apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);
+ };
+ for (int b = threadIdx.x; b < kBins; b += blockDim.x) coarse[b] = 0u;
+ if (threadIdx.x == 0) sh_ncoarse = 0u;
+ __syncthreads();
+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) atomicAdd(&coarse[ordered(score(v)) >> 21], 1u);
+ __syncthreads();
+ { // the bin holding the k-th largest: suffix counts over the bins (2 bins a thread, a block scan)
+ const int nt = (int) blockDim.x, per = (kBins + nt - 1) / nt;
+ const int b_hi = kBins - 1 - (int) threadIdx.x * per; // this thread's bins, from the top down
+ unsigned int mine = 0u;
+ for (int j = 0; j < per && b_hi - j >= 0; ++j) mine += coarse[b_hi - j];
+ scan[threadIdx.x] = mine;
+ __syncthreads();
+ for (int off = 1; off < nt; off <<= 1) {
+ const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;
+ __syncthreads();
+ scan[threadIdx.x] += add;
+ __syncthreads();
+ }
+ unsigned int above = scan[threadIdx.x] - mine; // tokens in the bins above this thread's
+ if (above < (unsigned int) k && above + mine >= (unsigned int) k) {
+ for (int j = 0; j < per && b_hi - j >= 0; ++j) {
+ above += coarse[b_hi - j];
+ if (above >= (unsigned int) k) { sh_bin = b_hi - j; sh_tie_base = above; break; }
+ }
+ }
+ __syncthreads();
+ }
+ const int bin = sh_bin;
+ const bool coarse_ok = sh_tie_base <= (unsigned int) kCap; // the tokens in that bin and above fit
+ if (coarse_ok) {
+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+ const unsigned int u = ordered(score(v));
+ if ((int) (u >> 21) >= bin) {
+ const unsigned int at = atomicAdd(&sh_ncoarse, 1u);
+ if (at < (unsigned int) kCap) { ccand_u[at] = u; ccand_i[at] = v; }
+ }
+ }
+ __syncthreads();
+ const int nc = (int) min(sh_ncoarse, (unsigned int) kCap);
+ for (int i = threadIdx.x; i < nc; i += blockDim.x) {
+ const unsigned int ui = ccand_u[i];
+ const int ii = ccand_i[i];
+ int r = 0;
+ for (int j = 0; j < nc && r < k; ++j) {
+ const unsigned int uj = ccand_u[j];
+ r += uj > ui || (uj == ui && ccand_i[j] < ii);
+ }
+ if (r < k) {
+ sel_ids[r] = ii;
+ sel_logit[r] = score(ii);
+ }
+ }
+ __syncthreads();
+ } else {
+ if (threadIdx.x == 0) {
+ sh_prefix = 0u;
+ sh_remaining = (unsigned int) k;
+ sh_ncand = 0u;
+ }
+ unsigned int mask = 0u;
+ for (int pass = 0; pass < 4; ++pass) {
+ const int shift = 24 - 8 * pass;
+ for (int b = threadIdx.x; b < 256; b += blockDim.x) hist[b] = 0u;
+ __syncthreads();
+ const unsigned int prefix = sh_prefix;
+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+ const unsigned int u = ordered(score(v));
+ if ((u & mask) == prefix) atomicAdd(&hist[(u >> shift) & 255u], 1u);
+ }
+ __syncthreads();
+ if (threadIdx.x == 0) { // the digit that holds the remaining-th largest, from the top
+ unsigned int cum = 0u, rem = sh_remaining;
+ int d = 255;
+ for (; d > 0; --d) {
+ if (cum + hist[d] >= rem) break;
+ cum += hist[d];
+ }
+ sh_remaining = rem - cum;
+ sh_prefix = prefix | ((unsigned int) d << shift);
+ }
+ mask |= 255u << shift;
+ __syncthreads();
+ }
+ const unsigned int T = sh_prefix; // the k-th largest ordered score
+ const unsigned int need = sh_remaining; // how many tokens equal to T are kept (the lowest-indexed ones)
+ // every token above T (fewer than k of them)
+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {
+ const unsigned int u = ordered(score(v));
+ if (u > T) {
+ const unsigned int at = atomicAdd(&sh_ncand, 1u);
+ if (at < (unsigned int) KMAX) { cand_u[at] = u; cand_i[at] = v; }
+ }
+ }
+ __syncthreads();
+ if (threadIdx.x == 0) sh_tie_base = sh_ncand;
+ // the ties in index order: each thread counts its own contiguous range, an exclusive scan orders the ranges
+ {
+ const int nt = (int) blockDim.x;
+ const int chunk = (n_vocab + nt - 1) / nt;
+ const int lo = (int) threadIdx.x * chunk, hi = min(n_vocab, lo + chunk);
+ unsigned int mine = 0u;
+ for (int v = lo; v < hi; ++v) mine += ordered(score(v)) == T;
+ scan[threadIdx.x] = mine;
+ __syncthreads();
+ for (int off = 1; off < nt; off <<= 1) { // Hillis-Steele inclusive scan
+ const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;
+ __syncthreads();
+ scan[threadIdx.x] += add;
+ __syncthreads();
+ }
+ unsigned int rank = scan[threadIdx.x] - mine; // ties before this range
+ for (int v = lo; v < hi && rank < need; ++v)
+ if (ordered(score(v)) == T) {
+ const unsigned int at = sh_tie_base + rank;
+ if (at < (unsigned int) KMAX) { cand_u[at] = T; cand_i[at] = v; }
+ ++rank;
+ }
+ }
+ __syncthreads();
+ // rank sort: candidate i's place is the number of candidates with a larger composite key
+ if ((int) threadIdx.x < k) {
+ const unsigned int ui = cand_u[threadIdx.x];
+ const int ii = cand_i[threadIdx.x];
+ int r = 0;
+ for (int j = 0; j < k; ++j) {
+ const unsigned int uj = cand_u[j];
+ r += uj > ui || (uj == ui && cand_i[j] < ii);
+ }
+ sel_ids[r] = ii;
+ sel_logit[r] = score(ii);
+ }
+ __syncthreads();
+ }
+
+ // ---- top_p over the top_k list (penalised logits, descending as the selection produced them), then min_p,
+ // then temperature and one Philox draw - llama.cpp's order (issue #53). Every thread computes the same chain
+ // redundantly over `sel_*` - the arithmetic is the serial kernel's, instruction for instruction - so they
+ // agree on `pick` and thread 0 writes it.
+ // (one thread: the chain is serial and in double precision, which consumer cards run at 1/64 rate - 1024
+ // threads computing it redundantly cost more than the whole selection above)
+ if (threadIdx.x != 0) return;
+ int n_keep = k;
+ float mx = sel_logit[0];
+ for (int i = 1; i < k; ++i) mx = fmaxf(mx, sel_logit[i]);
+ if (p.top_p < 1.0f) {
+ double sum = 0.0;
+ for (int i = 0; i < k; ++i) sum += exp((double) sel_logit[i] - (double) mx);
+ double cum = 0.0;
+ int cut = k;
+ for (int i = 0; i < k; ++i) {
+ cum += exp((double) sel_logit[i] - (double) mx) / sum;
+ if (cum >= (double) p.top_p) { cut = i + 1; break; }
+ }
+ if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;
+ n_keep = cut;
+ }
+ // ---- min_p on top_p's survivors: the descending prefix whose probability is at least `min_p` of the top
+ // token's. In logit space the threshold is `sel_logit[0] + logf(min_p)` - equivalent to `p >= min_p * p_max`
+ // without the overflow an exp of raw logits risks. 0 disables, and the head itself always survives
+ // (`expf(0) == 1 >= min_p` for min_p in 0..1), so the count never reaches zero.
+ if (p.min_p > 0.0f) {
+ const float thresh = sel_logit[0] + logf(p.min_p);
+ for (int i = 0; i < n_keep; ++i)
+ if (sel_logit[i] < thresh) { n_keep = i; break; }
+ }
+ // temperature only: the penalties were applied once, before the selection (issue #53: they were applied a
+ // second time here, after the temperature scaling - llama.cpp's chain has one penalties stage)
+ auto scaled = [&](int i) { return sel_logit[i] * inv_t; };
+ float smx = scaled(0);
+ for (int i = 1; i < n_keep; ++i) smx = fmaxf(smx, scaled(i));
+ double sum = 0.0;
+ for (int i = 0; i < n_keep; ++i) sum += exp((double) scaled(i) - (double) smx);
+ if constexpr (MODE == 0) {
+ const float u = philox_uniform(p.seed, p.counter + (uint64_t) t);
+ double cum = 0.0;
+ int pick = sel_ids[n_keep - 1];
+ for (int i = 0; i < n_keep; ++i) {
+ cum += exp((double) scaled(i) - (double) smx) / sum;
+ if ((double) u < cum) { pick = sel_ids[i]; break; }
+ }
+ if (threadIdx.x == 0) out[t] = pick;
+ } else {
+ double w[KMAX]; // the kept list's probabilities, as the plain draw computes them
+ for (int i = 0; i < n_keep; ++i) w[i] = exp((double) scaled(i) - (double) smx) / sum;
+ // the plain draw over w with a given uniform (MODE 0's loop)
+ auto draw = [&](float u) -> int {
+ double cum = 0.0;
+ int pick = sel_ids[n_keep - 1];
+ for (int i = 0; i < n_keep; ++i) {
+ cum += w[i];
+ if ((double) u < cum) { pick = sel_ids[i]; break; }
+ }
+ return pick;
+ };
+ if constexpr (MODE == 1) {
+ SpecList* q = ql + t;
+ q->n = n_keep;
+ for (int i = 0; i < n_keep; ++i) { q->ids[i] = sel_ids[i]; q->p[i] = w[i]; }
+ out[t] = draw(philox_uniform(p.seed ^ kSpecDraftKey, p.counter + (uint64_t) t));
+ } else {
+ const SpecList* q = ql + t;
+ const int d = draft[t];
+ const int qn = q->n < KMAX ? q->n : KMAX;
+ auto qof = [&](int id) -> double {
+ for (int j = 0; j < qn; ++j)
+ if (q->ids[j] == id) return q->p[j];
+ return 0.0;
+ };
+ double pd = 0.0;
+ for (int i = 0; i < n_keep; ++i)
+ if (sel_ids[i] == d) { pd = w[i]; break; }
+ const double qd = qof(d);
+ int pick = -1;
+ if (qd <= 0.0) {
+ pick = draw(philox_uniform(p.seed, p.counter + (uint64_t) t)); // not a draft from q: the plain draw
+ } else if ((double) philox_uniform(p.seed ^ kSpecAcceptKey, p.counter + (uint64_t) t) * qd < pd) {
+ pick = d; // kept: probability min(1, p(d)/q(d))
+ } else {
+ // rejected: a draw from max(0, p - q), renormalised - over p's kept list (p is 0 outside it)
+ double r[KMAX], z = 0.0;
+ for (int i = 0; i < n_keep; ++i) {
+ const double x = w[i] - qof(sel_ids[i]);
+ r[i] = x > 0.0 ? x : 0.0;
+ z += r[i];
+ }
+ const float u = philox_uniform(p.seed ^ kSpecResidKey, p.counter + (uint64_t) t);
+ if (z > 0.0) {
+ double cum = 0.0;
+ for (int i = 0; i < n_keep; ++i) {
+ if (r[i] <= 0.0) continue;
+ pick = sel_ids[i]; // the last one with residual mass, if rounding leaves cum below u
+ cum += r[i] / z;
+ if ((double) u < cum) break;
+ }
+ } else {
+ pick = draw(u); // p == q to the last bit: a rejection had probability 0 - never in practice
+ }
+ }
+ out[t] = pick;
+ }
+ }
+}
+
+} // namespace
+
+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,
+ const SamplerParams& p, int* out, void* stream) {
+ if (n_tokens <= 0 || n_vocab <= 0) return;
+ if (p.penalty_last_n > 0 && (history == nullptr || history_len <= 0)) {
+ std::fprintf(stderr, "sample_tokens: penalty_last_n %d needs a history (got %p, len %d)\n",
+ p.penalty_last_n, (const void*) history, history_len);
+ std::exit(1);
+ }
+ const unsigned shmem = (history != nullptr && history_len > 0 && p.penalty_last_n > 0)
+ ? (unsigned) ((n_vocab + 31) / 32) * sizeof(unsigned) // the penalty bitmap
+ : 0;
+ if (p.greedy || p.temperature <= 0.0f) {
+ // One block per token, 1,024 threads over the vocabulary. See `sampler_greedy_kernel`.
+ const int gthreads = 1024;
+ sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(
+ logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);
+ } else {
+ // The same block-per-token shape: the selection's k argmax rounds reduce inside the block. See
+ // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.
+ sampler_kernel<0><<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(
+ logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);
+ }
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "sample_tokens launch: %s\n", cudaGetErrorString(e));
+ std::exit(1);
+ }
+ if (stream == nullptr) cudaDeviceSynchronize();
+}
+
+namespace {
+void spec_check(const char* who, int n_tokens, int n_vocab, const SamplerParams& p) {
+ if (p.greedy || p.temperature <= 0.0f || p.penalty_last_n > 0 || n_vocab <= 0) {
+ std::fprintf(stderr, "%s: a sampled request without penalties only (greedy %d, temperature %g, penalty_last_n "
+ "%d, n_vocab %d)\n", who, (int) p.greedy, p.temperature, p.penalty_last_n, n_vocab);
+ std::exit(1);
+ }
+ (void) n_tokens;
+}
+void spec_launched(const char* who, void* stream) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::fprintf(stderr, "%s launch: %s\n", who, cudaGetErrorString(e));
+ std::exit(1);
+ }
+ if (stream == nullptr) cudaDeviceSynchronize();
+}
+} // namespace
+
+void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,
+ void* stream) {
+ if (n_tokens <= 0) return;
+ spec_check("sample_draft", n_tokens, n_vocab, p);
+ sampler_kernel<1><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,
+ p, out, q, nullptr);
+ spec_launched("sample_draft", stream);
+}
+
+void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,
+ const int* draft, int* out, void* stream) {
+ if (n_tokens <= 0) return;
+ spec_check("sample_verify", n_tokens, n_vocab, p);
+ sampler_kernel<2><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,
+ p, out, const_cast<SpecList*>(q), draft);
+ spec_launched("sample_verify", stream);
+}
+
+} // namespace strata::kernels
diff --git a/tests/dsv41/ds4_cpu_bench.cpp b/tests/dsv41/ds4_cpu_bench.cpp
new file mode 100644
index 000000000..f5666216e
--- /dev/null
+++ b/tests/dsv41/ds4_cpu_bench.cpp
@@ -0,0 +1,105 @@
+// tests/ds4_cpu_bench.cpp - how fast does this CPU run DeepSeek-V4.1-Flash's EXL3 experts (5120 -> 2304 -> 5120, K 3)?
+//
+// ds4_cpu_bench [--threads N] [--node 0|1|-1] [--mode band|rows] [--experts N] [--per-call N] [--iters N]
+//
+// Random trellis words in memory bound to --node (-1: wherever the first touch puts them), --threads workers pinned to
+// that node's physical cores (CPU i is on node i % 2 here, CPUs 0..43 the physical cores), each call computing
+// --per-call experts' gate and up (all columns) from one activation: the time of a matrix pass and its GB/s.
+#include "strata/kernels/cpu/exl3_cpu.hpp"
+#include "../src/ds4/ds4_pool.hpp"
+
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace kc = strata::kernels::cpu;
+
+static void* node_alloc(size_t bytes, int node) {
+ void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (p == MAP_FAILED) return nullptr;
+ madvise(p, bytes, MADV_HUGEPAGE);
+ if (node >= 0) {
+ unsigned long mask = 1ul << node;
+ if (syscall(SYS_mbind, p, bytes, 2 /*MPOL_BIND*/, &mask, 64, 0) != 0) std::perror("mbind");
+ }
+ return p;
+}
+
+int main(int argc, char** argv) {
+ int threads = 22, node = 0, n_exp = 64, per_call = 2, iters = 40;
+ std::string mode = "rows";
+ for (int i = 1; i + 1 < argc; i += 2) {
+ const std::string a = argv[i];
+ if (a == "--threads") threads = std::atoi(argv[i + 1]);
+ else if (a == "--node") node = std::atoi(argv[i + 1]);
+ else if (a == "--mode") mode = argv[i + 1];
+ else if (a == "--experts") n_exp = std::atoi(argv[i + 1]);
+ else if (a == "--per-call") per_call = std::atoi(argv[i + 1]);
+ else if (a == "--iters") iters = std::atoi(argv[i + 1]);
+ }
+ const int E = 5120, FF = 2304, K = 3;
+ const size_t tb = (size_t) E / 16 * FF / 16 * 32 * K; // one matrix's trellis
+ std::vector<int> cpus;
+ for (int c = 0; c < 44; ++c)
+ if (node < 0 || c % 2 == node) cpus.push_back(c);
+ strata::ds4::Pool pool(threads - 1, std::vector<int>(cpus.begin() + 1, cpus.end()));
+ {
+ cpu_set_t set;
+ CPU_ZERO(&set);
+ CPU_SET(cpus[0], &set);
+ sched_setaffinity(0, sizeof set, &set);
+ }
+ uint8_t* mem = (uint8_t*) node_alloc(tb * n_exp, node);
+ std::vector<uint16_t> suh(E), svh(FF);
+ for (auto& v : suh) v = 0x3c00; // 1.0
+ for (auto& v : svh) v = 0x3c00;
+ pool.run(n_exp, [&](int e) {
+ std::mt19937 r(e);
+ uint32_t* w = (uint32_t*) (mem + tb * e);
+ for (size_t i = 0; i < tb / 4; ++i) w[i] = r();
+ });
+ std::vector<float> x(E, 0.5f), xh(E), acc((size_t) 64 * FF), y((size_t) per_call * FF);
+ const float xs = 0.0f;
+ kc::Exl3Part p0;
+ p0.suh = suh.data();
+ p0.svh = svh.data();
+ p0.k = E;
+ p0.n = FF;
+ p0.K = K;
+ kc::exl3_cpu_prepare(p0, x.data(), xh.data());
+ const int GQ = E / 256; // rows mode: pieces of 256 inputs over every column
+ std::vector<float> accp((size_t) per_call * GQ * FF);
+ auto t0 = std::chrono::steady_clock::now();
+ long long bytes = 0;
+ for (int it = 0; it < iters; ++it) {
+ const int base = (it * per_call) % (n_exp - per_call + 1);
+ if (mode == "band") {
+ pool.run(per_call * (FF / 128), [&](int j) {
+ kc::Exl3Part p = p0;
+ p.trellis = (const uint32_t*) (mem + tb * (base + j / (FF / 128)));
+ kc::exl3_cpu_band(p, xh.data(), xs, (j % (FF / 128)) * 128,
+ y.data() + (size_t) (j / (FF / 128)) * FF + (j % (FF / 128)) * 128);
+ });
+ } else {
+ pool.run(per_call * GQ, [&](int j) {
+ kc::Exl3Part p = p0;
+ p.trellis = (const uint32_t*) (mem + tb * (base + j / GQ));
+ const int q = j % GQ;
+ kc::exl3_cpu_rows_acc(p, xh.data(), q * 16, (q + 1) * 16, accp.data() + (size_t) j * FF);
+ });
+ }
+ bytes += (long long) tb * per_call;
+ }
+ const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+ std::printf("%s node %d threads %d per-call %d: %.3f ms per matrix, %.1f GB/s\n", mode.c_str(), node, threads,
+ per_call, ms / (iters * per_call), bytes / 1e6 / ms);
+ return 0;
+}
diff --git a/tests/dsv41/ds4_gguf_test.cu b/tests/dsv41/ds4_gguf_test.cu
new file mode 100644
index 000000000..a24059bcb
--- /dev/null
+++ b/tests/dsv41/ds4_gguf_test.cu
@@ -0,0 +1,1616 @@
+// tests/ds4_gguf_test.cu - maya-ds4's GGUF kernels (src/ds4/ds4_gguf.cu, ds4_gguf_cpu.cpp) against a plain
+// double-precision reference of random BF16 matrices and MXFP4 experts: the GEMVs (BF16 / MXFP4, 1..16 activations,
+// aligned or not), the token experts (gate / up / swiglu / down / combine, experts off the device, the CPU part, the
+// shared expert), a window's grouped experts (bit for bit the token path's), the prompt path's expert sets and the CPU
+// lane's rows; and the KV caches' packed rows (ds4_kernels.cu) against the BF16 rows bit for bit. Needs a GPU, no
+// model. ds4_gguf_test [n_embd n_ff] | --kv (the caches alone)
+#include "../src/ds4/ds4_gguf.cuh"
+#include "../src/ds4/ds4_gguf_cpu.hpp"
+#include "../src/ds4/ds4_iq.cuh"
+#include "../src/ds4/ds4_kernels.cuh"
+#include "../src/ds4/ds4_q6k.cuh"
+
+#include "ggml.h" // (the low-bit types' reference values: ggml's own dequantize_row_*)
+
+#include <cuda_fp16.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <functional>
+#include <thread>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace gg = strata::ds4::gguf;
+namespace x3 = strata::kernels::exl3;
+
+namespace {
+
+int g_fail = 0;
+std::mt19937 rng(1234);
+
+#define CU(x) \
+ do { \
+ cudaError_t e_ = (x); \
+ if (e_ != cudaSuccess) { \
+ std::printf("CUDA %s: %s (line %d)\n", #x, cudaGetErrorString(e_), __LINE__); \
+ std::exit(1); \
+ } \
+ } while (0)
+
+float frand(float a = 1.0f) { return std::uniform_real_distribution<float>(-a, a)(rng); }
+
+float bf16f(uint16_t b) {
+ uint32_t u = (uint32_t) b << 16;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+uint16_t f2bf(float f) { // truncating is fine: the reference reads back the same bits
+ uint32_t u;
+ std::memcpy(&u, &f, 4);
+ return (uint16_t) (u >> 16);
+}
+const float kVal[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+double e8m0_half(int e) { return std::ldexp(1.0, e - 128); }
+
+// random MXFP4 rows [n][k]: scales around 2^-6, every code
+std::vector<uint8_t> mx_rand(int n, int k) {
+ std::vector<uint8_t> w(gg::mx_bytes(n, k));
+ for (size_t b = 0; b < w.size() / 17; ++b) {
+ w[b * 17] = (uint8_t) (120 + rng() % 6);
+ for (int j = 1; j < 17; ++j) w[b * 17 + j] = (uint8_t) (rng() & 0xff);
+ }
+ return w;
+}
+double mx_val(const uint8_t* row, int j) { // value j of a row
+ const uint8_t* p = row + (size_t) (j / 32) * 17;
+ const int jj = j % 32;
+ const int code = jj < 16 ? (p[1 + jj] & 15) : (p[1 + jj - 16] >> 4);
+ return kVal[code] * e8m0_half(p[0]);
+}
+// random Q8_0 rows [n][k], and a value of one
+std::vector<uint8_t> q8_rand(int n, int k) {
+ std::vector<uint8_t> w(gg::q8_row_bytes(k) * n);
+ for (size_t b = 0; b < w.size() / 34; ++b) {
+ const __half d = __float2half(0.001f + 0.002f * std::fabs(frand()));
+ std::memcpy(&w[b * 34], &d, 2);
+ for (int j = 2; j < 34; ++j) w[b * 34 + j] = (uint8_t) (rng() & 0xff);
+ }
+ return w;
+}
+double q8_val(const uint8_t* row, int j) {
+ const uint8_t* p = row + (size_t) (j / 32) * 34;
+ __half d;
+ std::memcpy(&d, p, 2);
+ return (double) __half2float(d) * (int8_t) p[2 + j % 32];
+}
+double mx_dot(const uint8_t* w, int k, int row, const float* x) {
+ const uint8_t* r = w + (size_t) row * gg::mx_row_bytes(k);
+ double s = 0;
+ for (int j = 0; j < k; ++j) s += mx_val(r, j) * x[j];
+ return s;
+}
+// x rounded as the kernels' q8_quant: per 32 values sc = max|x| / 127 (F32), q = round-half-even(x / sc)
+std::vector<float> q8_round(const float* x, int k) {
+ std::vector<float> v(k);
+ for (int b = 0; b < k / 32; ++b) {
+ float am = 0.0f;
+ for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));
+ const float sc = am / 127.0f;
+ for (int j = 0; j < 32; ++j) {
+ const float q = am > 0.0f ? std::nearbyint(x[b * 32 + j] / sc) : 0.0f;
+ v[b * 32 + j] = (float) ((double) q * sc);
+ }
+ }
+ return v;
+}
+double swiglu(double g, double u, double lim) {
+ if (lim > 0) {
+ g = std::min(g, lim);
+ u = std::min(std::max(u, -lim), lim);
+ }
+ return g / (1.0 + std::exp(-g)) * u;
+}
+
+void expect(const char* what, const float* got, const double* want, size_t n, double rtol) {
+ double worst = 0, scale = 1e-30;
+ for (size_t i = 0; i < n; ++i) scale = std::max(scale, std::fabs(want[i]));
+ size_t at = 0;
+ for (size_t i = 0; i < n; ++i) {
+ const double e = std::fabs(got[i] - want[i]) / scale;
+ if (!(e <= worst)) {
+ worst = e;
+ at = i;
+ }
+ }
+ const bool ok = worst <= rtol;
+ if (!ok) ++g_fail;
+ std::printf("%-44s %s max rel err %.2e (at %zu: %.6g vs %.6g)\n", what, ok ? "ok " : "FAIL", worst, at,
+ (double) got[at], want[at]);
+}
+
+template <class T>
+T* dev(const std::vector<T>& h) {
+ T* p = nullptr;
+ CU(cudaMalloc(&p, std::max<size_t>(1, h.size()) * sizeof(T)));
+ if (!h.empty()) CU(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice));
+ return p;
+}
+template <class T>
+std::vector<T> host(const T* p, size_t n) {
+ std::vector<T> h(n);
+ CU(cudaMemcpy(h.data(), p, n * sizeof(T), cudaMemcpyDeviceToHost));
+ return h;
+}
+
+void test_mv(cudaStream_t s) {
+ for (int fmt : {gg::kFmtBF16, gg::kFmtMXFP4, gg::kFmtQ8_0})
+ for (int nc : {1, 3, 16})
+ for (int off : {0, 1}) { // off 1: activations not 16-byte aligned (the scalar kernels)
+ const int k = 1280, n = 520;
+ std::vector<uint16_t> wb;
+ std::vector<uint8_t> wm;
+ if (fmt == gg::kFmtBF16) {
+ wb.resize((size_t) n * k);
+ for (auto& v : wb) v = f2bf(frand(0.05f));
+ } else {
+ wm = fmt == gg::kFmtMXFP4 ? mx_rand(n, k) : q8_rand(n, k);
+ }
+ const size_t xcs = (size_t) k + 4 + off;
+ std::vector<float> x(xcs * nc + 8), bias(n);
+ for (auto& v : x) v = frand();
+ for (auto& v : bias) v = frand(0.1f);
+ float* dx = dev(x);
+ float* db = dev(bias);
+ std::vector<float> y0((size_t) n * nc, 0.0f);
+ float* dy = dev(y0);
+ void* dw = fmt == gg::kFmtBF16 ? (void*) dev(wb) : (void*) dev(wm);
+ x3::Mat m;
+ m.fmt = fmt;
+ m.raw = dw;
+ m.k = k;
+ m.n = n;
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = dx + off;
+ j.y = dy;
+ j.bias = db;
+ j.alpha = 0.5f;
+ j.ncol = nc;
+ j.xf_cs = xcs;
+ j.y_cs = n;
+ if (!gg::mv(&j, 1, s)) ++g_fail;
+ CU(cudaStreamSynchronize(s));
+ std::vector<float> y = host(dy, (size_t) n * nc);
+ std::vector<double> want((size_t) n * nc);
+ for (int c = 0; c < nc; ++c)
+ for (int r = 0; r < n; ++r) {
+ const float* xc = x.data() + off + c * xcs;
+ double d = 0;
+ if (fmt == gg::kFmtBF16)
+ for (int i = 0; i < k; ++i) d += (double) bf16f(wb[(size_t) r * k + i]) * xc[i];
+ else if (fmt == gg::kFmtMXFP4)
+ d = mx_dot(wm.data(), k, r, xc);
+ else {
+ const std::vector<float> xr = q8_round(xc, k);
+ for (int i = 0; i < k; ++i) d += q8_val(wm.data() + (size_t) r * gg::q8_row_bytes(k), i) * xr[i];
+ }
+ want[(size_t) c * n + r] = 0.5 * d + bias[r];
+ }
+ char what[96];
+ std::snprintf(what, sizeof(what), "mv %s, %d activations%s", fmt == gg::kFmtBF16 ? "BF16" : fmt == gg::kFmtMXFP4 ? "MXFP4" : "Q8_0", nc,
+ off ? ", unaligned" : "");
+ expect(what, y.data(), want.data(), want.size(), 2e-5);
+ // the first activation alone gives the same bits as in the batch
+ if (nc > 1) {
+ j.ncol = 1;
+ std::vector<float> y1(n, 0.0f);
+ float* dy1 = dev(y1);
+ j.y = dy1;
+ gg::mv(&j, 1, s);
+ CU(cudaStreamSynchronize(s));
+ y1 = host(dy1, n);
+ const bool same = std::memcmp(y1.data(), y.data(), (size_t) n * 4) == 0;
+ if (!same) ++g_fail;
+ std::printf("%-44s %s\n", " ... one activation alone, bit for bit", same ? "ok" : "FAIL");
+ cudaFree(dy1);
+ }
+ cudaFree(dx), cudaFree(db), cudaFree(dy), cudaFree(dw);
+ }
+ // rows(): BF16 -> F16 of a view
+ {
+ const int k = 256, n = 384;
+ std::vector<uint16_t> wb((size_t) n * k);
+ for (auto& v : wb) v = f2bf(frand(2.0f));
+ uint16_t* dw = dev(wb);
+ x3::Mat m;
+ m.fmt = gg::kFmtBF16;
+ m.raw = dw;
+ m.k = k;
+ m.n = n;
+ const x3::Mat v = gg::view(m, 128, 128);
+ std::vector<uint16_t> z((size_t) 128 * k);
+ uint16_t* dd = dev(z);
+ gg::rows(v, dd, false, s);
+ CU(cudaStreamSynchronize(s));
+ z = host(dd, z.size());
+ std::vector<float> got(z.size());
+ std::vector<double> want(z.size());
+ for (size_t i = 0; i < z.size(); ++i) {
+ got[i] = __half2float(*(const __half*) &z[i]);
+ want[i] = bf16f(wb[(size_t) 128 * k + i]);
+ }
+ expect("rows BF16 -> F16 (a view)", got.data(), want.data(), want.size(), 1e-3);
+ cudaFree(dw), cudaFree(dd);
+ }
+ { // rows(): Q8_0 -> F16 of a view
+ const int k = 256, n = 384;
+ std::vector<uint8_t> wq = q8_rand(n, k);
+ uint8_t* dw = dev(wq);
+ x3::Mat m;
+ m.fmt = gg::kFmtQ8_0;
+ m.raw = dw;
+ m.k = k;
+ m.n = n;
+ const x3::Mat v = gg::view(m, 128, 128);
+ std::vector<uint16_t> z((size_t) 128 * k);
+ uint16_t* dd = dev(z);
+ gg::rows(v, dd, false, s);
+ CU(cudaStreamSynchronize(s));
+ z = host(dd, z.size());
+ std::vector<float> got(z.size());
+ std::vector<double> want(z.size());
+ for (size_t i = 0; i < z.size(); ++i) {
+ got[i] = __half2float(*(const __half*) &z[i]);
+ want[i] = q8_val(wq.data() + (128 + i / k) * gg::q8_row_bytes(k), (int) (i % k));
+ }
+ expect("rows Q8_0 -> F16 (a view)", got.data(), want.data(), want.size(), 1e-3);
+ cudaFree(dw), cudaFree(dd);
+ }
+}
+
+struct Experts {
+ int D, F, n_exp;
+ x3::ExpertLayout lay;
+ size_t blob;
+ std::vector<uint8_t> host; // n_exp blobs
+ uint8_t* dev = nullptr;
+};
+Experts make_experts(int D, int F, int n_exp) {
+ Experts E{D, F, n_exp, {}, 0, {}, nullptr};
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ E.lay.trellis[r] = off;
+ off += (gg::mx_bytes(r < 2 ? F : D, r < 2 ? D : F) + 255) / 256 * 256;
+ }
+ E.lay.fmt = gg::kFmtMXFP4;
+ E.blob = off;
+ E.host.assign(E.blob * n_exp, 0);
+ for (int e = 0; e < n_exp; ++e)
+ for (int r = 0; r < 3; ++r) {
+ const std::vector<uint8_t> m = mx_rand(r < 2 ? F : D, r < 2 ? D : F);
+ std::memcpy(E.host.data() + e * E.blob + E.lay.trellis[r], m.data(), m.size());
+ }
+ E.dev = dev(E.host);
+ return E;
+}
+const uint8_t* emat(const Experts& E, int e, int r) { return E.host.data() + e * E.blob + E.lay.trellis[r]; }
+// reference of one token's expert e: h then the down output
+// (x and h rounded to int8 per 32 first, as the dp4a kernels round them)
+void ref_expert(const Experts& E, int e, const float* x, double lim, std::vector<float>& h, std::vector<double>& y) {
+ const std::vector<float> xr = q8_round(x, E.D);
+ h.resize(E.F);
+ for (int f = 0; f < E.F; ++f)
+ h[f] = (float) swiglu(mx_dot(emat(E, e, 0), E.D, f, xr.data()), mx_dot(emat(E, e, 1), E.D, f, xr.data()), lim);
+ const std::vector<float> hr = q8_round(h.data(), E.F);
+ y.resize(E.D);
+ for (int d = 0; d < E.D; ++d) y[d] = mx_dot(emat(E, e, 2), E.F, d, hr.data());
+}
+
+void test_experts(int D, int F, cudaStream_t s) {
+ const int n_exp = 12, k = 6, T = 5;
+ const float lim = 10.0f;
+ Experts E = make_experts(D, F, n_exp);
+ // T tokens' plans [T][8]: picks, weights, some picks off the device (ptr 0), token 1 with a CPU part
+ std::vector<int> pick((size_t) T * 8, 0);
+ std::vector<float> pw((size_t) T * 8, 0.0f);
+ std::vector<unsigned long long> pp((size_t) T * 8, 0ull);
+ for (int t = 0; t < T; ++t) {
+ std::vector<int> all(n_exp);
+ for (int i = 0; i < n_exp; ++i) all[i] = i;
+ std::shuffle(all.begin(), all.end(), rng);
+ for (int i = 0; i < k; ++i) {
+ pick[t * 8 + i] = all[i];
+ pw[t * 8 + i] = 0.2f + 0.1f * frand();
+ const bool on = !(t == 2 && i == 3) && !(t == 4 && i == 0);
+ pp[t * 8 + i] = on ? (unsigned long long) (E.dev + (size_t) all[i] * E.blob) : 0ull;
+ }
+ }
+ std::vector<float> x((size_t) T * D), sh((size_t) T * D), cpu((size_t) T * D);
+ for (auto& v : x) v = frand();
+ for (auto& v : sh) v = frand(0.1f);
+ for (auto& v : cpu) v = frand(0.1f);
+ std::vector<int> cflag(T, 0);
+ cflag[1] = 1;
+ float *dx = dev(x), *dsh = dev(sh), *dcpu = dev(cpu), *dpw = dev(pw);
+ unsigned long long* dpp = dev(pp);
+ int* dflag = dev(cflag);
+ // reference outputs
+ std::vector<double> want((size_t) T * D, 0.0);
+ std::vector<float> hh;
+ std::vector<double> yy;
+ for (int t = 0; t < T; ++t) {
+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] = 0;
+ for (int i = 0; i < k; ++i) {
+ if (pp[t * 8 + i] == 0ull) continue;
+ ref_expert(E, pick[t * 8 + i], x.data() + (size_t) t * D, lim, hh, yy);
+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] += pw[t * 8 + i] * yy[d];
+ }
+ for (int d = 0; d < D; ++d)
+ want[(size_t) t * D + d] += (cflag[t] ? cpu[(size_t) t * D + d] : 0.0f) + sh[(size_t) t * D + d];
+ }
+ // the token path, token by token
+ std::vector<float> tok((size_t) T * D);
+ float* dh = dev(std::vector<float>((size_t) 8 * F));
+ float* dout = dev(std::vector<float>(D));
+ for (int t = 0; t < T; ++t) {
+ x3::MoeArgs a;
+ a.plan_ptr = dpp + t * 8;
+ a.plan_w = dpw + t * 8;
+ a.cpu_part = dcpu + (size_t) t * D;
+ a.cpu_flag = dflag + t;
+ a.lay = E.lay;
+ a.k = k;
+ a.n_embd = D;
+ a.n_ff = F;
+ a.limit = lim;
+ gg::moe_gate_up(a, dx + (size_t) t * D, dh, s);
+ gg::moe_down(a, dh, dsh + (size_t) t * D, dout, s);
+ CU(cudaStreamSynchronize(s));
+ CU(cudaMemcpy(tok.data() + (size_t) t * D, dout, (size_t) D * 4, cudaMemcpyDeviceToHost));
+ }
+ char what[96];
+ std::snprintf(what, sizeof(what), "token experts (D %d, F %d)", D, F);
+ expect(what, tok.data(), want.data(), want.size(), 1e-3); // (an h value can round to its other int8)
+ // the window: groups by expert (the device ones), entries e = t * 8 + i
+ std::vector<unsigned long long> gptr;
+ std::vector<int> gstart, gent;
+ for (int x0 = 0; x0 < n_exp; ++x0) {
+ std::vector<int> ents;
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < k; ++i)
+ if (pick[t * 8 + i] == x0 && pp[t * 8 + i] != 0ull) ents.push_back(t * 8 + i);
+ if (ents.empty()) continue;
+ gptr.push_back((unsigned long long) (E.dev + (size_t) x0 * E.blob));
+ gstart.push_back((int) gent.size());
+ for (int e : ents) gent.push_back(e);
+ }
+ gstart.push_back((int) gent.size());
+ std::vector<int> ng(1, (int) gptr.size());
+ unsigned long long* dgp = dev(gptr);
+ int *dgs = dev(gstart), *dge = dev(gent), *dng = dev(ng);
+ float* dwh = dev(std::vector<float>((size_t) T * 8 * F));
+ float* dpart = dev(std::vector<float>((size_t) T * 8 * D));
+ float* dwo = dev(std::vector<float>((size_t) T * D));
+ x3::WinArgs w;
+ w.grp_ptr = dgp;
+ w.grp_start = dgs;
+ w.grp_ent = dge;
+ w.n_groups = dng;
+ w.plan_ptr = dpp;
+ w.plan_w = dpw;
+ w.cpu_part = dcpu;
+ w.cpu_stride = D;
+ w.cpu_flag = dflag;
+ w.x = dx;
+ w.x_stride = D;
+ w.h = dwh;
+ w.part = dpart;
+ w.sh_out = dsh;
+ w.out = dwo;
+ w.out_stride = D;
+ w.lay = E.lay;
+ w.T = T;
+ w.k = k;
+ w.n_embd = D;
+ w.n_ff = F;
+ w.limit = lim;
+ if (!gg::win_experts(w, s)) ++g_fail;
+ CU(cudaStreamSynchronize(s));
+ std::vector<float> win = host(dwo, (size_t) T * D);
+ std::snprintf(what, sizeof(what), "window experts (T %d)", T);
+ expect(what, win.data(), want.data(), want.size(), 1e-3);
+ const bool same = std::memcmp(win.data(), tok.data(), win.size() * 4) == 0;
+ if (!same) ++g_fail;
+ std::printf("%-44s %s\n", " ... bit for bit the token path's", same ? "ok" : "FAIL");
+
+ // the prompt path's set: rows sorted by expert (bounds), each row a token index (row_idx) - experts of a few rows (the
+ // light kernels), then of more (the tiles)
+ for (int R : {100, 200, 400}) {
+ std::vector<int> owner(R), idx(R);
+ for (int r = 0; r < R; ++r) owner[r] = (int) (rng() % n_exp);
+ std::sort(owner.begin(), owner.end());
+ std::vector<int> bounds(n_exp + 1, 0);
+ for (int e = 0, r = 0; e <= n_exp; ++e) {
+ while (r < R && owner[r] < e) ++r;
+ bounds[e] = r;
+ }
+ int max_rows = 0;
+ for (int e = 0; e < n_exp; ++e) max_rows = std::max(max_rows, bounds[e + 1] - bounds[e]);
+ const int NT = 40;
+ std::vector<float> xs((size_t) NT * D);
+ for (auto& v : xs) v = frand();
+ for (int r = 0; r < R; ++r) idx[r] = (int) (rng() % NT);
+ float* dxs = dev(xs);
+ int *dbd = dev(bounds), *didx = dev(idx);
+ uint16_t* dh16 = dev(std::vector<uint16_t>((size_t) R * F));
+ float* dy = dev(std::vector<float>((size_t) R * D));
+ x3::SetArgs sa;
+ sa.wbase = E.dev;
+ sa.stride = E.blob;
+ sa.lay = E.lay;
+ sa.n_exp = n_exp;
+ sa.bounds = dbd;
+ sa.row0 = 0;
+ sa.max_rows = max_rows;
+ sa.n_embd = D;
+ sa.n_ff = F;
+ sa.limit = lim;
+ gg::set_gate_up(sa, dxs, D, didx, dh16, F, s);
+ gg::set_down(sa, dh16, F, dy, D, s);
+ CU(cudaStreamSynchronize(s));
+ std::vector<uint16_t> h16 = host(dh16, (size_t) R * F);
+ std::vector<float> y = host(dy, (size_t) R * D);
+ std::vector<float> hgot((size_t) R * F);
+ std::vector<double> hwant((size_t) R * F), ywant((size_t) R * D);
+ // (an expert of at most 16 rows takes the light kernels: its inputs rounded to int8 per 32, as the token path's)
+ int n_light = 0;
+ for (int e = 0; e < n_exp; ++e) n_light += bounds[e + 1] - bounds[e] <= 16 ? bounds[e + 1] - bounds[e] : 0;
+ for (int r = 0; r < R; ++r) {
+ const bool light = bounds[owner[r] + 1] - bounds[owner[r]] <= 16;
+ const std::vector<float> xq = light ? q8_round(xs.data() + (size_t) idx[r] * D, D) : std::vector<float>();
+ const float* xr = light ? xq.data() : xs.data() + (size_t) idx[r] * D;
+ std::vector<float> hr(F);
+ for (int f = 0; f < F; ++f) {
+ hwant[(size_t) r * F + f] =
+ swiglu(mx_dot(emat(E, owner[r], 0), D, f, xr), mx_dot(emat(E, owner[r], 1), D, f, xr), lim);
+ hgot[(size_t) r * F + f] = __half2float(*(const __half*) &h16[(size_t) r * F + f]);
+ hr[f] = hgot[(size_t) r * F + f]; // the down from the F16 h the kernel kept
+ }
+ if (light) hr = q8_round(hr.data(), F);
+ for (int d = 0; d < D; ++d) ywant[(size_t) r * D + d] = mx_dot(emat(E, owner[r], 2), F, d, hr.data());
+ }
+ char sw[64];
+ std::snprintf(sw, sizeof(sw), "set gate/up (F16 h), %d rows, %d light", R, n_light);
+ expect(sw, hgot.data(), hwant.data(), hwant.size(), 2e-3);
+ std::snprintf(sw, sizeof(sw), "set down, %d rows, %d light", R, n_light);
+ expect(sw, y.data(), ywant.data(), ywant.size(), n_light > 0 ? 1e-4 : 5e-5);
+ cudaFree(dxs), cudaFree(dbd), cudaFree(didx), cudaFree(dh16), cudaFree(dy);
+ }
+ // the CPU lane's rows
+ {
+ const int n = 3;
+ const float* xp[n] = {x.data(), x.data() + D, x.data() + 2 * D};
+ std::vector<float> y((size_t) n * 100);
+ float* yp[n] = {y.data(), y.data() + 100, y.data() + 200};
+ gg::mx_cpu_rows(emat(E, 5, 0), gg::mx_row_bytes(D), D, 7, 107, xp, n, yp);
+ std::vector<double> want2((size_t) n * 100);
+ for (int i = 0; i < n; ++i)
+ for (int r = 7; r < 107; ++r) want2[(size_t) i * 100 + r - 7] = mx_dot(emat(E, 5, 0), D, r, xp[i]);
+ expect("CPU lane rows", y.data(), want2.data(), want2.size(), 5e-5);
+ }
+ // the CPU lane's int8 rows (gg::mx_rows), every tier this CPU runs: the gate and the down shapes, 1..9 tokens (past
+ // a call's 8), against the double reference on the q8-rounded activations; the tiers against the scalar one; the
+ // activation's bytes (vector and scalar quant, whole and by bands) against q8_round's values
+ {
+ const int NT = 9, R0 = 5, R1 = 69;
+ for (int down = 0; down < 2; ++down) {
+ const int k = down ? F : D;
+ const uint8_t* w = emat(E, 7, down ? 2 : 0);
+ const size_t rb = gg::mx_row_bytes(k), ab = gg::mx_act_bytes(k);
+ std::vector<float> xs((size_t) NT * k);
+ for (auto& v : xs) v = down ? frand(0.05f) : frand();
+ for (int j = 0; j < 32; ++j) xs[(size_t) 2 * k + 32 + j] = 0.0f; // (an all-zero block: scale 0)
+ std::vector<uint8_t> act((size_t) NT * ab), act_s(ab), act_b(ab);
+ const void* ap[NT];
+ bool q_ok = true;
+ for (int i = 0; i < NT; ++i) {
+ uint8_t* a = act.data() + (size_t) i * ab;
+ gg::mx_quant(xs.data() + (size_t) i * k, k, a);
+ gg::mx_quant_blocks_scalar(xs.data() + (size_t) i * k, 0, k / 32, act_s.data(), k);
+ for (int b0 = 0; b0 < k / 32; b0 += 2) // (by bands of 2 blocks, as the lane's jobs write h)
+ gg::mx_quant_blocks(xs.data() + (size_t) i * k + 32 * b0, b0, b0 + 2, act_b.data(), k);
+ q_ok = q_ok && std::memcmp(a, act_s.data(), ab) == 0 && std::memcmp(a, act_b.data(), ab) == 0;
+ const std::vector<float> xr = q8_round(xs.data() + (size_t) i * k, k);
+ for (int b = 0; b < k / 32; ++b)
+ for (int j = 0; j < 32; ++j) {
+ const int q = gg::mx_act_value(a, b, j);
+ const float sc = gg::mx_act_scale(a, k, b);
+ q_ok = q_ok && (float) ((double) q * sc) == xr[(size_t) b * 32 + j];
+ }
+ ap[i] = a;
+ }
+ if (!q_ok) ++g_fail;
+ std::printf("%-44s %s\n", down ? "CPU int8 quant (down's k)" : "CPU int8 quant (gate's k)",
+ q_ok ? "ok vector = scalar = bands = q8_round" : "FAIL");
+ std::vector<double> want3((size_t) NT * (R1 - R0));
+ for (int i = 0; i < NT; ++i) {
+ const std::vector<float> xr = q8_round(xs.data() + (size_t) i * k, k);
+ for (int r = R0; r < R1; ++r) want3[(size_t) i * (R1 - R0) + r - R0] = mx_dot(w, k, r, xr.data());
+ }
+ std::vector<float> ref((size_t) NT * (R1 - R0));
+ for (int t = 0; t < gg::kMxTiers; ++t) {
+ if (!gg::mx_tier_ok(t)) continue;
+ for (int n : {1, 2, 3, 8, 9}) {
+ std::vector<float> y((size_t) n * (R1 - R0));
+ std::vector<float*> yp((size_t) n);
+ for (int i = 0; i < n; ++i) yp[(size_t) i] = y.data() + (size_t) i * (R1 - R0);
+ gg::mx_rows_tier(t, w, rb, k, R0, R1, ap, n, yp.data(), t == gg::kMxScalar ? 0 : 256);
+ char what[96];
+ std::snprintf(what, sizeof what, "CPU int8 rows %s %s n=%d", down ? "down" : "gate",
+ gg::mx_tier_name(t), n);
+ expect(what, y.data(), want3.data(), y.size(), 5e-5);
+ if (t == gg::kMxScalar && n == NT) ref = y;
+ if (t != gg::kMxScalar && !ref.empty()) { // (every tier the same bits as the scalar one)
+ const bool same = std::memcmp(y.data(), ref.data(), y.size() * sizeof(float)) == 0;
+ if (!same) ++g_fail;
+ std::printf(" %-42s %s\n", what, same ? "ok bit-identical to the scalar tier" : "FAIL differs from the scalar tier");
+ }
+ }
+ }
+ }
+ std::printf("CPU int8 rows: this CPU's tier %s\n", gg::mx_tier_name(gg::mx_tier()));
+ }
+ cudaFree(E.dev), cudaFree(dx), cudaFree(dsh), cudaFree(dcpu), cudaFree(dpw), cudaFree(dpp), cudaFree(dflag);
+ cudaFree(dh), cudaFree(dout), cudaFree(dgp), cudaFree(dgs), cudaFree(dge), cudaFree(dng), cudaFree(dwh);
+ cudaFree(dpart), cudaFree(dwo);
+}
+
+// ---------------------------------------------------------------- the low-bit types (ds4_iq)
+// random blocks of a type: random bytes, the block scales small normal values (every grid index, sign and code)
+std::vector<uint8_t> q_rand(int type, int n, int k) {
+ const size_t rb = gg::type_row_bytes(type, k), bb = (size_t) gg::iq::block_bytes(type);
+ std::vector<uint8_t> w(rb * n);
+ for (auto& v : w) v = (uint8_t) (rng() & 0xff);
+ auto put = [&](size_t at, float f) {
+ const __half h = __float2half(f);
+ std::memcpy(&w[at], &h, 2);
+ };
+ for (size_t o = 0; o < w.size(); o += bb) {
+ const float d = 0.001f + 0.002f * std::fabs(frand());
+ switch (type) {
+ case gg::iq::kQ4_K: put(o, d), put(o + 2, 0.5f * d); break;
+ case gg::iq::kQ2_K: put(o + 80, d), put(o + 82, 0.5f * d); break;
+ case gg::iq::kQ3_K: put(o + 108, d); break;
+ default: put(o, d); break; // (the i-quants: d first)
+ }
+ }
+ return w;
+}
+// a row's dot with x in double, the weights ggml's dequantize_row_* values
+double q_dot(int type, const uint8_t* row, int k, const float* x) {
+ static thread_local std::vector<float> w;
+ w.resize(k);
+ ggml_get_type_traits((ggml_type) type)->to_float(row, w.data(), k);
+ double s = 0;
+ for (int i = 0; i < k; ++i) s += (double) w[i] * x[i];
+ return s;
+}
+const char* tn(int t) { return gg::type_name(t); }
+
+void test_mv_q(int type, cudaStream_t s) {
+ for (int nc : {1, 3, 16}) {
+ const int k = 768, n = 136;
+ std::vector<uint8_t> wq = q_rand(type, n, k);
+ const size_t xcs = (size_t) k + 8;
+ std::vector<float> x(xcs * nc), bias(n);
+ for (auto& v : x) v = frand();
+ for (auto& v : bias) v = frand(0.1f);
+ float *dx = dev(x), *db = dev(bias), *dy = dev(std::vector<float>((size_t) n * nc));
+ uint8_t* dw = dev(wq);
+ x3::Mat m;
+ m.fmt = gg::kFmtGG;
+ m.K = type;
+ m.raw = dw;
+ m.k = k;
+ m.n = n;
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = dx;
+ j.y = dy;
+ j.bias = db;
+ j.alpha = 0.5f;
+ j.ncol = nc;
+ j.xf_cs = xcs;
+ j.y_cs = n;
+ if (!gg::mv(&j, 1, s)) ++g_fail;
+ CU(cudaStreamSynchronize(s));
+ std::vector<float> y = host(dy, (size_t) n * nc);
+ std::vector<double> want((size_t) n * nc);
+ for (int c = 0; c < nc; ++c) {
+ const std::vector<float> xr = q8_round(x.data() + c * xcs, k);
+ for (int r = 0; r < n; ++r)
+ want[(size_t) c * n + r] = 0.5 * q_dot(type, wq.data() + r * gg::type_row_bytes(type, k), k, xr.data()) + bias[r];
+ }
+ char what[96];
+ std::snprintf(what, sizeof(what), "mv %s, %d activations", tn(type), nc);
+ expect(what, y.data(), want.data(), want.size(), 2e-5);
+ cudaFree(dx), cudaFree(db), cudaFree(dy), cudaFree(dw);
+ }
+}
+
+struct QExperts {
+ int D, F, n_exp, tg, td;
+ x3::ExpertLayout lay;
+ size_t blob;
+ std::vector<uint8_t> host;
+ uint8_t* dev = nullptr;
+ const uint8_t* mat(int e, int r) const { return host.data() + e * blob + lay.trellis[r]; }
+ size_t rb(int r) const { return gg::type_row_bytes(r < 2 ? tg : td, r < 2 ? D : F); }
+ double dot(int e, int r, int row, const float* x) const {
+ return q_dot(r < 2 ? tg : td, mat(e, r) + row * rb(r), r < 2 ? D : F, x);
+ }
+};
+QExperts make_q(int D, int F, int n_exp, int tg, int td) {
+ QExperts E{D, F, n_exp, tg, td, {}, 0, {}, nullptr};
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ E.lay.trellis[r] = off;
+ E.lay.K[r] = r < 2 ? tg : td;
+ off += ((r < 2 ? F : D) * gg::type_row_bytes(E.lay.K[r], r < 2 ? D : F) + 255) / 256 * 256;
+ }
+ E.lay.fmt = gg::kFmtMXFP4;
+ E.blob = off;
+ E.host.assign(E.blob * n_exp, 0);
+ for (int e = 0; e < n_exp; ++e)
+ for (int r = 0; r < 3; ++r) {
+ const std::vector<uint8_t> m = q_rand(E.lay.K[r], r < 2 ? F : D, r < 2 ? D : F);
+ std::memcpy(E.host.data() + e * E.blob + E.lay.trellis[r], m.data(), m.size());
+ }
+ E.dev = dev(E.host);
+ return E;
+}
+
+// test_experts' checks for low-bit experts: the token path, the window (bit for bit the token path's), the prompt sets
+// and the CPU lane (ggml-cpu's arithmetic: Q8_K activations, so against a reference on those)
+void test_experts_q(int D, int F, int tg, int td, cudaStream_t s) {
+ const int n_exp = 10, k = 6, T = 4;
+ const float lim = 10.0f;
+ QExperts E = make_q(D, F, n_exp, tg, td);
+ std::vector<int> pick((size_t) T * 8, 0);
+ std::vector<float> pw((size_t) T * 8, 0.0f);
+ std::vector<unsigned long long> pp((size_t) T * 8, 0ull);
+ for (int t = 0; t < T; ++t) {
+ std::vector<int> all(n_exp);
+ for (int i = 0; i < n_exp; ++i) all[i] = i;
+ std::shuffle(all.begin(), all.end(), rng);
+ for (int i = 0; i < k; ++i) {
+ pick[t * 8 + i] = all[i];
+ pw[t * 8 + i] = 0.2f + 0.1f * frand();
+ const bool on = !(t == 2 && i == 3);
+ pp[t * 8 + i] = on ? (unsigned long long) (E.dev + (size_t) all[i] * E.blob) : 0ull;
+ }
+ }
+ std::vector<float> x((size_t) T * D), sh((size_t) T * D), cpu((size_t) T * D);
+ for (auto& v : x) v = frand();
+ for (auto& v : sh) v = frand(0.1f);
+ for (auto& v : cpu) v = frand(0.1f);
+ std::vector<int> cflag(T, 0);
+ cflag[1] = 1;
+ float *dx = dev(x), *dsh = dev(sh), *dcpu = dev(cpu), *dpw = dev(pw);
+ unsigned long long* dpp = dev(pp);
+ int* dflag = dev(cflag);
+ std::vector<double> want((size_t) T * D, 0.0);
+ for (int t = 0; t < T; ++t) {
+ const std::vector<float> xr = q8_round(x.data() + (size_t) t * D, D);
+ for (int i = 0; i < k; ++i) {
+ if (pp[t * 8 + i] == 0ull) continue;
+ const int e = pick[t * 8 + i];
+ std::vector<float> h(F);
+ for (int f = 0; f < F; ++f) h[f] = (float) swiglu(E.dot(e, 0, f, xr.data()), E.dot(e, 1, f, xr.data()), lim);
+ const std::vector<float> hr = q8_round(h.data(), F);
+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] += pw[t * 8 + i] * E.dot(e, 2, d, hr.data());
+ }
+ for (int d = 0; d < D; ++d)
+ want[(size_t) t * D + d] += (cflag[t] ? cpu[(size_t) t * D + d] : 0.0f) + sh[(size_t) t * D + d];
+ }
+ std::vector<float> tok((size_t) T * D);
+ float* dh = dev(std::vector<float>((size_t) 8 * F));
+ float* dout = dev(std::vector<float>(D));
+ for (int t = 0; t < T; ++t) {
+ x3::MoeArgs a;
+ a.plan_ptr = dpp + t * 8;
+ a.plan_w = dpw + t * 8;
+ a.cpu_part = dcpu + (size_t) t * D;
+ a.cpu_flag = dflag + t;
+ a.lay = E.lay;
+ a.k = k;
+ a.n_embd = D;
+ a.n_ff = F;
+ a.limit = lim;
+ gg::moe_gate_up(a, dx + (size_t) t * D, dh, s);
+ gg::moe_down(a, dh, dsh + (size_t) t * D, dout, s);
+ CU(cudaStreamSynchronize(s));
+ CU(cudaMemcpy(tok.data() + (size_t) t * D, dout, (size_t) D * 4, cudaMemcpyDeviceToHost));
+ }
+ char what[96];
+ std::snprintf(what, sizeof(what), "%s/%s token experts (D %d, F %d)", tn(tg), tn(td), D, F);
+ expect(what, tok.data(), want.data(), want.size(), 1e-3);
+ // the window
+ std::vector<unsigned long long> gptr;
+ std::vector<int> gstart, gent;
+ for (int x0 = 0; x0 < n_exp; ++x0) {
+ std::vector<int> ents;
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < k; ++i)
+ if (pick[t * 8 + i] == x0 && pp[t * 8 + i] != 0ull) ents.push_back(t * 8 + i);
+ if (ents.empty()) continue;
+ gptr.push_back((unsigned long long) (E.dev + (size_t) x0 * E.blob));
+ gstart.push_back((int) gent.size());
+ for (int e : ents) gent.push_back(e);
+ }
+ gstart.push_back((int) gent.size());
+ std::vector<int> ng(1, (int) gptr.size());
+ unsigned long long* dgp = dev(gptr);
+ int *dgs = dev(gstart), *dge = dev(gent), *dng = dev(ng);
+ float* dwh = dev(std::vector<float>((size_t) T * 8 * F));
+ float* dpart = dev(std::vector<float>((size_t) T * 8 * D));
+ float* dwo = dev(std::vector<float>((size_t) T * D));
+ x3::WinArgs w;
+ w.grp_ptr = dgp;
+ w.grp_start = dgs;
+ w.grp_ent = dge;
+ w.n_groups = dng;
+ w.plan_ptr = dpp;
+ w.plan_w = dpw;
+ w.cpu_part = dcpu;
+ w.cpu_stride = D;
+ w.cpu_flag = dflag;
+ w.x = dx;
+ w.x_stride = D;
+ w.h = dwh;
+ w.part = dpart;
+ w.sh_out = dsh;
+ w.out = dwo;
+ w.out_stride = D;
+ w.lay = E.lay;
+ w.T = T;
+ w.k = k;
+ w.n_embd = D;
+ w.n_ff = F;
+ w.limit = lim;
+ if (!gg::win_experts(w, s)) ++g_fail;
+ CU(cudaStreamSynchronize(s));
+ std::vector<float> win = host(dwo, (size_t) T * D);
+ std::snprintf(what, sizeof(what), " window experts (T %d)", T);
+ expect(what, win.data(), want.data(), want.size(), 1e-3);
+ const bool same = std::memcmp(win.data(), tok.data(), win.size() * 4) == 0;
+ if (!same) ++g_fail;
+ std::printf("%-44s %s\n", " ... bit for bit the token path's", same ? "ok" : "FAIL");
+ for (int R : {90, 200, 400}) { // the prompt path's set: experts of a few rows (the light kernels), then of more
+ const int NT = 30;
+ std::vector<int> owner(R), idx(R);
+ for (int r = 0; r < R; ++r) owner[r] = (int) (rng() % n_exp);
+ std::sort(owner.begin(), owner.end());
+ std::vector<int> bounds(n_exp + 1, 0);
+ for (int e = 0, r = 0; e <= n_exp; ++e) {
+ while (r < R && owner[r] < e) ++r;
+ bounds[e] = r;
+ }
+ int max_rows = 0;
+ for (int e = 0; e < n_exp; ++e) max_rows = std::max(max_rows, bounds[e + 1] - bounds[e]);
+ std::vector<float> xs((size_t) NT * D);
+ for (auto& v : xs) v = frand();
+ for (int r = 0; r < R; ++r) idx[r] = (int) (rng() % NT);
+ float* dxs = dev(xs);
+ int *dbd = dev(bounds), *didx = dev(idx);
+ uint16_t* dh16 = dev(std::vector<uint16_t>((size_t) R * F));
+ float* dy = dev(std::vector<float>((size_t) R * D));
+ x3::SetArgs sa;
+ sa.wbase = E.dev;
+ sa.stride = E.blob;
+ sa.lay = E.lay;
+ sa.n_exp = n_exp;
+ sa.bounds = dbd;
+ sa.row0 = 0;
+ sa.max_rows = max_rows;
+ sa.n_embd = D;
+ sa.n_ff = F;
+ sa.limit = lim;
+ gg::set_gate_up(sa, dxs, D, didx, dh16, F, s);
+ gg::set_down(sa, dh16, F, dy, D, s);
+ CU(cudaStreamSynchronize(s));
+ std::vector<uint16_t> h16 = host(dh16, (size_t) R * F);
+ std::vector<float> y = host(dy, (size_t) R * D);
+ std::vector<float> hgot((size_t) R * F);
+ std::vector<double> hwant((size_t) R * F), ywant((size_t) R * D);
+ // (the light kernels, experts of <= 16 rows: the inputs rounded to int8 per 32 as the token path rounds them; the
+ // GEMM tiles: F16 inputs and weights - a low-bit weight is not always an F16 value)
+ int n_light = 0;
+ for (int e = 0; e < n_exp; ++e) n_light += bounds[e + 1] - bounds[e] <= 16 ? bounds[e + 1] - bounds[e] : 0;
+ for (int r = 0; r < R; ++r) {
+ const bool light = bounds[owner[r] + 1] - bounds[owner[r]] <= 16;
+ const std::vector<float> xq = light ? q8_round(xs.data() + (size_t) idx[r] * D, D) : std::vector<float>();
+ const float* xr = light ? xq.data() : xs.data() + (size_t) idx[r] * D;
+ std::vector<float> hr(F);
+ for (int f = 0; f < F; ++f) {
+ hwant[(size_t) r * F + f] = swiglu(E.dot(owner[r], 0, f, xr), E.dot(owner[r], 1, f, xr), lim);
+ hgot[(size_t) r * F + f] = __half2float(*(const __half*) &h16[(size_t) r * F + f]);
+ hr[f] = hgot[(size_t) r * F + f];
+ }
+ if (light) hr = q8_round(hr.data(), F);
+ for (int d = 0; d < D; ++d) ywant[(size_t) r * D + d] = E.dot(owner[r], 2, d, hr.data());
+ }
+ char sw[64];
+ std::snprintf(sw, sizeof(sw), " set gate/up (F16 h), %d rows, %d light", R, n_light);
+ expect(sw, hgot.data(), hwant.data(), hwant.size(), n_light == R ? 2e-3 : 8e-3); // (IQ4_XS: the widest weights in F16)
+ std::snprintf(sw, sizeof(sw), " set down, %d rows, %d light", R, n_light);
+ expect(sw, y.data(), ywant.data(), ywant.size(), n_light == R ? 1e-4 : 2e-3);
+ cudaFree(dxs), cudaFree(dbd), cudaFree(didx), cudaFree(dh16), cudaFree(dy);
+ }
+ { // the CPU lane: 3 tokens of expert 4, gate / up rows [7, 107) then all its down rows from the CPU's own h
+ gg::IqCpu fm;
+ std::string err;
+ if (!gg::iq_cpu_fmt(tg, td, D, F, fm, err)) {
+ std::printf("%-44s FAIL %s\n", " CPU lane", err.c_str());
+ ++g_fail;
+ } else {
+ const int n = 3, e = 4, r0 = 7, r1 = std::min(107, F);
+ std::vector<std::vector<uint8_t>> aq(n, std::vector<uint8_t>(fm.act_bytes)),
+ hq(n, std::vector<uint8_t>(fm.h_bytes));
+ std::vector<std::vector<float>> ff(n, std::vector<float>(F, 0.0f)), yo(n, std::vector<float>(D, 0.0f));
+ const void* ap[n];
+ const void* hp[n];
+ float* fp[n];
+ float* yp[n];
+ for (int i = 0; i < n; ++i) {
+ gg::iq_cpu_quant(fm.gu_act, x.data() + (size_t) i * D, D, aq[i].data());
+ ap[i] = aq[i].data();
+ fp[i] = ff[i].data();
+ }
+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, r0, r1, lim);
+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, 0, r0, lim);
+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, r1, F, lim);
+ for (int i = 0; i < n; ++i) {
+ gg::iq_cpu_quant(fm.d_act, ff[i].data(), F, hq[i].data());
+ hp[i] = hq[i].data();
+ yp[i] = yo[i].data();
+ }
+ gg::iq_cpu_down(fm, E.mat(e, 2), hp, n, yp, 0, D);
+ // the reference on the Q8_K values the CPU took (block_q8_K: F32 d, int8 qs[256], int16 bsums[16])
+ auto q8k = [](const std::vector<uint8_t>& q, int kk) {
+ std::vector<float> v(kk);
+ for (int b = 0; b < kk / 256; ++b) {
+ const uint8_t* p = q.data() + (size_t) b * 292;
+ float d;
+ std::memcpy(&d, p, 4);
+ for (int j = 0; j < 256; ++j) v[b * 256 + j] = d * (float) (int8_t) p[4 + j];
+ }
+ return v;
+ };
+ const bool k8 = fm.gu_act == GGML_TYPE_Q8_K && fm.d_act == GGML_TYPE_Q8_K;
+ std::vector<float> got_h, got_y;
+ std::vector<double> want_h, want_y;
+ for (int i = 0; i < n && k8; ++i) {
+ const std::vector<float> xa = q8k(aq[i], D), ha = q8k(hq[i], F);
+ for (int f = 0; f < F; ++f) {
+ got_h.push_back(ff[i][f]);
+ want_h.push_back(swiglu(E.dot(e, 0, f, xa.data()), E.dot(e, 1, f, xa.data()), lim));
+ }
+ for (int d = 0; d < D; ++d) {
+ got_y.push_back(yo[i][d]);
+ want_y.push_back(E.dot(e, 2, d, ha.data()));
+ }
+ }
+ if (!k8) {
+ std::printf("%-44s FAIL activation types %d / %d\n", " CPU lane", fm.gu_act, fm.d_act);
+ ++g_fail;
+ } else {
+ expect(" CPU lane gate/up (Q8_K)", got_h.data(), want_h.data(), want_h.size(), 1e-4);
+ expect(" CPU lane down (Q8_K)", got_y.data(), want_y.data(), want_y.size(), 1e-4);
+ }
+ }
+ }
+ cudaFree(E.dev), cudaFree(dx), cudaFree(dsh), cudaFree(dcpu), cudaFree(dpw), cudaFree(dpp), cudaFree(dflag);
+ cudaFree(dh), cudaFree(dout), cudaFree(dgp), cudaFree(dgs), cudaFree(dge), cudaFree(dng), cudaFree(dwh);
+ cudaFree(dpart), cudaFree(dwo);
+}
+
+// the role pairs the tests take: every type as gate / up and as down
+const std::pair<int, int> kQPairs[] = {
+ {gg::iq::kIQ2_XXS, gg::iq::kIQ3_XXS}, {gg::iq::kIQ2_S, gg::iq::kIQ3_S}, {gg::iq::kIQ3_XXS, gg::iq::kIQ3_S},
+ {gg::iq::kIQ4_XS, gg::iq::kQ4_K}, {gg::iq::kQ2_K, gg::iq::kQ3_K}, {gg::iq::kQ4_K, gg::iq::kIQ4_XS},
+ {gg::iq::kQ3_K, gg::iq::kQ2_K}, {gg::iq::kIQ3_S, gg::iq::kIQ2_XXS}, {gg::iq::kIQ3_S, gg::iq::kIQ2_S}};
+
+// a token's 6 low-bit experts at full size: GB/s of the weights read
+void bench_q(cudaStream_t s) {
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ const int D = 5120, F = 2304;
+ for (const auto& pr : kQPairs) {
+ QExperts E = make_q(D, F, 6, pr.first, pr.second);
+ std::vector<unsigned long long> pp(8, 0ull);
+ std::vector<float> pw(8, 0.1f);
+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);
+ unsigned long long* dpp = dev(pp);
+ float* dpw = dev(pw);
+ float* dx = dev(std::vector<float>(D, 0.01f));
+ float* dh = dev(std::vector<float>((size_t) 8 * F));
+ float* dout = dev(std::vector<float>(D));
+ x3::MoeArgs ma;
+ ma.plan_ptr = dpp;
+ ma.plan_w = dpw;
+ ma.lay = E.lay;
+ ma.k = 6;
+ ma.n_embd = D;
+ ma.n_ff = F;
+ ma.limit = 10.0f;
+ const double gu = 6.0 * 2 * F * E.rb(0), dn = 6.0 * D * E.rb(2);
+ auto timed = [&](const char* what, double bytes, auto&& fn) {
+ fn();
+ CU(cudaStreamSynchronize(s));
+ cudaEventRecord(a, s);
+ for (int i = 0; i < 50; ++i) fn();
+ cudaEventRecord(b, s);
+ CU(cudaEventSynchronize(b));
+ float ms = 0;
+ cudaEventElapsedTime(&ms, a, b);
+ std::printf("bench %-40s %8.1f us %6.0f GB/s\n", what, ms * 1000 / 50, bytes / (ms / 50 * 1e-3) / 1e9);
+ };
+ char what[96];
+ std::snprintf(what, sizeof(what), "6 %s/%s experts (gate/up + down)", tn(pr.first), tn(pr.second));
+ timed(what, gu + dn, [&] {
+ gg::moe_gate_up(ma, dx, dh, s);
+ gg::moe_down(ma, dh, nullptr, dout, s);
+ });
+ timed(" gate/up alone", gu, [&] { gg::moe_gate_up(ma, dx, dh, s); });
+ timed(" down + combine alone", dn, [&] { gg::moe_down(ma, dh, nullptr, dout, s); });
+ cudaFree(E.dev), cudaFree(dpp), cudaFree(dpw), cudaFree(dx), cudaFree(dh), cudaFree(dout);
+ }
+ // the prompt path's sets: 32 experts x R rows (128: an 8K chunk's share of 384 experts; 8: a short prompt's - the
+ // weights' bandwidth), the MXFP4 ones first
+ for (int R : {128, 8}) {
+ const int NE = 32;
+ for (int pi = -1; pi < (int) (sizeof(kQPairs) / sizeof(kQPairs[0])); ++pi) {
+ x3::ExpertLayout lay;
+ size_t blob;
+ double wbytes;
+ uint8_t* wdev;
+ char what[96];
+ if (pi < 0) {
+ Experts M = make_experts(D, F, NE);
+ lay = M.lay, blob = M.blob, wdev = M.dev;
+ wbytes = 3.0 * gg::mx_bytes(F, D);
+ std::snprintf(what, sizeof(what), "set MXFP4, %d experts x %d rows", NE, R);
+ } else {
+ QExperts E = make_q(D, F, NE, kQPairs[pi].first, kQPairs[pi].second);
+ lay = E.lay, blob = E.blob, wdev = E.dev;
+ wbytes = 2.0 * F * E.rb(0) + (double) D * E.rb(2);
+ std::snprintf(what, sizeof(what), "set %s/%s, %d experts x %d rows", tn(kQPairs[pi].first),
+ tn(kQPairs[pi].second), NE, R);
+ }
+ std::vector<int> bounds(NE + 1), idx(NE * R);
+ for (int e = 0; e <= NE; ++e) bounds[e] = e * R;
+ for (int i = 0; i < NE * R; ++i) idx[i] = i;
+ int *db = dev(bounds), *di = dev(idx);
+ float* xs = dev(std::vector<float>((size_t) NE * R * D, 0.01f));
+ uint16_t* h16 = dev(std::vector<uint16_t>((size_t) NE * R * F));
+ float* y = dev(std::vector<float>((size_t) NE * R * D));
+ x3::SetArgs sa;
+ sa.wbase = wdev;
+ sa.stride = blob;
+ sa.lay = lay;
+ sa.n_exp = NE;
+ sa.bounds = db;
+ sa.max_rows = R;
+ sa.n_embd = D;
+ sa.n_ff = F;
+ sa.limit = 10.0f;
+ auto one = [&] {
+ gg::set_gate_up(sa, xs, D, di, h16, F, s);
+ gg::set_down(sa, h16, F, y, D, s);
+ };
+ one();
+ CU(cudaStreamSynchronize(s));
+ cudaEventRecord(a, s);
+ for (int i = 0; i < 5; ++i) one();
+ cudaEventRecord(b, s);
+ CU(cudaEventSynchronize(b));
+ float ms = 0;
+ cudaEventElapsedTime(&ms, a, b);
+ ms /= 5;
+ std::printf("bench %-40s %8.2f ms %6.1f TFLOP/s %6.0f GB/s of weights\n", what, ms,
+ 2.0 * NE * R * 3.0 * D * F / (ms * 1e-3) / 1e12, NE * wbytes / (ms * 1e-3) / 1e9);
+ cudaFree(wdev), cudaFree(db), cudaFree(di), cudaFree(xs), cudaFree(h16), cudaFree(y);
+ }
+ }
+}
+
+// the CPU lane's speed: one full-size expert (gate / up rows, then down rows) for 1 and 4 tokens, on `nth` threads (rows
+// split between them), GB/s of weights read - MXFP4 (mx_cpu_rows) and the low-bit pairs (ggml-cpu's rows)
+void bench_cpu(int nth) {
+ const int D = 5120, F = 2304, reps = 6;
+ auto run = [&](const char* what, double bytes, const std::function<void(int, int)>& rows, int n_rows) {
+ auto pass = [&] {
+ std::vector<std::thread> th;
+ for (int t = 0; t < nth; ++t)
+ th.emplace_back([&, t] { rows(n_rows * t / nth, n_rows * (t + 1) / nth); });
+ for (auto& x : th) x.join();
+ };
+ pass();
+ const auto t0 = std::chrono::steady_clock::now();
+ for (int i = 0; i < reps; ++i) pass();
+ const double s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() / reps;
+ std::printf("cpu %-46s %8.2f ms %6.1f GB/s (%d threads)\n", what, s * 1e3, bytes / s / 1e9, nth);
+ };
+ std::vector<float> x((size_t) 4 * D), h((size_t) 4 * F);
+ for (auto& v : x) v = frand();
+ for (auto& v : h) v = frand(0.2f);
+ for (int n : {1, 4}) {
+ Experts M = make_experts(D, F, 1);
+ const float* xp[4] = {x.data(), x.data() + D, x.data() + 2 * D, x.data() + 3 * D};
+ const float* hp[4] = {h.data(), h.data() + F, h.data() + 2 * F, h.data() + 3 * F};
+ std::vector<float> yo((size_t) 4 * D), yg((size_t) 4 * F);
+ char what[96];
+ std::snprintf(what, sizeof(what), "MXFP4 expert, %d token%s", n, n > 1 ? "s" : "");
+ run(what, 2.0 * gg::mx_bytes(F, D), [&](int r0, int r1) {
+ float* yp[4] = {yg.data(), yg.data() + F, yg.data() + 2 * F, yg.data() + 3 * F};
+ for (int i = 0; i < n; ++i) yp[i] += r0;
+ gg::mx_cpu_rows(emat(M, 0, 0), gg::mx_row_bytes(D), D, r0, r1, xp, n, yp);
+ gg::mx_cpu_rows(emat(M, 0, 1), gg::mx_row_bytes(D), D, r0, r1, xp, n, yp);
+ }, F);
+ run(" down", (double) gg::mx_bytes(D, F), [&](int r0, int r1) {
+ float* yp[4] = {yo.data() + r0, yo.data() + D + r0, yo.data() + 2 * D + r0, yo.data() + 3 * D + r0};
+ gg::mx_cpu_rows(emat(M, 0, 2), gg::mx_row_bytes(F), F, r0, r1, hp, n, yp);
+ }, D);
+ cudaFree(M.dev);
+ for (const auto& pr : kQPairs) {
+ QExperts E = make_q(D, F, 1, pr.first, pr.second);
+ gg::IqCpu fm;
+ std::string err;
+ if (!gg::iq_cpu_fmt(pr.first, pr.second, D, F, fm, err)) continue;
+ std::vector<uint8_t> aq((size_t) 4 * fm.act_bytes), hq((size_t) 4 * fm.h_bytes);
+ const void* ap[4];
+ const void* hqp[4];
+ for (int i = 0; i < 4; ++i) {
+ gg::iq_cpu_quant(fm.gu_act, xp[i], D, aq.data() + i * fm.act_bytes);
+ gg::iq_cpu_quant(fm.d_act, hp[i], F, hq.data() + i * fm.h_bytes);
+ ap[i] = aq.data() + i * fm.act_bytes;
+ hqp[i] = hq.data() + i * fm.h_bytes;
+ }
+ std::snprintf(what, sizeof(what), "%s/%s expert, %d token%s", tn(pr.first), tn(pr.second), n, n > 1 ? "s" : "");
+ run(what, 2.0 * F * E.rb(0), [&](int r0, int r1) {
+ float* fp[4] = {yg.data(), yg.data() + F, yg.data() + 2 * F, yg.data() + 3 * F};
+ gg::iq_cpu_gate_up(fm, E.mat(0, 0), E.mat(0, 1), ap, n, fp, r0, r1, 10.0f);
+ }, F);
+ run(" down", (double) D * E.rb(2), [&](int r0, int r1) {
+ float* yp[4] = {yo.data(), yo.data() + D, yo.data() + 2 * D, yo.data() + 3 * D};
+ gg::iq_cpu_down(fm, E.mat(0, 2), hqp, n, yp, r0, r1);
+ }, D);
+ cudaFree(E.dev);
+ }
+ }
+}
+
+// the decode kernels' speed: GB/s of weights read (a token's dense GEMVs and its experts)
+void bench(cudaStream_t s) {
+ cudaEvent_t a, b;
+ cudaEventCreate(&a);
+ cudaEventCreate(&b);
+ auto timed = [&](const char* what, double bytes, auto&& fn) {
+ fn();
+ CU(cudaStreamSynchronize(s));
+ cudaEventRecord(a, s);
+ const int reps = 50;
+ for (int i = 0; i < reps; ++i) fn();
+ cudaEventRecord(b, s);
+ CU(cudaEventSynchronize(b));
+ float ms = 0;
+ cudaEventElapsedTime(&ms, a, b);
+ std::printf("bench %-40s %8.1f us %6.0f GB/s\n", what, ms * 1000 / reps, bytes / (ms / reps * 1e-3) / 1e9);
+ };
+ const std::pair<int, int> shapes[] = {{1280, 32768}, {5120, 1280}, {8192, 5120}, {5120, 129280}};
+ for (int fmt : {gg::kFmtBF16, gg::kFmtQ8_0})
+ for (const auto& kn : shapes) {
+ const int k = kn.first, n = kn.second;
+ const size_t bytes = fmt == gg::kFmtBF16 ? (size_t) n * k * 2 : gg::q8_row_bytes(k) * n;
+ uint8_t* dw;
+ CU(cudaMalloc(&dw, bytes));
+ CU(cudaMemset(dw, 0, bytes));
+ float *dx, *dy;
+ CU(cudaMalloc(&dx, (size_t) k * 4));
+ CU(cudaMemset(dx, 0, (size_t) k * 4));
+ CU(cudaMalloc(&dy, (size_t) n * 4));
+ x3::Mat m;
+ m.fmt = fmt;
+ m.raw = dw;
+ m.k = k;
+ m.n = n;
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = dx;
+ j.y = dy;
+ char what[64];
+ std::snprintf(what, sizeof(what), "mv %s %dx%d", fmt == gg::kFmtBF16 ? "BF16" : "Q8_0", n, k);
+ timed(what, (double) bytes, [&] { gg::mv(&j, 1, s); });
+ cudaFree(dw), cudaFree(dx), cudaFree(dy);
+ }
+ { // a token's 6 experts at full size
+ const int D = 5120, F = 2304;
+ Experts E = make_experts(D, F, 6);
+ std::vector<unsigned long long> pp(8, 0ull);
+ std::vector<float> pw(8, 0.1f);
+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);
+ unsigned long long* dpp = dev(pp);
+ float* dpw = dev(pw);
+ float* dx = dev(std::vector<float>(D, 0.01f));
+ float* dh = dev(std::vector<float>((size_t) 8 * F));
+ float* dout = dev(std::vector<float>(D));
+ x3::MoeArgs ma;
+ ma.plan_ptr = dpp;
+ ma.plan_w = dpw;
+ ma.lay = E.lay;
+ ma.k = 6;
+ ma.n_embd = D;
+ ma.n_ff = F;
+ ma.limit = 10.0f;
+ timed("6 MXFP4 experts (gate/up + down)", 6.0 * (2 * gg::mx_bytes(F, D) + gg::mx_bytes(D, F)), [&] {
+ gg::moe_gate_up(ma, dx, dh, s);
+ gg::moe_down(ma, dh, nullptr, dout, s);
+ });
+ cudaFree(E.dev);
+ { // the prompt path: 32 experts x 128 rows each (an 8K chunk's share of 384 experts)
+ const int NE = 32, R = 128;
+ Experts S = make_experts(D, F, NE);
+ std::vector<int> bounds(NE + 1), idx(NE * R);
+ for (int e = 0; e <= NE; ++e) bounds[e] = e * R;
+ for (int i = 0; i < NE * R; ++i) idx[i] = i;
+ int *db = dev(bounds), *di = dev(idx);
+ float* xs = dev(std::vector<float>((size_t) NE * R * D, 0.01f));
+ uint16_t* h16 = dev(std::vector<uint16_t>((size_t) NE * R * F));
+ float* y = dev(std::vector<float>((size_t) NE * R * D));
+ x3::SetArgs sa;
+ sa.wbase = S.dev;
+ sa.stride = S.blob;
+ sa.lay = S.lay;
+ sa.n_exp = NE;
+ sa.bounds = db;
+ sa.max_rows = R;
+ sa.n_embd = D;
+ sa.n_ff = F;
+ sa.limit = 10.0f;
+ const double flops = 2.0 * NE * R * 3.0 * D * F;
+ cudaEvent_t a0, a1;
+ cudaEventCreate(&a0);
+ cudaEventCreate(&a1);
+ gg::set_gate_up(sa, xs, D, di, h16, F, s);
+ gg::set_down(sa, h16, F, y, D, s);
+ cudaEventRecord(a0, s);
+ for (int i = 0; i < 5; ++i) {
+ gg::set_gate_up(sa, xs, D, di, h16, F, s);
+ gg::set_down(sa, h16, F, y, D, s);
+ }
+ cudaEventRecord(a1, s);
+ CU(cudaEventSynchronize(a1));
+ float ms = 0;
+ cudaEventElapsedTime(&ms, a0, a1);
+ std::printf("bench %-40s %8.1f ms %6.1f TFLOP/s\n", "prompt set: 32 experts x 128 rows", ms / 5,
+ flops / (ms / 5 * 1e-3) / 1e12);
+ cudaFree(S.dev), cudaFree(db), cudaFree(di), cudaFree(xs), cudaFree(h16), cudaFree(y);
+ }
+ E.dev = nullptr;
+ E = make_experts(D, F, 6);
+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);
+ CU(cudaMemcpy(dpp, pp.data(), pp.size() * 8, cudaMemcpyHostToDevice));
+ timed(" gate/up alone", 6.0 * 2 * gg::mx_bytes(F, D), [&] { gg::moe_gate_up(ma, dx, dh, s); });
+ timed(" down + combine alone", 6.0 * gg::mx_bytes(D, F), [&] { gg::moe_down(ma, dh, nullptr, dout, s); });
+ cudaFree(E.dev), cudaFree(dpp), cudaFree(dpw), cudaFree(dx), cudaFree(dh), cudaFree(dout);
+ }
+}
+
+// ---- the caches' packed rows (ds4_kernels.cu): DeepSeek's FP8 / FP4 roundings kept as codes and scales. Each packed
+// row against the BF16 row the same finish writes - a host decode of every value bit for bit - then the readers (the
+// window ring and its rollback, the sparse attention over the compressed KV, the indexer's scores) the same bits from
+// either, at full size (a 64K context's compressed rows)
+namespace dk = strata::ds4;
+
+float h_e2m1(uint32_t c) {
+ static const float m[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f};
+ const float v = m[c & 7u];
+ return (c & 8u) ? -v : v;
+}
+float h_e4m3(uint32_t q) {
+ const int e = (int) ((q >> 3) & 15u), m = (int) (q & 7u);
+ const float v = e == 0 ? std::ldexp((float) m, -9) : std::ldexp(1.0f + (float) m / 8.0f, e - 7);
+ return (q & 0x80u) ? -v : v;
+}
+float h_ue8m0(uint32_t b) {
+ const uint32_t u = b << 23;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+// value i of a packed row of d values
+float h_unpack(int fmt, const uint8_t* row, int d, int i) {
+ if (fmt == dk::kKvFp8) return h_e4m3(row[i]) * h_ue8m0(row[d + i / 32]);
+ const uint32_t c = (uint32_t) row[i / 2] >> (4 * (i & 1));
+ return h_e2m1(c) * (fmt == dk::kKvFp4 ? h_e4m3(row[d / 2 + i / 16]) : h_ue8m0(row[d / 2 + i / 32]));
+}
+
+// n rows of d values: groups of 16 at scales 2^-30 .. 2^10, some all zero, some with negative zeros, some tiny
+std::vector<float> kv_rows(int n, int d) {
+ std::vector<float> x((size_t) n * d);
+ for (int r = 0; r < n; ++r)
+ for (int g0 = 0; g0 < d; g0 += 16) {
+ const int kind = (int) (rng() % 16u);
+ const float sc = std::ldexp(1.0f, (int) (rng() % 41u) - 30);
+ for (int e = g0; e < g0 + 16; ++e) {
+ float v = frand(sc);
+ if (kind == 0) v = 0.0f;
+ else if (kind == 1 && e % 3 == 0) v = -0.0f;
+ else if (kind == 2) v = frand(1e-7f);
+ else if (kind == 3 && e == g0) v = sc * 1000.0f; // (one value far above the rest)
+ x[(size_t) r * d + e] = v;
+ }
+ }
+ return x;
+}
+
+// finish(quant) of n rows of d: the BF16 rows (dst) and the packed rows (pdst), both from the same input
+void kv_finish(const float* x, int n, int d, int quant, uint16_t* bf, uint8_t* pk, cudaStream_t s) {
+ dk::Finish f;
+ f.quant = quant;
+ f.dst = bf;
+ f.dst_stride = d;
+ dk::finish(const_cast<float*>(x), n, d, d, f, s);
+ dk::Finish p;
+ p.quant = quant;
+ p.pdst = pk;
+ p.pdst_stride = (long long) dk::kv_row_bytes(quant, d);
+ dk::finish(const_cast<float*>(x), n, d, d, p, s);
+}
+
+// every packed value decoded on the host against the BF16 row's bits
+void kv_check_rows(const char* what, int fmt, const std::vector<uint16_t>& bf, const std::vector<uint8_t>& pk, int n,
+ int d) {
+ const size_t rb = dk::kv_row_bytes(fmt, d);
+ size_t bad = 0, at = 0;
+ for (int r = 0; r < n; ++r)
+ for (int i = 0; i < d; ++i) {
+ const float v = h_unpack(fmt, pk.data() + (size_t) r * rb, d, i);
+ uint32_t u;
+ std::memcpy(&u, &v, 4);
+ if ((u & 0xffffu) != 0u || (uint16_t) (u >> 16) != bf[(size_t) r * d + i]) {
+ if (bad++ == 0) at = (size_t) r * d + i;
+ }
+ }
+ if (bad) ++g_fail;
+ std::printf("%-44s %s %d rows x %d, %zu of %zu values differ (first at %zu), %zu B a row vs %d\n", what,
+ bad ? "FAIL" : "ok ", n, d, bad, (size_t) n * d, at, rb, d * 2);
+}
+
+void kv_same(const char* what, const void* a, const void* b, size_t bytes) {
+ const bool ok = std::memcmp(a, b, bytes) == 0;
+ if (!ok) ++g_fail;
+ std::printf("%-44s %s %zu bytes %s\n", what, ok ? "ok " : "FAIL", bytes, ok ? "identical" : "DIFFER");
+}
+
+void test_kv_pack(cudaStream_t s) {
+ const int HD = 512, IX = 128, NC = 65536 / 4 + 1, WIN = 128;
+ // the compressed KV (FP4, an E4M3 scale per 16) and the index keys (FP4, a power-of-two scale per 32)
+ std::vector<float> xc = kv_rows(NC, HD), xi = kv_rows(NC, IX);
+ float *d_xc = dev(xc), *d_xi = dev(xi);
+ uint16_t *c16 = nullptr, *i16 = nullptr;
+ uint8_t *c4 = nullptr, *i4 = nullptr;
+ CU(cudaMalloc(&c16, (size_t) NC * HD * 2));
+ CU(cudaMalloc(&i16, (size_t) NC * IX * 2));
+ CU(cudaMalloc(&c4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4, HD)));
+ CU(cudaMalloc(&i4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4p, IX)));
+ kv_finish(d_xc, NC, HD, dk::kKvFp4, c16, c4, s);
+ kv_finish(d_xi, NC, IX, dk::kKvFp4p, i16, i4, s);
+ CU(cudaStreamSynchronize(s));
+ kv_check_rows("kv pack: compressed KV (FP4 + E4M3 / 16)", dk::kKvFp4, host(c16, (size_t) NC * HD),
+ host(c4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4, HD)), NC, HD);
+ kv_check_rows("kv pack: index keys (FP4 + 2^k / 32)", dk::kKvFp4p, host(i16, (size_t) NC * IX),
+ host(i4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4p, IX)), NC, IX);
+ kv_same("kv pack: finish leaves its input", xc.data(), host(d_xc, xc.size()).data(), xc.size() * 4);
+
+ // the window ring (FP8, a power-of-two scale per 32): batches of 128 rows through the finish's BF16 rows, into the
+ // ring (its scales derived from the BF16 values) and back out
+ {
+ const size_t rb = dk::kv_row_bytes(dk::kKvFp8, HD);
+ std::vector<float> xw = kv_rows(WIN, HD);
+ float* d_xw = dev(xw);
+ uint16_t *ext = nullptr, *ext2 = nullptr;
+ uint8_t *ring = nullptr, *pk = nullptr, *bak = nullptr;
+ CU(cudaMalloc(&ext, (size_t) 2 * WIN * HD * 2));
+ CU(cudaMalloc(&ext2, (size_t) WIN * HD * 2));
+ CU(cudaMalloc(&ring, (size_t) WIN * rb));
+ CU(cudaMalloc(&pk, (size_t) WIN * rb));
+ CU(cudaMalloc(&bak, (size_t) dk::kWin * rb));
+ size_t bad = 0, bad_pk = 0, n_vals = 0;
+ for (int b = 0; b < 64; ++b) {
+ if (b > 0) {
+ xw = kv_rows(WIN, HD);
+ CU(cudaMemcpy(d_xw, xw.data(), xw.size() * 4, cudaMemcpyHostToDevice));
+ }
+ kv_finish(d_xw, WIN, HD, dk::kKvFp8, ext + (size_t) WIN * HD, pk, s);
+ dk::ext_to_ring(ext, ring, 0, WIN, WIN, HD, s);
+ dk::ring_to_ext(ring, ext2, WIN, WIN, HD, s); // (p0 = n: the rows as they lie)
+ CU(cudaStreamSynchronize(s));
+ const std::vector<uint16_t> want = host(ext + (size_t) WIN * HD, (size_t) WIN * HD),
+ got = host(ext2, (size_t) WIN * HD);
+ for (size_t i = 0; i < want.size(); ++i) bad += want[i] != got[i];
+ const std::vector<uint8_t> hr = host(ring, (size_t) WIN * rb), hp = host(pk, (size_t) WIN * rb);
+ for (int r = 0; r < WIN; ++r) // the ring's codes against the finish's own packing: the same values
+ for (int i = 0; i < HD; ++i)
+ bad_pk += h_unpack(dk::kKvFp8, hr.data() + (size_t) r * rb, HD, i) !=
+ h_unpack(dk::kKvFp8, hp.data() + (size_t) r * rb, HD, i) ||
+ std::signbit(h_unpack(dk::kKvFp8, hr.data() + (size_t) r * rb, HD, i)) !=
+ std::signbit(h_unpack(dk::kKvFp8, hp.data() + (size_t) r * rb, HD, i));
+ n_vals += want.size();
+ }
+ if (bad || bad_pk) ++g_fail;
+ std::printf("%-44s %s %zu values, %zu differ through the ring, %zu from the finish's packing; %zu B a row vs %d\n",
+ "kv pack: window ring (FP8 + 2^k / 32)", bad || bad_pk ? "FAIL" : "ok ", n_vals, bad, bad_pk, rb,
+ HD * 2);
+ // a speculative window's rollback: 16 rows over the ring at position 1000 (backed up), the first 5 kept
+ const std::vector<uint8_t> before = host(ring, (size_t) WIN * rb);
+ xw = kv_rows(dk::kWin, HD);
+ CU(cudaMemcpy(d_xw, xw.data(), (size_t) dk::kWin * HD * 4, cudaMemcpyHostToDevice));
+ kv_finish(d_xw, dk::kWin, HD, dk::kKvFp8, ext + (size_t) WIN * HD, pk, s);
+ dk::ext_to_ring(ext, ring, 1000, WIN, dk::kWin, HD, s, bak);
+ CU(cudaStreamSynchronize(s));
+ std::vector<uint8_t> want = before;
+ const std::vector<uint8_t> after = host(ring, (size_t) WIN * rb);
+ for (int t = 0; t < 5; ++t) {
+ const size_t r = (size_t) ((1000 + t) % WIN);
+ std::memcpy(want.data() + r * rb, after.data() + r * rb, rb);
+ }
+ dk::ring_restore(ring, bak, 1000, WIN, 5, dk::kWin, HD, s);
+ CU(cudaStreamSynchronize(s));
+ kv_same("kv pack: window ring rollback", want.data(), host(ring, (size_t) WIN * rb).data(), want.size());
+ CU(cudaFree(d_xw));
+ CU(cudaFree(ext));
+ CU(cudaFree(ext2));
+ CU(cudaFree(ring));
+ CU(cudaFree(pk));
+ CU(cudaFree(bak));
+ }
+
+ // the sparse attention: the window rows and 512 selected compressed rows each, 64 heads - the BF16 rows' kernel
+ // against the packed rows' every way (decoded once a query into the scratch; staged, every number of heads a
+ // block), and the time of each (a decode step's query, a few, a verification's 16, chunk slabs of 16 and 256)
+ {
+ const int TM = 256, NH = 64, SEL = 512, ST = 16;
+ std::vector<float> q((size_t) TM * NH * HD), sink(NH);
+ for (float& v : q) v = frand(0.2f);
+ for (float& v : sink) v = frand(1.0f);
+ std::vector<int> sel((size_t) TM * SEL);
+ for (int& v : sel) v = (int) (rng() % (unsigned) NC);
+ std::vector<float> xw = kv_rows(WIN + TM, HD);
+ float *d_q = dev(q), *d_sink = dev(sink), *d_xw = dev(xw), *o1 = nullptr, *o2 = nullptr;
+ int* d_sel = dev(sel);
+ uint16_t* ext = nullptr;
+ uint8_t* pk = nullptr;
+ dk::AttnScratch scr;
+ scr.max_t = ST;
+ scr.cap = SEL;
+ CU(cudaMalloc(&scr.rows, (size_t) ST * SEL * HD * 2));
+ CU(cudaMalloc(&ext, (size_t) (WIN + TM) * HD * 2));
+ CU(cudaMalloc(&pk, (size_t) (WIN + TM) * dk::kv_row_bytes(dk::kKvFp8, HD)));
+ CU(cudaMalloc(&o1, q.size() * 4));
+ CU(cudaMalloc(&o2, q.size() * 4));
+ kv_finish(d_xw, WIN + TM, HD, dk::kKvFp8, ext, pk, s);
+ cudaEvent_t e0, e1;
+ CU(cudaEventCreate(&e0));
+ CU(cudaEventCreate(&e1));
+ const float scale = 1.0f / std::sqrt((float) HD);
+ const struct {
+ int T, gen, iters;
+ const char* what;
+ } shapes[] = {{1, 0, 400, "1 query"}, {2, 0, 400, "2 queries"}, {4, 0, 200, "4 queries"},
+ {8, 0, 200, "8 queries"}, {16, 0, 100, "16 queries"}, {16, 1, 100, "a chunk slab of 16"},
+ {256, 1, 10, "a chunk slab of 256"}};
+ for (const auto& sh : shapes) {
+ const int T = sh.T;
+ dk::AttnRows ar;
+ if (sh.gen) { // a chunk's rows at 40000.. (ratio 4: every query its 512)
+ ar.gen = 1;
+ ar.p0 = 40000;
+ ar.t0 = 0;
+ ar.win = WIN;
+ ar.R = 4;
+ ar.kcap = SEL;
+ } else
+ for (int t = 0; t < T; ++t) {
+ ar.win0[t] = t + 1;
+ ar.nwin[t] = WIN;
+ ar.nsel[t] = SEL - 31 * t;
+ }
+ const size_t n = (size_t) T * NH * HD;
+ dk::sparse_attn(d_q, ext, c16, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvBf16);
+ CU(cudaStreamSynchronize(s));
+ const std::vector<float> want = host(o1, n);
+ bool same = true;
+ for (int g : {-1, 0, 1, 2, 4, 8}) {
+ CU(cudaMemset(o2, 0xff, n * 4));
+ dk::sparse_attn(d_q, ext, c4, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o2, s, dk::kKvFp4, g, &scr);
+ CU(cudaStreamSynchronize(s));
+ same = same && std::memcmp(want.data(), host(o2, n).data(), n * 4) == 0;
+ }
+ // the time: BF16 rows; packed as the engine runs it (0), through the scratch (-1), staged (its own G)
+ float us[4];
+ const int ways[4] = {99, 0, -1, -2};
+ for (int w = 0; w < 4; ++w) {
+ CU(cudaEventRecord(e0, s));
+ for (int it = 0; it < sh.iters; ++it) {
+ if (ways[w] == 99)
+ dk::sparse_attn(d_q, ext, c16, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvBf16);
+ else
+ dk::sparse_attn(d_q, ext, c4, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvFp4,
+ ways[w] == -2 ? 0 : ways[w], ways[w] == -2 ? nullptr : &scr);
+ }
+ CU(cudaEventRecord(e1, s));
+ CU(cudaEventSynchronize(e1));
+ float ms = 0.0f;
+ CU(cudaEventElapsedTime(&ms, e0, e1));
+ us[w] = ms * 1000.0f / sh.iters;
+ }
+ if (!same) ++g_fail;
+ char what[96];
+ std::snprintf(what, sizeof what, "kv pack: sparse attention, %s", sh.what);
+ std::printf("%-44s %s %s every way; BF16 rows %.1f us a call, packed %.1f (decoded once a query %.1f, "
+ "staged %.1f)\n",
+ what, same ? "ok " : "FAIL", same ? "identical" : "DIFFER", us[0], us[1], us[2], us[3]);
+ }
+ CU(cudaEventDestroy(e0));
+ CU(cudaEventDestroy(e1));
+ CU(cudaFree(d_q));
+ CU(cudaFree(d_sink));
+ CU(cudaFree(d_xw));
+ CU(cudaFree(d_sel));
+ CU(cudaFree(ext));
+ CU(cudaFree(pk));
+ CU(cudaFree(o1));
+ CU(cudaFree(o2));
+ CU(cudaFree(scr.rows));
+ }
+
+ // the indexer's scores over every key (32 heads), BF16 vs packed keys
+ {
+ const int T = 4, NH = 32;
+ std::vector<float> q((size_t) T * NH * IX), w((size_t) T * 64);
+ for (float& v : q) v = frand(1.0f);
+ for (float& v : w) v = frand(0.3f);
+ float *d_q = dev(q), *d_w = dev(w), *o1 = nullptr, *o2 = nullptr;
+ const long long ss = NC + 64;
+ CU(cudaMalloc(&o1, (size_t) T * ss * 4));
+ CU(cudaMalloc(&o2, (size_t) T * ss * 4));
+ CU(cudaMemset(o1, 0, (size_t) T * ss * 4));
+ CU(cudaMemset(o2, 0, (size_t) T * ss * 4));
+ dk::Rows lim;
+ for (int t = 0; t < T; ++t) {
+ lim.n[t] = NC - 3 * t;
+ lim.k[t] = 512;
+ }
+ dk::index_score(d_q, d_w, i16, NH, IX, T, NC, nullptr, 0, 1, lim, o1, ss, s, dk::kKvBf16);
+ dk::index_score(d_q, d_w, i4, NH, IX, T, NC, nullptr, 0, 1, lim, o2, ss, s, dk::kKvFp4p);
+ CU(cudaStreamSynchronize(s));
+ kv_same("kv pack: index scores", host(o1, (size_t) T * ss).data(), host(o2, (size_t) T * ss).data(),
+ (size_t) T * ss * 4);
+ { // (the time of each over the 16385 keys, 50 times)
+ cudaEvent_t e0, e1;
+ CU(cudaEventCreate(&e0));
+ CU(cudaEventCreate(&e1));
+ float ms[2];
+ for (int f = 0; f < 2; ++f) {
+ CU(cudaEventRecord(e0, s));
+ for (int it = 0; it < 50; ++it)
+ dk::index_score(d_q, d_w, f ? (const void*) i4 : (const void*) i16, NH, IX, T, NC, nullptr, 0, 1, lim,
+ o1, ss, s, f ? dk::kKvFp4p : dk::kKvBf16);
+ CU(cudaEventRecord(e1, s));
+ CU(cudaEventSynchronize(e1));
+ CU(cudaEventElapsedTime(&ms[f], e0, e1));
+ }
+ std::printf("%-44s BF16 keys %.1f us a call, packed %.1f us\n", "kv pack: index score time", ms[0] * 20.0f,
+ ms[1] * 20.0f);
+ CU(cudaEventDestroy(e0));
+ CU(cudaEventDestroy(e1));
+ }
+ CU(cudaFree(d_q));
+ CU(cudaFree(d_w));
+ CU(cudaFree(o1));
+ CU(cudaFree(o2));
+ }
+ CU(cudaFree(d_xc));
+ CU(cudaFree(d_xi));
+ CU(cudaFree(c16));
+ CU(cudaFree(i16));
+ CU(cudaFree(c4));
+ CU(cudaFree(i4));
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ std::printf("kv pack: %s\n", cudaGetErrorString(e));
+ ++g_fail;
+ }
+}
+
+} // namespace
+
+// ---------------------------------------------------------------- Engram rows in Q6_K
+__global__ void q6k_rows_kernel(const uint8_t* rows, int hd, float* out) { // (engram_dequant's Q6_K branch)
+ const int r = blockIdx.x;
+ for (int e = threadIdx.x; e < hd; e += blockDim.x)
+ out[(size_t) r * hd + e] =
+ strata::ds4::q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * strata::ds4::kQ6kBytes, e % 256);
+}
+void test_engram_q6k(cudaStream_t s) {
+ const int n = 64, hd = 512, sb = hd / 256; // rows of two superblocks, each row hd bytes apart (as the engine keeps)
+ std::vector<uint8_t> rows((size_t) n * hd, 0), packed((size_t) n * sb * 210);
+ for (auto& v : packed) v = (uint8_t) (rng() & 0xff);
+ for (size_t b = 0; b < (size_t) n * sb; ++b) { // d: a small normal F16
+ const __half h = __float2half(0.0005f + 0.002f * std::fabs(frand()));
+ std::memcpy(&packed[b * 210 + 208], &h, 2);
+ }
+ for (int r = 0; r < n; ++r) std::memcpy(&rows[(size_t) r * hd], &packed[(size_t) r * sb * 210], (size_t) sb * 210);
+ std::vector<float> want((size_t) n * hd);
+ ggml_get_type_traits(GGML_TYPE_Q6_K)->to_float(packed.data(), want.data(), (int64_t) n * hd);
+ uint8_t* d_rows = dev(rows);
+ float* d_out = dev(std::vector<float>((size_t) n * hd));
+ q6k_rows_kernel<<<n, 256, 0, s>>>(d_rows, hd, d_out);
+ CU(cudaStreamSynchronize(s));
+ const std::vector<float> got = host(d_out, (size_t) n * hd);
+ const bool same = std::memcmp(got.data(), want.data(), got.size() * 4) == 0;
+ if (!same) ++g_fail;
+ std::printf("%-44s %s\n", "Engram Q6_K rows = ggml's, bit for bit", same ? "ok" : "FAIL");
+ cudaFree(d_rows), cudaFree(d_out);
+}
+
+int main(int argc, char** argv) {
+ cudaStream_t s;
+ CU(cudaStreamCreate(&s));
+ if (!gg::prepare(s)) return 1;
+ if (argc >= 2 && std::strcmp(argv[1], "--bench") == 0) {
+ bench(s);
+ bench_q(s);
+ return 0;
+ }
+ if (argc >= 2 && std::strcmp(argv[1], "--bench-q") == 0) {
+ bench_q(s);
+ return 0;
+ }
+ if (argc >= 2 && std::strcmp(argv[1], "--bench-cpu") == 0) { // [threads]
+ bench_cpu(argc >= 3 ? std::atoi(argv[2]) : 1);
+ return 0;
+ }
+ if (argc >= 2 && std::strcmp(argv[1], "--kv") == 0) { // the caches' packing alone
+ test_kv_pack(s);
+ std::printf("%s (%d failed)\n", g_fail ? "FAILED" : "ALL OK", g_fail);
+ return g_fail ? 1 : 0;
+ }
+ test_kv_pack(s);
+ test_mv(s);
+ test_engram_q6k(s);
+ test_experts(512, 256, s);
+ for (int t : {gg::iq::kIQ2_XXS, gg::iq::kIQ2_S, gg::iq::kIQ3_XXS, gg::iq::kIQ3_S, gg::iq::kIQ4_XS, gg::iq::kQ2_K,
+ gg::iq::kQ3_K, gg::iq::kQ4_K})
+ test_mv_q(t, s);
+ for (const auto& pr : kQPairs) test_experts_q(512, 256, pr.first, pr.second, s);
+ if (argc >= 3) {
+ test_experts(std::atoi(argv[1]), std::atoi(argv[2]), s);
+ for (int i : {0, 4}) test_experts_q(std::atoi(argv[1]), std::atoi(argv[2]), kQPairs[i].first, kQPairs[i].second, s);
+ }
+ if (gg::launch_errors() != 0) {
+ std::printf("launch errors: %d\n", gg::launch_errors());
+ ++g_fail;
+ }
+ std::printf("%s (%d failed)\n", g_fail ? "FAILED" : "ALL OK", g_fail);
+ return g_fail ? 1 : 0;
+}
diff --git a/tests/dsv41/ds4_mx_bench.cpp b/tests/dsv41/ds4_mx_bench.cpp
new file mode 100644
index 000000000..af6b77650
--- /dev/null
+++ b/tests/dsv41/ds4_mx_bench.cpp
@@ -0,0 +1,381 @@
+// tests/ds4_mx_bench.cpp - how fast does this CPU run DeepSeek-V4.1-Flash's MXFP4 experts on the CPU lane?
+//
+// ds4_mx_bench [--threads 1,6,12] [--T 1,2,4,8,16] [--experts 16] [--per-layer 3] [--secs 0.4] [--pf 0]
+// [--kernels float,avx2,avxvnni,avx512,avx512vnni,ggml,scalar] [--file PATH] [--make-file PATH]
+//
+// Experts of the real shape (gate / up 2304 rows x 5120, down 5120 x 2304, MXFP4: 18.8 MB each) with random codes in
+// anonymous memory (huge pages where the OS gives them), or in a file's mapping (--file: 4 KiB page-cache pages, as the
+// engine reads the GGUF; --make-file writes one). A "layer" is --per-layer experts, as the lane runs them: the gate /
+// up rows of every expert in 64-row jobs on the pool, then (a barrier) their down rows; the layers walk through the
+// experts, so the weights come from RAM, not the cache. Prints the weights' GB/s for each kernel x tokens x threads
+// (T tokens a call: each row read once for up to 8 of them), the int8 tiers' agreement with the scalar tier, and the
+// cost of the activation quant and of madvise(WILLNEED) on cached pages.
+#include "../src/ds4/ds4_gguf_cpu.hpp"
+#include "../src/ds4/ds4_pool.hpp"
+
+#if defined(STRATA_NATIVE_EXPERTS)
+#include "ggml-cpu.h"
+#include "ggml.h"
+#endif
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace gg = strata::ds4::gguf;
+namespace os = strata::ds4::os;
+using strata::ds4::Pool;
+
+namespace {
+
+constexpr int E = 5120, FF = 2304, RB = 64;
+const size_t kGr = (size_t) E / 32 * 17, kDr = (size_t) FF / 32 * 17; // row bytes
+const size_t kMat = kGr * FF; // gate (= up = down) bytes
+const size_t kExpert = 3 * kMat;
+
+double now_s() {
+ return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+std::vector<int> ints(const std::string& s) {
+ std::vector<int> v;
+ size_t a = 0;
+ while (a < s.size()) {
+ size_t b = s.find(',', a);
+ if (b == std::string::npos) b = s.size();
+ v.push_back(std::atoi(s.substr(a, b - a).c_str()));
+ a = b + 1;
+ }
+ return v;
+}
+std::vector<std::string> words(const std::string& s) {
+ std::vector<std::string> v;
+ size_t a = 0;
+ while (a < s.size()) {
+ size_t b = s.find(',', a);
+ if (b == std::string::npos) b = s.size();
+ v.push_back(s.substr(a, b - a));
+ a = b + 1;
+ }
+ return v;
+}
+
+// the logical CPUs, one of each core first, then their second threads (pinning order)
+std::vector<int> cpu_order() {
+ std::vector<int> first, rest;
+#if defined(_WIN32)
+ DWORD len = 0;
+ GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len);
+ std::vector<uint8_t> buf(len);
+ if (len > 0 && GetLogicalProcessorInformationEx(RelationProcessorCore,
+ (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len)) {
+ for (size_t o = 0; o < buf.size();) {
+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (buf.data() + o);
+ if (e->Processor.GroupCount > 0 && e->Processor.GroupMask[0].Group == 0) {
+ const KAFFINITY m = e->Processor.GroupMask[0].Mask;
+ bool f = true;
+ for (int b = 0; b < 64; ++b)
+ if ((m >> b) & 1) {
+ (f ? first : rest).push_back(b);
+ f = false;
+ }
+ }
+ o += e->Size;
+ }
+ }
+#else
+ for (int c = 0; c < 1024; ++c) {
+ std::FILE* f = std::fopen(("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list").c_str(), "r");
+ if (f == nullptr) break;
+ int s0 = -1;
+ if (std::fscanf(f, "%d", &s0) != 1) s0 = c;
+ std::fclose(f);
+ (s0 == c ? first : rest).push_back(c);
+ }
+#endif
+ if (first.empty())
+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency()); ++c) first.push_back((int) c);
+ std::sort(first.begin(), first.end());
+ std::sort(rest.begin(), rest.end());
+ first.insert(first.end(), rest.begin(), rest.end());
+ return first;
+}
+
+void fill_mx(uint8_t* p, size_t bytes, uint64_t seed) { // random MXFP4 blocks, scales 2^-14 .. 2^-6 as weights'
+ std::mt19937_64 rng(seed);
+ for (size_t b = 0; b + 17 <= bytes; b += 17) {
+ const uint64_t r = rng(), s = rng();
+ p[b] = (uint8_t) (114 + r % 9);
+ std::memcpy(p + b + 1, &r, 8);
+ std::memcpy(p + b + 9, &s, 8);
+ }
+}
+
+enum Kern { kFloat = 100, kGgml = 101 };
+int kern_id(const std::string& s) {
+ if (s == "float") return kFloat;
+ if (s == "ggml") return kGgml;
+ for (int t = 0; t < gg::kMxTiers; ++t)
+ if (s == gg::mx_tier_name(t)) return t;
+ return -1;
+}
+
+struct Acts { // T tokens' activations of E and FF values in every form a kernel takes
+ int T = 0;
+ std::vector<float> xf, hf; // F32 (rounded as q8_round) [T][E], [T][FF]
+ std::vector<uint8_t> xq, hq; // mx_quant's [T][2E], [T][2FF]
+ std::vector<uint8_t> x8, h8; // ggml's block_q8_0 [T][E / 32 * 34]
+ void make(int t, uint64_t seed) {
+ T = t;
+ std::mt19937 rng((unsigned) seed);
+ std::normal_distribution<float> nd(0.0f, 1.0f);
+ xf.resize((size_t) T * E);
+ hf.resize((size_t) T * FF);
+ for (auto& v : xf) v = nd(rng);
+ for (auto& v : hf) v = 0.05f * nd(rng);
+ xq.resize((size_t) T * gg::mx_act_bytes(E));
+ hq.resize((size_t) T * gg::mx_act_bytes(FF));
+ x8.resize((size_t) T * E / 32 * 34);
+ h8.resize((size_t) T * FF / 32 * 34);
+ for (int i = 0; i < T; ++i) {
+ gg::mx_quant(&xf[(size_t) i * E], E, &xq[(size_t) i * gg::mx_act_bytes(E)]);
+ gg::mx_quant(&hf[(size_t) i * FF], FF, &hq[(size_t) i * gg::mx_act_bytes(FF)]);
+#if defined(STRATA_NATIVE_EXPERTS)
+ const auto* q8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_0);
+ q8->from_float(&xf[(size_t) i * E], &x8[(size_t) i * E / 32 * 34], E);
+ q8->from_float(&hf[(size_t) i * FF], &h8[(size_t) i * FF / 32 * 34], FF);
+#endif
+ gg::q8_round(&xf[(size_t) i * E], E, &xf[(size_t) i * E]);
+ gg::q8_round(&hf[(size_t) i * FF], FF, &hf[(size_t) i * FF]);
+ }
+ }
+};
+
+// rows [r0, r1) of one matrix (k values a row) against the T activations, kernel `kern`
+void rows(int kern, const uint8_t* w, size_t rb, int k, int r0, int r1, const Acts& a, float* const* y, int pf) {
+ const bool down = k == FF;
+ if (kern == kFloat) {
+ for (int i0 = 0; i0 < a.T; i0 += 8) {
+ const float* xp[8];
+ const int n = std::min(8, a.T - i0);
+ for (int i = 0; i < n; ++i) xp[i] = (down ? a.hf.data() : a.xf.data()) + (size_t) (i0 + i) * k;
+ gg::mx_cpu_rows(w, rb, k, r0, r1, xp, n, y + i0);
+ }
+ return;
+ }
+ if (kern == kGgml) {
+#if defined(STRATA_NATIVE_EXPERTS)
+ static const ggml_vec_dot_t dot = ggml_get_type_traits_cpu(GGML_TYPE_MXFP4)->vec_dot;
+ const uint8_t* x8 = down ? a.h8.data() : a.x8.data();
+ for (int r = r0; r < r1; ++r)
+ for (int i = 0; i < a.T; ++i) dot(k, &y[i][r - r0], 0, w + (size_t) r * rb, 0, x8 + (size_t) i * k / 32 * 34, 0, 1);
+#endif
+ return;
+ }
+ const void* ap[16];
+ for (int i = 0; i < a.T; ++i) ap[i] = (down ? a.hq.data() : a.xq.data()) + (size_t) i * gg::mx_act_bytes(k);
+ gg::mx_rows_tier(kern, w, rb, k, r0, r1, ap, a.T, y, pf);
+}
+
+struct Bench {
+ const uint8_t* mem = nullptr;
+ int n_exp = 16, per_layer = 3;
+ double secs = 0.4;
+ // GB/s of weights with `pool` (its size the threads), kernel kern, the T tokens of a
+ double run(Pool& pool, int kern, const Acts& a, int pf) {
+ const int GB = FF / RB, DB = E / RB;
+ std::vector<std::vector<float>> ys((size_t) pool.size() * 64);
+ int layer = 0;
+ size_t bytes = 0;
+ auto one_layer = [&] {
+ const int e0 = (layer * per_layer) % n_exp;
+ std::function<void(int)> fgu = [&](int j) {
+ const int e = (e0 + j / (2 * GB)) % n_exp, m = (j / GB) % 2, b = j % GB;
+ float yb[16][RB];
+ float* yp[16];
+ for (int i = 0; i < a.T; ++i) yp[i] = yb[i];
+ rows(kern, mem + (size_t) e * kExpert + (size_t) m * kMat, kGr, E, b * RB, b * RB + RB, a, yp, pf);
+ };
+ pool.run(per_layer * 2 * GB, fgu);
+ std::function<void(int)> fd = [&](int j) {
+ const int e = (e0 + j / DB) % n_exp, b = j % DB;
+ float yb[16][RB];
+ float* yp[16];
+ for (int i = 0; i < a.T; ++i) yp[i] = yb[i];
+ rows(kern, mem + (size_t) e * kExpert + 2 * kMat, kDr, FF, b * RB, b * RB + RB, a, yp, pf);
+ };
+ pool.run(per_layer * DB, fd);
+ bytes += (size_t) per_layer * kExpert;
+ ++layer;
+ };
+ for (int w = 0; w < 2; ++w) one_layer(); // (warm: the threads, the code)
+ bytes = 0;
+ const double t0 = now_s();
+ double t = t0;
+ while (t - t0 < secs) {
+ one_layer();
+ t = now_s();
+ }
+ return bytes / (t - t0) / 1e9;
+ }
+};
+
+
+} // namespace
+
+int main(int argc, char** argv) {
+ std::vector<int> threads = {1}, Ts = {1, 2, 4, 8, 16}, pfs = {0};
+ std::vector<std::string> kerns = {"float", "avx2", "avxvnni", "avx512", "avx512vnni", "ggml"};
+ std::string file, make_file;
+ Bench B;
+ for (int i = 1; i + 1 < argc; i += 2) {
+ const std::string a = argv[i], v = argv[i + 1];
+ if (a == "--threads") threads = ints(v);
+ else if (a == "--T") Ts = ints(v);
+ else if (a == "--pf") pfs = ints(v);
+ else if (a == "--experts") B.n_exp = std::atoi(v.c_str());
+ else if (a == "--per-layer") B.per_layer = std::atoi(v.c_str());
+ else if (a == "--secs") B.secs = std::atof(v.c_str());
+ else if (a == "--kernels") kerns = words(v);
+ else if (a == "--file") file = v;
+ else if (a == "--make-file") make_file = v;
+ }
+#if defined(STRATA_NATIVE_EXPERTS)
+ ggml_cpu_init();
+#endif
+ const size_t total = (size_t) B.n_exp * kExpert;
+ if (!make_file.empty()) {
+ std::FILE* f = std::fopen(make_file.c_str(), "wb");
+ if (f == nullptr) return 1;
+ std::vector<uint8_t> buf(kExpert);
+ for (int e = 0; e < B.n_exp; ++e) {
+ fill_mx(buf.data(), buf.size(), 1000 + (uint64_t) e);
+ std::fwrite(buf.data(), 1, buf.size(), f);
+ }
+ std::fclose(f);
+ std::printf("wrote %s: %d experts, %.0f MB\n", make_file.c_str(), B.n_exp, total / 1e6);
+ return 0;
+ }
+ os::fd_t fd = os::kBadFd;
+ if (!file.empty()) {
+ fd = os::open_ro(file);
+ if (fd == os::kBadFd || os::file_size(fd) < total) {
+ std::fprintf(stderr, "%s: cannot open, or smaller than %d experts\n", file.c_str(), B.n_exp);
+ return 1;
+ }
+ B.mem = (const uint8_t*) os::map_ro(fd, total);
+ volatile uint64_t s = 0; // (into the page cache and the mapping)
+ for (size_t o = 0; o < total; o += 4096) s += B.mem[o];
+ } else {
+ uint8_t* m = (uint8_t*) os::anon_alloc(total, -1, 1);
+ if (m == nullptr) return 1;
+ for (int e = 0; e < B.n_exp; ++e) fill_mx(m + (size_t) e * kExpert, kExpert, 1000 + (uint64_t) e);
+ B.mem = m;
+ }
+ std::printf("CPU tiers:");
+ for (int t = 0; t < gg::kMxTiers; ++t)
+ if (gg::mx_tier_ok(t)) std::printf(" %s", gg::mx_tier_name(t));
+ std::printf(" (mx_rows takes %s: %s); weights %s, %d experts (%.0f MB), %d a layer\n", gg::mx_tier_name(gg::mx_tier()),
+ gg::mx_tier_report(), file.empty() ? "anonymous memory" : file.c_str(), B.n_exp, total / 1e6, B.per_layer);
+
+ // ---- the int8 tiers against the scalar tier, and the old F32 path against both (T = 3, 64 rows of each kind)
+ {
+ Acts a;
+ a.make(3, 7);
+ for (int down = 0; down < 2; ++down) {
+ const int k = down ? FF : E;
+ const size_t rb = down ? kDr : kGr;
+ const uint8_t* w = B.mem + (down ? 2 * kMat : 0);
+ float ref[3][RB], got[3][RB];
+ float* rp[3] = {ref[0], ref[1], ref[2]};
+ float* gp[3] = {got[0], got[1], got[2]};
+ rows(gg::kMxScalar, w, rb, k, 0, RB, a, rp, 0);
+ for (int t = 0; t < gg::kMxTiers + 1; ++t) {
+ const int kern = t < gg::kMxTiers ? t : kFloat;
+ if (kern != kFloat && (kern == gg::kMxScalar || !gg::mx_tier_ok(kern))) continue;
+ rows(kern, w, rb, k, 0, RB, a, gp, 0);
+ double md = 0, mx = 0;
+ for (int i = 0; i < 3; ++i)
+ for (int r = 0; r < RB; ++r) {
+ md = std::max(md, (double) std::fabs(got[i][r] - ref[i][r]));
+ mx = std::max(mx, (double) std::fabs(ref[i][r]));
+ }
+ std::printf(" %-6s %-10s vs scalar: max |diff| %.3g (max |y| %.3g, rel %.2g)%s\n", down ? "down" : "gate",
+ kern == kFloat ? "float" : gg::mx_tier_name(kern), md, mx, md / std::max(mx, 1e-30),
+ kern != kFloat && std::memcmp(got, ref, sizeof got) == 0 ? " bit-identical" : "");
+ }
+ }
+ // the quant's two forms: the same bytes
+ std::vector<uint8_t> q1(gg::mx_act_bytes(E)), q2(gg::mx_act_bytes(E));
+ gg::mx_quant_blocks(a.xf.data(), 0, E / 32, q1.data(), E);
+ gg::mx_quant_blocks_scalar(a.xf.data(), 0, E / 32, q2.data(), E);
+ std::printf(" quant: vector vs scalar bytes %s\n", q1 == q2 ? "identical" : "DIFFER");
+ }
+
+ // ---- the quant and madvise costs
+ {
+ std::vector<float> x(E), r(E);
+ std::mt19937 rng(3);
+ std::normal_distribution<float> nd;
+ for (auto& v : x) v = nd(rng);
+ std::vector<uint8_t> q(gg::mx_act_bytes(E));
+ const int N = 20000;
+ double t0 = now_s();
+ for (int i = 0; i < N; ++i) gg::mx_quant(x.data(), E, q.data());
+ const double tq = (now_s() - t0) / N;
+ t0 = now_s();
+ for (int i = 0; i < N / 10; ++i) gg::q8_round(x.data(), E, r.data());
+ const double tr = (now_s() - t0) / (N / 10);
+ std::printf("activation of %d: mx_quant %.2f us, q8_round (old, scalar) %.2f us\n", E, tq * 1e6, tr * 1e6);
+ if (!file.empty()) {
+ t0 = now_s();
+ int n = 0;
+ for (int e = 0; e < B.n_exp; ++e)
+ for (int m = 0; m < 3; ++m, ++n)
+ os::prefetch({{B.mem + (size_t) e * kExpert + (size_t) m * kMat, kMat}});
+ std::printf("madvise(WILLNEED) of a cached expert matrix (%.1f MB): %.1f us\n", kMat / 1e6,
+ (now_s() - t0) / n * 1e6);
+ t0 = now_s();
+ int nres = 0;
+ for (int rep = 0; rep < 10; ++rep)
+ for (int e = 0; e < B.n_exp; ++e)
+ for (int m = 0; m < 3; ++m)
+ nres += os::resident_sampled(B.mem + (size_t) e * kExpert + (size_t) m * kMat, kMat);
+ std::printf("os::resident_sampled (8 pages) of an expert matrix: %.2f us (%d of %d resident)\n",
+ (now_s() - t0) / (10 * n) * 1e6, nres / 10, n);
+ }
+ }
+
+ // ---- the table
+ const std::vector<int> order = cpu_order();
+ std::printf("\n%-11s %3s %4s %5s %9s %10s\n", "kernel", "T", "thr", "pf", "GB/s", "GB/s/thr");
+ for (int nt : threads) {
+ nt = std::max(1, std::min(nt, (int) order.size()));
+ Pool pool(nt - 1, std::vector<int>(order.begin() + 1, order.begin() + nt), 20000);
+ os::pin_self(order[0]);
+ for (const std::string& ks : kerns) {
+ const int kern = kern_id(ks);
+ if (kern < 0 || (kern < gg::kMxTiers && !gg::mx_tier_ok(kern))) continue;
+#if !defined(STRATA_NATIVE_EXPERTS)
+ if (kern == kGgml) continue;
+#endif
+ for (int T : Ts) {
+ if (T < 1 || T > 16) continue;
+ Acts a;
+ a.make(T, 11 + (uint64_t) T);
+ for (int pf : pfs) {
+ if (pf != 0 && (kern == kFloat || kern == kGgml || kern == gg::kMxScalar)) continue;
+ const double g = B.run(pool, kern, a, pf);
+ std::printf("%-11s %3d %4d %5d %9.2f %10.2f\n", ks.c_str(), T, nt, pf, g, g / nt);
+ std::fflush(stdout);
+ }
+ }
+ }
+ }
+ return 0;
+}
diff --git a/third_party/project-maya/LICENSE b/third_party/project-maya/LICENSE
new file mode 100644
index 000000000..7cf2d3348
--- /dev/null
+++ b/third_party/project-maya/LICENSE
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Peasant Smith and the Project Maya contributors
+Copyright (c) 2026 Niko1221 and the Strata contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/third_party/project-maya/NOTICE.md b/third_party/project-maya/NOTICE.md
new file mode 100644
index 000000000..0b3bf289f
--- /dev/null
+++ b/third_party/project-maya/NOTICE.md
@@ -0,0 +1,11 @@
+# Project Maya (maya-ds4), DeepSeek-V4.1 engine
+
+The files under `src/dsv41/`, `include/strata/dsv41/`, `tests/dsv41/` and `tools/dsv41/` come from
+https://github.com/mw00/project-maya (release v1.1.17, commit 8d9a9b5d), its DeepSeek engine `src/ds4`, `tests/ds4_*`,
+`tools/ds4` and the kernel sources it builds on (`exl3`, `sampler`). Project Maya is itself built on Strata.
+
+Copyright (c) 2026 Peasant Smith (mw00) and the Project Maya contributors, including needmorevram.
+MIT License, the full text is in `LICENSE` beside this file. The notice stays with every copy.
+
+Changes made while porting into Strata are listed in the git history of the `src/dsv41` directory
+(the first commit of each file is the unmodified upstream copy).
diff --git a/tools/dsv41/bytecheck.sh b/tools/dsv41/bytecheck.sh
new file mode 100644
index 000000000..18f606f2d
--- /dev/null
+++ b/tools/dsv41/bytecheck.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+# tools/ds4/bytecheck.sh - maya-ds4's byte-identity check of two builds (a change that must not move a bit: the same
+# prompt through each, every logit row dumped, the files compared byte for byte, the greedy tokens compared).
+#
+# tools/ds4/bytecheck.sh <base maya-ds4> <new maya-ds4> <pack> <out dir> <ids file> [<check> ...]
+# checks: windows (DS4_CHUNK_MIN=100000: windows of 16), chunked (the prompt as one chunked run), drafter
+# (--dspark on); default: windows chunked. 64 greedy tokens after the prompt.
+# env: NEW_RESERVE_MIB (default 600, the engine's --reserve-mib default; the packed KV caches: 624, 626 with the
+# drafter - see "placement" below); GPU (default 0); CTX (default 8192)
+#
+# Placement. --no-expert-cache --expert-profile none fix WHICH experts VRAM holds (by index) and keep them there, but
+# not HOW MANY: that follows the free VRAM at load. A change that frees or uses VRAM (smaller caches, a new buffer)
+# moves an expert or more between VRAM and the CPU lane / the prompt path's streamed sets, and the prompt path sums a
+# token's experts batch by batch - another order, other bits (the windows path only differs when a moved expert is
+# picked). So whenever a change frees or uses VRAM, give the build with more free VRAM that much more reserve:
+# NEW_RESERVE_MIB = 600 + what it frees, measured, not added up from the allocations (the driver rounds them): load
+# both once (the check's own first pair will do), and with E experts in VRAM each and F MiB free after the load,
+# freed = (F_new - F_base) + (E_new - E_base) * the MiB of an expert (the load line: experts GB / E). Then check that
+# the load lines' "(N of M in VRAM)" agree - this script prints them. The packed KV caches on a 32 GB V100 at 8K:
+# 24 MiB (626 with the drafter's 26; the allocations' sum said 18-19, and 618 / 619 left one expert more).
+#
+# A prompt that a checkpoint splits (600 tokens, then 400 more from the checkpoint) is not the same bits as the 1000 in
+# one chunked run: chunk boundaries change the arithmetic. Compare a split run with the same split in the other build.
+set -u
+base=$1
+new=$2
+pack=$3
+out=$4
+ids=$(cat "$5")
+shift 5
+checks=${*:-windows chunked}
+mkdir -p "$out"
+common="--pack $pack --max-context ${CTX:-8192} --vision off --verbose --no-expert-cache --expert-profile none --n 64"
+run() { # <build> <check> <exe> [env ...] -- <args ...>
+ local b=$1 c=$2 exe=$3
+ shift 3
+ local envs=()
+ while [ "$1" != "--" ]; do
+ envs+=("$1")
+ shift
+ done
+ shift
+ local reserve=600
+ [ "$b" = new ] && reserve=${NEW_RESERVE_MIB:-600}
+ env CUDA_VISIBLE_DEVICES=${GPU:-0} ${envs[@]+"${envs[@]}"} "$exe" $common --reserve-mib $reserve "$@" --ids "$ids" \
+ --dump-logits "$out/${b}_$c.f32" > "$out/${b}_$c.out" 2> "$out/${b}_$c.err"
+ echo "$b $c rc=$?"
+}
+for c in $checks; do
+ for b in base new; do
+ exe=$base
+ [ $b = new ] && exe=$new
+ case $c in
+ windows) run $b $c "$exe" DS4_CHUNK_MIN=100000 -- --dspark none ;;
+ chunked) run $b $c "$exe" -- --dspark none --window 1048576 ;;
+ drafter) run $b $c "$exe" -- --dspark on ;;
+ *) echo "unknown check $c" && exit 2 ;;
+ esac
+ done
+ echo "== $c"
+ for b in base new; do
+ echo " $b: $(grep -h 'in VRAM' "$out/${b}_$c.err" | head -1 | sed 's/.*experts/experts/')"
+ done
+ echo " logits: $(cmp -s "$out/base_$c.f32" "$out/new_$c.f32" && echo IDENTICAL || echo DIFFER);" \
+ "tokens: $(cmp -s <(grep -h '^[0-9]' "$out/base_$c.out") <(grep -h '^[0-9]' "$out/new_$c.out") && echo IDENTICAL || echo DIFFER)"
+done
diff --git a/tools/dsv41/chat_template.jinja b/tools/dsv41/chat_template.jinja
new file mode 100644
index 000000000..7b983a45f
--- /dev/null
+++ b/tools/dsv41/chat_template.jinja
@@ -0,0 +1,93 @@
+{#- DeepSeek-V4.1's prompt format (the model repo's encoding/encoding.py), as a chat template for Maya's server.
+ Turns: <|User|> ... <|Assistant|> (reasoning</think>)answer<|end▁of▁sentence|>. Thinking mode ends the prompt
+ with <think> and starts with a reasoning-effort line (low 50, high 75, max 100); chat mode ends it with </think>.
+ Tool results (role "tool") are <tool_result> blocks of a user turn, merged with the user messages around them;
+ tool calls are DSML blocks; the tools are listed in the system turn. Earlier turns keep their reasoning only when
+ tools are given (encoding.py's drop_thinking). A picture in a message's content list is its placeholder
+ <|deepseek_image|>, a block of its own between the text blocks (the server expands it into the picture's span). -#}
+{%- set thinking = enable_thinking is not defined or enable_thinking -%}
+{%- set efforts = {'low': 50, 'medium': 75, 'high': 75, 'xhigh': 100, 'max': 100} -%}
+{%- if reasoning_effort is defined and reasoning_effort is number -%}
+{%- set budget = reasoning_effort -%}
+{%- elif reasoning_effort is defined and reasoning_effort in efforts -%}
+{%- set budget = efforts[reasoning_effort] -%}
+{%- else -%}
+{%- set budget = 75 -%}
+{%- endif -%}
+{%- set has_tools = tools is defined and tools is not none and tools | length > 0 -%}
+{%- set keep = has_tools -%}
+{%- set ns = namespace(last_user=-1, schemas='') -%}
+{%- if has_tools -%}{%- for t in tools -%}{%- set ns.schemas = ns.schemas ~ (t | tojson) ~ ('' if loop.last else '\n') -%}{%- endfor -%}{%- endif -%}
+{%- for m in messages -%}{%- if m.role == 'user' -%}{%- set ns.last_user = loop.index0 -%}{%- endif -%}{%- endfor -%}
+{%- macro text(c) -%}
+{%- if c is string -%}{{ c }}{%- elif c is iterable -%}
+{%- for p in c if p.type in ['text', 'image'] -%}{%- if not loop.first -%}{{ '\n\n' }}{%- endif -%}{{ p.text if p.type == 'text' else '<|deepseek_image|>' }}{%- endfor -%}
+{%- endif -%}
+{%- endmacro -%}
+{%- macro tools_block() -%}
+## Tools
+
+You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:
+
+<|DSML| calls>
+<|DSML| invoke name="$TOOL_NAME">
+<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>
+...
+</|DSML| invoke>
+<|DSML| invoke name="$TOOL_NAME2">
+...
+</|DSML| invoke>
+</|DSML| calls>
+
+String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
+
+If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
+
+Otherwise, output directly after </think> with tool calls or final response.
+
+### Available Tool Schemas
+
+{{ ns.schemas }}
+
+You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
+{% endmacro -%}
+{{- '<|begin▁of▁sentence|>' -}}
+{%- set effort_line = ('Reasoning Effort: ' ~ budget ~ ' (range 1-100, the higher the value, the more thorough the reasoning)\n\n') if thinking else '' -%}
+{%- if messages and messages[0].role == 'system' -%}
+{{- '<|System|>' ~ effort_line ~ text(messages[0].content) -}}
+{%- if has_tools -%}{{- '\n\n' ~ tools_block() -}}{%- endif -%}
+{%- elif has_tools -%}
+{{- '<|System|>' ~ effort_line ~ '\n\n' ~ tools_block() -}}
+{%- elif thinking -%}
+{{- '<|System|>' ~ effort_line -}}
+{%- endif -%}
+{%- for m in messages -%}
+{%- set i = loop.index0 -%}
+{%- set prev = messages[i - 1].role if i > 0 else '' -%}
+{%- set next = messages[i + 1].role if i + 1 < messages | length else '' -%}
+{%- if m.role == 'system' and i == 0 -%}
+{%- elif m.role in ['user', 'tool', 'system'] -%}
+{%- if prev in ['user', 'tool'] or (prev == 'system' and i - 1 > 0) -%}{{- '\n\n' -}}{%- else -%}{{- '<|User|>' -}}{%- endif -%}
+{%- if m.role == 'tool' -%}{{- '<tool_result>' ~ text(m.content) ~ '</tool_result>' -}}{%- else -%}{{- text(m.content) -}}{%- endif -%}
+{%- if next not in ['user', 'tool', 'system'] -%}
+{%- if thinking and (keep or i >= ns.last_user) -%}{{- '<|Assistant|><think>' -}}{%- else -%}{{- '<|Assistant|></think>' -}}{%- endif -%}
+{%- endif -%}
+{%- elif m.role == 'assistant' -%}
+{%- if thinking and (keep or i > ns.last_user) -%}{{- (m.reasoning_content or '') ~ '</think>' -}}{%- endif -%}
+{{- text(m.content) -}}
+{%- if m.tool_calls -%}
+{{- '\n\n<|DSML| calls>\n' -}}
+{%- for c in m.tool_calls -%}
+{%- set f = c.function if c.function is defined else c -%}
+{{- '<|DSML| invoke name="' ~ f.name ~ '">\n' -}}
+{%- for k, v in (f.arguments or {}).items() -%}
+{{- '<|DSML| parameter name="' ~ k ~ '" string="' ~ ('true' if v is string else 'false') ~ '">' ~ (v if v is string else v | tojson) ~ '</|DSML| parameter>' -}}
+{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}
+{%- endfor -%}
+{{- '\n</|DSML| invoke>' -}}{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}
+{%- endfor -%}
+{{- '\n</|DSML| calls>' -}}
+{%- endif -%}
+{{- '<|end▁of▁sentence|>' -}}
+{%- endif -%}
+{%- endfor -%}
diff --git a/tools/dsv41/compare.py b/tools/dsv41/compare.py
new file mode 100644
index 000000000..9dfd91b43
--- /dev/null
+++ b/tools/dsv41/compare.py
@@ -0,0 +1,60 @@
+"""tools/ds4/compare.py - the engine's logits (maya-ds4 --dump-logits) against the reference's (tools/ds4/ref.py --dump).
+
+ python tools/ds4/compare.py --ref ref.npz --engine engine.f32 [--vocab 129280]
+
+Per position: the cosine of the two logit rows, the largest difference, whether the top token agrees, and the KL
+divergence of the engine's distribution from the reference's.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+import numpy as np
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--ref", required=True)
+ ap.add_argument("--engine", required=True)
+ ap.add_argument("--vocab", type=int, default=129280)
+ ap.add_argument("--layers", help="the engine's DS4_DUMP_H file: each layer's residual stream against the ref's")
+ a = ap.parse_args()
+ r = np.load(a.ref)
+ ref = r["logits"].astype(np.float64)
+ eng = np.fromfile(a.engine, dtype=np.float32).reshape(-1, a.vocab).astype(np.float64)
+ n = min(len(ref), len(eng))
+ agree = 0
+ kls = []
+ for i in range(n):
+ x, y = ref[i], eng[i]
+ cos = float(x @ y / (np.linalg.norm(x) * np.linalg.norm(y)))
+ px = np.exp(x - x.max())
+ px /= px.sum()
+ py = np.exp(y - y.max())
+ py /= py.sum()
+ kl = float(np.sum(px * (np.log(px + 1e-30) - np.log(py + 1e-30))))
+ kls.append(kl)
+ top = int(x.argmax()) == int(y.argmax())
+ agree += top
+ print("pos %3d cos %.6f max|d| %8.4f top %s (ref %6d eng %6d) KL %.2e" %
+ (i, cos, np.abs(x - y).max(), "same" if top else "DIFF", x.argmax(), y.argmax(), kl))
+ print("%d positions: top-1 agrees at %d, mean KL %.3e, max KL %.3e" % (n, agree, np.mean(kls), np.max(kls)))
+ if a.layers:
+ hs = r["hs"] # [layers, T, hc, dim]
+ nl, T = hs.shape[0], hs.shape[1]
+ eh = np.fromfile(a.layers, dtype=np.float32).reshape(-1, nl, *hs.shape[2:]) # [T, layers, hc, dim]
+ T = min(T, len(eh))
+ print("relative error of each layer's residual stream (rows: layers, columns: positions)")
+ print(" " + " ".join("%6d" % t for t in range(T)))
+ for li in range(nl):
+ row = []
+ for t in range(T):
+ x, y = hs[li, t].astype(np.float64), eh[t, li].astype(np.float64)
+ row.append(np.linalg.norm(x - y) / np.linalg.norm(x))
+ print("L%-3d " % li + " ".join("%6.4f" % v for v in row))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/download-3.5bpw.json b/tools/dsv41/download-3.5bpw.json
new file mode 100644
index 000000000..f9f45dcc2
--- /dev/null
+++ b/tools/dsv41/download-3.5bpw.json
@@ -0,0 +1,239 @@
+{
+ "_": "DeepSeek-V4.1-Flash-1017 at 3.5 bits for maya-ds4 (maya.py's DeepSeek setup): every file pinned to a revision, with its size and sha256. The EXL3 quant's repo has no Engram tables: they come from DeepSeek's base repo (they changed in the 1017 release - the older V4.1's must not be used). The repo's assets/ and encoding/ folders (its README's picture, DeepSeek's prompt-format reference) are not needed and not listed.",
+ "name": "DeepSeek-V4.1-Flash-1017",
+ "repo": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-3.5bpw",
+ "revision": "04a492fc3b4d25ddbab8f2d2f07f235ec77c9010",
+ "license": "MIT",
+ "files": {
+ ".gitattributes": [
+ 1697,
+ "5c43150d6197107ff8d2b87f76f9f88b7a1a6ab1af968d7828ab33a62e9d06f4"
+ ],
+ "LICENSE": [
+ 1084,
+ "f2c6c602815669d292889e5be8c802f2ed950653b77999b1584e8e6aed25d040"
+ ],
+ "README.md": [
+ 12779,
+ "3dfe8dd8e4861a3e2a95d72962d2c323fef9d663e829f1c1c134893ff23964d4"
+ ],
+ "chat_template.jinja": [
+ 34813,
+ "2ed8bc3372c9c68577f232229e78ebfe37a53ae7964694d845533e485ed738e8"
+ ],
+ "config.json": [
+ 3628,
+ "2cbc57cd18428f62aba490870ceda44d2b0639649c05754d6040a5be8e976cf4"
+ ],
+ "exl3_k_map.json": [
+ 1323,
+ "efc15305bfe0cc0a728d89943c542bd885fc6cbdc8863f892ae9a8330e5c172c"
+ ],
+ "model-00001-of-00044.safetensors": [
+ 1323827293,
+ "ba5ccb4d39882d08800e856150bce42b23584f3d9943520ae4cff5647d5a3045"
+ ],
+ "model-00002-of-00044.safetensors": [
+ 7060206910,
+ "012504ba47677b9bb8727bb89cb8c6327a066ef10d2ade179f7ffd5c8151ff84"
+ ],
+ "model-00003-of-00044.safetensors": [
+ 6942014187,
+ "9f9d5228106bd5c8064d672f5ae24cce6005be8c2cf33592ef0478655c9e9b80"
+ ],
+ "model-00004-of-00044.safetensors": [
+ 6950309596,
+ "8cae1bf802a2e10c59e1f3e057b58f183a0f64e6bf4a1c76107168a9cbb9d532"
+ ],
+ "model-00005-of-00044.safetensors": [
+ 6942014187,
+ "b583b45c6a5848dc33a7219d2c34149663779059b11b1bba5a9e76951b333af7"
+ ],
+ "model-00006-of-00044.safetensors": [
+ 6942014187,
+ "c4d2e5290f5253262adb8b83b91f37406f24d0a1ce14c7c6844734aa2cbe064a"
+ ],
+ "model-00007-of-00044.safetensors": [
+ 6942014187,
+ "65a639c07a74c59da458298adc99bca87036f8316f9c09ac409e2c9b68d414ba"
+ ],
+ "model-00008-of-00044.safetensors": [
+ 6942014187,
+ "90f9448977e45f2db5859ca7c9d33ea34a6cdaffc4c28c0d8bb030938db4d614"
+ ],
+ "model-00009-of-00044.safetensors": [
+ 6942014187,
+ "06cd3e7672a90c4edfb9ff98b77d11261e39a21558af314f2b235ea5b90b7518"
+ ],
+ "model-00010-of-00044.safetensors": [
+ 6950309596,
+ "286c1c0597f29d154391d9f8252cabc560a89be68b50c1f227d0b078200336b2"
+ ],
+ "model-00011-of-00044.safetensors": [
+ 6942014187,
+ "b644d168923b319816185fc99b77478db7698067e5e8c67fae619f56a6b18405"
+ ],
+ "model-00012-of-00044.safetensors": [
+ 5243325245,
+ "e8e278e5d6a44c81d6fe16c9231aa2cb3823cb8d26e55bdb491345f4e0797f28"
+ ],
+ "model-00013-of-00044.safetensors": [
+ 5243325245,
+ "39e25072f8f8de381b3176cb56eb0ec7caa8ab70e85956b57d7f9452291cd386"
+ ],
+ "model-00014-of-00044.safetensors": [
+ 5243325245,
+ "3e6e26fad54c7cbc90fbda7209a2f389ff73e58160fb60326bcaa5635e52d811"
+ ],
+ "model-00015-of-00044.safetensors": [
+ 5361517974,
+ "e34e397fb634b06a8558643d6b621a2b560b68c0766aac9c0252451ea3daffe2"
+ ],
+ "model-00016-of-00044.safetensors": [
+ 5251620673,
+ "db9782ab6346e2dbcf259b79289e6dbc6b4fe183c4936bc1c1c892e2604fd6c7"
+ ],
+ "model-00017-of-00044.safetensors": [
+ 5243325245,
+ "711c747176c8494d0cef290e98de5026658bd79a3ef51ed4d43ce403acbfa481"
+ ],
+ "model-00018-of-00044.safetensors": [
+ 5243325245,
+ "111ce8d8496d6265098337f5b5ff79e71a31ba340eb2b682d0042dbca4dc954a"
+ ],
+ "model-00019-of-00044.safetensors": [
+ 5243325245,
+ "6feedcd839b5593b275761107593ad9f9a0244098dffed66ae6f88ead8844c0d"
+ ],
+ "model-00020-of-00044.safetensors": [
+ 5243325245,
+ "bf22fe666d473899b7d6f7bfa1709ad3cffe93f69d39a3da272eac854b39a882"
+ ],
+ "model-00021-of-00044.safetensors": [
+ 5243325245,
+ "afef0df9f451e9392cbf4704c9f8ee9183695e543b669d811b6eca88c21a7fe2"
+ ],
+ "model-00022-of-00044.safetensors": [
+ 5249642888,
+ "987c51d505f4b49442481ad8596256d846067e8c1fa8edb6c90bd6fc3f85d454"
+ ],
+ "model-00023-of-00044.safetensors": [
+ 5243325245,
+ "38cfb33a3a47483a1b1f7b3eebd22ffc9087437d45c8561062a72a8798fa48ef"
+ ],
+ "model-00024-of-00044.safetensors": [
+ 5243325245,
+ "aec09849c4916f7254129324bb6aa73eb3685448709534f66fbb31009e0edaf1"
+ ],
+ "model-00025-of-00044.safetensors": [
+ 5243325245,
+ "6e0730da79761933690a243e2dd7dd8ecada43c95d9ec96fcc3a520b9f3194a2"
+ ],
+ "model-00026-of-00044.safetensors": [
+ 5247596381,
+ "d9a15b99de6190d10099c62383e283c32ef42fdab8ec37ebf355448669eeb027"
+ ],
+ "model-00027-of-00044.safetensors": [
+ 5243325245,
+ "c673fa72f6aca977106267afb63f218f241f49a57eb833311ec8affe16009560"
+ ],
+ "model-00028-of-00044.safetensors": [
+ 5243325245,
+ "e54fe28e42bd8eec92fa07ac0e4be0850149b20b74e15d7af37d5314a9842deb"
+ ],
+ "model-00029-of-00044.safetensors": [
+ 5243325245,
+ "f69616591acdc45b7760db5ef9fab39dfb66558a908778aab230bd8c0d9faa3d"
+ ],
+ "model-00030-of-00044.safetensors": [
+ 5247596381,
+ "ac4862535de5828c521e7e5e13906f993395940693cb66fbd323ce9360cd1917"
+ ],
+ "model-00031-of-00044.safetensors": [
+ 5243325245,
+ "0beae804fe362965413ba7d26c9c902cf44dd77824838239878332c95170dc7b"
+ ],
+ "model-00032-of-00044.safetensors": [
+ 6942018869,
+ "48df9384d595f94b0093b9182b63b4c48216c3c1e486f7aa43a894e93521785d"
+ ],
+ "model-00033-of-00044.safetensors": [
+ 6942018869,
+ "a2cd7f6f24043d39ad247a24dd642ce78cb75a2f332fd17779aa012817354c28"
+ ],
+ "model-00034-of-00044.safetensors": [
+ 6946290005,
+ "e2f1620bc902230c92361b8689d70f7a72e88283da24a8202561782390c91f70"
+ ],
+ "model-00035-of-00044.safetensors": [
+ 6942018869,
+ "c9fc5076793da76174d3d92ec780fedc73cd2f31a79d26a3ef6f26f8f23c0f8a"
+ ],
+ "model-00036-of-00044.safetensors": [
+ 6942018869,
+ "905176e9573296c98eb4636c8c1c1559291ac6bdf04c7be5ec97f11eddd30a52"
+ ],
+ "model-00037-of-00044.safetensors": [
+ 6942018869,
+ "3f146f50cd59ce9d656f729260d56e5fce4df9e3fcb625102764800abb74ed0e"
+ ],
+ "model-00038-of-00044.safetensors": [
+ 6946290005,
+ "eb83d2598c932d5f367606615a9a92e9b45f958e928850eee45ce53bb01db05a"
+ ],
+ "model-00039-of-00044.safetensors": [
+ 6942018869,
+ "be69239f0303332cabbf3e0d390d32a45e165ebb8bc514aa186896e792bd17bc"
+ ],
+ "model-00040-of-00044.safetensors": [
+ 6942018869,
+ "4ad6d074dcdec1da6425abd105ec105bdcfd594ea7847396cb6d949e86c34df1"
+ ],
+ "model-00041-of-00044.safetensors": [
+ 6942029194,
+ "2900611e1b5e83d8ebeefbd4e0956ddb83356e360edde30f0fd076f48ac86d38"
+ ],
+ "model-00042-of-00044.safetensors": [
+ 4086008391,
+ "f84c4f2f383ec95f46beef3acc71b73809c9e0b7d54d483db27524ecbe70bea0"
+ ],
+ "model-00043-of-00044.safetensors": [
+ 7329857517,
+ "d673275ee4833a78ba76e3a537c50290d0e0ed419b35f68c598b23168f081ee4"
+ ],
+ "model-00044-of-00044.safetensors": [
+ 833651138,
+ "4ddb11bc29f6728e95258e2b52faeb91471d3ae145b8f8bfa3e267dd47d3ce19"
+ ],
+ "model.safetensors.index.json": [
+ 15504210,
+ "002568f81f402b357a92e2601e6eb664216ab8b27f02c6053d4233a9fdbc72e4"
+ ],
+ "quantization_config.json": [
+ 54653015,
+ "493549dee7a0c686cef48bd298bf4ec4f89f4445da2784123726f41854484c5f"
+ ],
+ "tokenizer.json": [
+ 6367257,
+ "c90dfa01249db1be4245780a052ede752e1361c612ac6d08e2bdada7d599476b"
+ ],
+ "tokenizer_config.json": [
+ 801,
+ "6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547"
+ ]
+ },
+ "engram": {
+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",
+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",
+ "files": {
+ "model-00047-of-00048.safetensors": [
+ 101535150936,
+ "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5"
+ ],
+ "model-00048-of-00048.safetensors": [
+ 101537926640,
+ "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/dsv41/download.json b/tools/dsv41/download.json
new file mode 100644
index 000000000..7af42fc54
--- /dev/null
+++ b/tools/dsv41/download.json
@@ -0,0 +1,231 @@
+{
+ "_": "DeepSeek-V4.1-Flash-1017 for maya-ds4 (maya.py's DeepSeek setup): every file pinned to a revision, with its size and sha256. The EXL3 quant's repo has no Engram tables: they come from DeepSeek's base repo (they changed in the 1017 release - the older V4.1's must not be used). The repo's assets/ and encoding/ folders (its README's picture, DeepSeek's prompt-format reference) are not needed and not listed.",
+ "name": "DeepSeek-V4.1-Flash-1017",
+ "repo": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw",
+ "revision": "c178a01284fc5c0e859d8668cffaac81ca8f4042",
+ "license": "MIT",
+ "files": {
+ ".gitattributes": [
+ 1697,
+ "5c43150d6197107ff8d2b87f76f9f88b7a1a6ab1af968d7828ab33a62e9d06f4"
+ ],
+ "LICENSE": [
+ 1084,
+ "f2c6c602815669d292889e5be8c802f2ed950653b77999b1584e8e6aed25d040"
+ ],
+ "README.md": [
+ 11290,
+ "80e9744ad84e77c5668b895f61086712f494da946212a5a9b9da334ee4a03cc3"
+ ],
+ "config.json": [
+ 3628,
+ "eb48f2216645b36f400036fa2d67d8ad6b48233b0f4a1bcb7e99fa6450dad253"
+ ],
+ "exl3_k_map.json": [
+ 1409,
+ "4359af687bdce7fccd4fc74a57c2f9154b42053447cdc6e0d1768e68fffbec0f"
+ ],
+ "model-00001-of-00043.safetensors": [
+ 1323827293,
+ "ba5ccb4d39882d08800e856150bce42b23584f3d9943520ae4cff5647d5a3045"
+ ],
+ "model-00002-of-00043.safetensors": [
+ 5322748742,
+ "11ed8a0cb267169a5f3ea0ceebb24ab09fb5b84b98e4d4c86dc2f06f1f5e0d40"
+ ],
+ "model-00003-of-00043.safetensors": [
+ 5223069939,
+ "1de02ee6b4d909eb5936e08e7c3825cbf2fa65693c5674deade2e70bcb773462"
+ ],
+ "model-00004-of-00043.safetensors": [
+ 5230054628,
+ "8e68a9e396d958d6645378b44ef3419a4c0b743da0e92b354d2163a4c8bf85ce"
+ ],
+ "model-00005-of-00043.safetensors": [
+ 5223069939,
+ "c14a66ba26f57e9577e1a656764b3e5828e09c7630ad7404b80c6e265d03a37d"
+ ],
+ "model-00006-of-00043.safetensors": [
+ 5223069939,
+ "2866602c20b535eec3bb1a737f62d5d6d2dfecbf362977d3a4f5f8e701580b6d"
+ ],
+ "model-00007-of-00043.safetensors": [
+ 5223069939,
+ "4d12814cd25cf07fe9c6791f0050a8afbc1038bae02fb201243f9c56de8171cd"
+ ],
+ "model-00008-of-00043.safetensors": [
+ 5223069939,
+ "cb88f50d0ddd81b98db00a6475c356511c3c21f843b2ccd9845fb4017c3f5d15"
+ ],
+ "model-00009-of-00043.safetensors": [
+ 5223069939,
+ "dbebd82073a27f79dd1353d3c309255f46e577eb1c2f9d8a10a52bc38f6d8bc2"
+ ],
+ "model-00010-of-00043.safetensors": [
+ 5230054628,
+ "8cde4e4124614502c2950e9f3a643027eceeeedbdf3c54cf786dbf125d3d536e"
+ ],
+ "model-00011-of-00043.safetensors": [
+ 5223069939,
+ "c96372d73271d2c59e5a959bb81551aaacb2abe8445f5b9ea9391e2b4a033601"
+ ],
+ "model-00012-of-00043.safetensors": [
+ 5223074621,
+ "337345b1ca174aebcff619bbc2c2a6e33881a42ad8f638956e69b5de4fe21d41"
+ ],
+ "model-00013-of-00043.safetensors": [
+ 5218650941,
+ "042b4274c48e55d09bfe51ac099915f8bc3bb0c7be9b0f9d5e75a9f5c2045497"
+ ],
+ "model-00014-of-00043.safetensors": [
+ 5218650941,
+ "bc27a27e60bebd88ad36f4896aa027911bc5f7fbb30ad9a25f95a59a6374d957"
+ ],
+ "model-00015-of-00043.safetensors": [
+ 5297522070,
+ "74d3f0d12e32a658d8f6c5f9a104826e473c53299267519b46d594619756ae66"
+ ],
+ "model-00016-of-00043.safetensors": [
+ 5225635649,
+ "0a1b0b3ecb817f448d9fb34ebf0757d4b00042220515ecb13f1b86908cb092f3"
+ ],
+ "model-00017-of-00043.safetensors": [
+ 5218650941,
+ "e4b4278a51ce9daf61290d4caba5b953f5a243832ffb5c8d29a0c259807facea"
+ ],
+ "model-00018-of-00043.safetensors": [
+ 5218650941,
+ "07e9d048f9cefb94b9a30f41e39b9390b6bf54a763c0b7e1dcd5d81144fc0849"
+ ],
+ "model-00019-of-00043.safetensors": [
+ 5218650941,
+ "2c6116ee6f584af8c4027cee8f5069d0d51a4e9b487dd73027127d29f4caf274"
+ ],
+ "model-00020-of-00043.safetensors": [
+ 3519956829,
+ "817d948752825a739073b95bc6171a35e4677c095ac617bd6772560ae0e4b5e4"
+ ],
+ "model-00021-of-00043.safetensors": [
+ 3519956829,
+ "4bdda16b5a00ac5ee0e39bc81271af6155caacb7c03928876b07344e20a4ba5f"
+ ],
+ "model-00022-of-00043.safetensors": [
+ 3525291432,
+ "00afb7bd831cd50added9b809cc2e9cec57ab44963f152f8bc42cd6d22f0a4e4"
+ ],
+ "model-00023-of-00043.safetensors": [
+ 3519956829,
+ "28ba9d401e2cd556d0145b31c03ed341f009e2849162e6d9f68438a067d74a59"
+ ],
+ "model-00024-of-00043.safetensors": [
+ 3519956829,
+ "1240d8744902914287da94cd8e2a5d9f17e7c39167bfab20c737a34d7605e50c"
+ ],
+ "model-00025-of-00043.safetensors": [
+ 5218650941,
+ "a053c3dc54c4ecfe295cfa4bd7fb44f1054ade1d596e46496f6791c87ffb60df"
+ ],
+ "model-00026-of-00043.safetensors": [
+ 5222266717,
+ "028c44abbea066c94da7ed0e64f07f8d36ae925a4ff437ef999f5a2b5ec6c9d4"
+ ],
+ "model-00027-of-00043.safetensors": [
+ 5218650941,
+ "69cbb8f2f00ddcb7a83b04d34ca9151c61f3319ec742d0ac5f06abf295a91eef"
+ ],
+ "model-00028-of-00043.safetensors": [
+ 5218650941,
+ "451c69700eb30a9e47ec80fe219aa537985c6afa3f7391db5699fba50d93e05b"
+ ],
+ "model-00029-of-00043.safetensors": [
+ 5218650941,
+ "906a9525cde8176d18066d27ae9d457ff55ee3134438dadea4ee63af8dce45ad"
+ ],
+ "model-00030-of-00043.safetensors": [
+ 5222266717,
+ "420ad929c1efa3f90ba4a39d28159d9a29175089cd9bd4fe32f25c6fb7e473f6"
+ ],
+ "model-00031-of-00043.safetensors": [
+ 5220125501,
+ "0312eb9c04dbe5f538efd94a21d1f76266a8cac9ea01250084aa6513657fc5df"
+ ],
+ "model-00032-of-00043.safetensors": [
+ 5223074621,
+ "4653d193e28120319fe5b3508ffed537d23f57b8d31b20f3f69371fd569e59b2"
+ ],
+ "model-00033-of-00043.safetensors": [
+ 5223074621,
+ "3a6bfc60065324e606082ae8ad68ff403285b24e7a720b3b2274e516a247252a"
+ ],
+ "model-00034-of-00043.safetensors": [
+ 5226690397,
+ "b1825ddd7016d3cefea0f13ec78aff33637c68e0eb721cdb12f83104c03b3760"
+ ],
+ "model-00035-of-00043.safetensors": [
+ 5223074621,
+ "9b513ca99a0e8403d172df623a4c0fe7ecaaa51e09015d9574efa90bd6414a32"
+ ],
+ "model-00036-of-00043.safetensors": [
+ 5223074621,
+ "206ca962082e2c4de968ba4cc82cc1837dccb0b8960f493e04891b281bbfe497"
+ ],
+ "model-00037-of-00043.safetensors": [
+ 5223074621,
+ "6ed69fdcd3bd32762ddefae671c7fce7f9b5a212cdbe1aaee64adaa5ef0eef2c"
+ ],
+ "model-00038-of-00043.safetensors": [
+ 5226690397,
+ "24a61b540656dc210b3bc3215b1713f0b4bee8b724176a4f5968e24b9540035c"
+ ],
+ "model-00039-of-00043.safetensors": [
+ 5223074621,
+ "29e5b40a97edcb91ed6a382b39bbb4d16ce40de2d1ddce2f75a8fd75e0ad609f"
+ ],
+ "model-00040-of-00043.safetensors": [
+ 5223074621,
+ "5c850a3950ce68a93a24336ccc417ac660fdbdb84bcc494aa6d57ea5c28246be"
+ ],
+ "model-00041-of-00043.safetensors": [
+ 5719789284,
+ "5a6f5791d3f5f9635e58db5a693f68c64c6c9e14357e8386a21e5fd232af032d"
+ ],
+ "model-00042-of-00043.safetensors": [
+ 4854649316,
+ "2f80f4a44e4608c6da139eccf2483122ac88a240046872965d546b4fa26ded06"
+ ],
+ "model-00043-of-00043.safetensors": [
+ 3500776125,
+ "c5b94d519d323921baacb34b6111e518ce2c36309d06c80fe4e42e3aa9662bc5"
+ ],
+ "model.safetensors.index.json": [
+ 15504210,
+ "778baeaf9fbcd351800536dc090ff1edddb72d102472b08de0edd9a52cb96656"
+ ],
+ "quantization_config.json": [
+ 54653012,
+ "696cdad521cdebea0642a3c22d01c9152dca4e5ccbf209b5e7f2b808f359bd27"
+ ],
+ "tokenizer.json": [
+ 6367257,
+ "c90dfa01249db1be4245780a052ede752e1361c612ac6d08e2bdada7d599476b"
+ ],
+ "tokenizer_config.json": [
+ 801,
+ "6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547"
+ ]
+ },
+ "engram": {
+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",
+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",
+ "files": {
+ "model-00047-of-00048.safetensors": [
+ 101535150936,
+ "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5"
+ ],
+ "model-00048-of-00048.safetensors": [
+ 101537926640,
+ "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd"
+ ]
+ }
+ }
+}
diff --git a/tools/dsv41/download_gguf.json b/tools/dsv41/download_gguf.json
new file mode 100644
index 000000000..72b911fc5
--- /dev/null
+++ b/tools/dsv41/download_gguf.json
@@ -0,0 +1,337 @@
+{
+ "_": "DeepSeek-V4.1-Flash-1017 as unsloth's GGUFs for maya-ds4 (maya.py's DeepSeek setup, tools/ds4/pack_gguf.py): dense weights BF16 (UD-Q8_K_XL) or Q8_0 (UD-Q4_K_XL), routed experts MXFP4, the Engram tables MXFP8 - every file pinned to a revision, with its size and sha256. Neither has the DSpark drafter or the vision tower (\"drafter\" and \"vision\" below: from DeepSeek's own checkpoint).",
+ "name": "DeepSeek-V4.1-Flash-1017",
+ "repo": "unsloth/DeepSeek-V4.1-Flash-1017-GGUF",
+ "revision": "c991b28e7c109eed10a55075ddb46f830b5b3059",
+ "license": "MIT",
+ "quants": {
+ "UD-Q8_K_XL": {
+ "files": {
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00009.gguf": [
+ 5780589,
+ "cc15d470afcb030558bbc4d8e2ccd840182e2e95de1a95198138efe63a5fd99f"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00002-of-00009.gguf": [
+ 50536121984,
+ "9cb2047676bd20e448be2377b480080c97fb38fae117f4846e07de2c65bb8522"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00003-of-00009.gguf": [
+ 50536121984,
+ "17d398dd327e7ff1991b382677814db75d970b074661757564b6f2ac3d0c8d95"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00004-of-00009.gguf": [
+ 50536122016,
+ "7ea8da1143b9a6bf1424a361eb1b0d6f66c7ce79eac9711162f212cc4f39c319"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00005-of-00009.gguf": [
+ 50536122016,
+ "1952da19a841176b46f9681c99ea57c674f8a8861e16d76fb35cec4befe039b2"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00006-of-00009.gguf": [
+ 50536122016,
+ "d6711781d8effab79b410af9c40920bffcc0c28976e509bc1227ff0c6669c33a"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00007-of-00009.gguf": [
+ 36097230048,
+ "2ebc8d198d03a14865e8a85df637285ceb5dd79b1ffc78424bd3c3e06df57823"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00009.gguf": [
+ 202758032592,
+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00009.gguf": [
+ 16835109952,
+ "df33722d1dcf5ffe0347ed8406f7acb3b18061f4ded2c13c2b0a184b8d41c9ea"
+ ]
+ },
+ "bytes": 508376763197,
+ "earlier": [
+ {
+ "_": "unsloth's layout before it re-sharded the GGUFs on 2026-10-19 (10 shards, the Engram tables in 00008 and 00009): used as it is when the folder holds it whole - nothing downloaded again",
+ "revision": "dc788bdb683b0af7a6e5315a7d98e9d082faf5e5",
+ "files": {
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00010.gguf": [
+ 5781234,
+ "e88691700c4778890cfa845bc21242cf101faa014c8a44d1315402e3f64e701a"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00002-of-00010.gguf": [
+ 48129640000,
+ "8b9e24ae7322af960c6cceb3cc79623ac79c173fba860da2b10fe5de04e763e2"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00003-of-00010.gguf": [
+ 48129640000,
+ "9b5047e0f84d64f673d638a6d85282bae16b53f39cf12559c4b8605b127baeca"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00004-of-00010.gguf": [
+ 48129640000,
+ "83a630d0dd9f2dc6e1d1223b265e78523a51ba6483ab70e3ae3f188978df2491"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00005-of-00010.gguf": [
+ 48129640000,
+ "90ecd6d773dd0eab5ce1a7bdff626617f8b59360ae53a2cd5d763bde0088b2e8"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00006-of-00010.gguf": [
+ 48129640000,
+ "e688bfa9a31f00ac33ac7cd739fbc0a903ae0d1601e85c485c23147c57a282fd"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00007-of-00010.gguf": [
+ 48129640000,
+ "daf02b75c0ad3562c8ccc505fcd94ce08584f967f485b9e9e3ebdbfcafcaf286"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00010.gguf": [
+ 101377628544,
+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00010.gguf": [
+ 101380404256,
+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00010-of-00010.gguf": [
+ 16835109952,
+ "93749d36aa1d2a73dba2bbaf903cc366e694769bc16c216ec2e0dff846e74cac"
+ ]
+ }
+ }
+ ]
+ },
+ "UD-Q4_K_XL": {
+ "files": {
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00001-of-00009.gguf": [
+ 5780589,
+ "cc15d470afcb030558bbc4d8e2ccd840182e2e95de1a95198138efe63a5fd99f"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00002-of-00009.gguf": [
+ 48129640000,
+ "17df217d8fc26a39f21ea40805dc6ab788c7ce8623a59a95c431051df7eafc94"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00003-of-00009.gguf": [
+ 48129640000,
+ "0c8065f620a049cb437024c54ff12c314c94603a1cc683f5191be4689d6d6d3b"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00004-of-00009.gguf": [
+ 48129640000,
+ "3bd8b2ebf8f51d8708a73ae0895f8d4acc7e4c983ba4e89d038492b203cc89ac"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00005-of-00009.gguf": [
+ 48129640000,
+ "929054a6bbf3701765779b58c37988b6c6284a3b8413901858b59c0e95f37cc3"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00006-of-00009.gguf": [
+ 48129640000,
+ "49d3de04bd6a09de786c9ef7db900eae02ee9fd391573f91a923013d7d5b24d3"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00007-of-00009.gguf": [
+ 48129640000,
+ "f18feae5878dd680f1ea87ecd1d84f4fb098480298b25a3162dcbdab8dc2ca5b"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00008-of-00009.gguf": [
+ 202758032592,
+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00009-of-00009.gguf": [
+ 9165923392,
+ "c21ad6105a780bc2f44eb7dd41f9b1a1efb4e8ec87618f60b733bc2652bd2ffe"
+ ]
+ },
+ "bytes": 500707576573,
+ "earlier": [
+ {
+ "_": "unsloth's layout before it re-sharded the GGUFs on 2026-10-19 (10 shards, the Engram tables in 00008 and 00009): used as it is when the folder holds it whole - nothing downloaded again",
+ "revision": "dc788bdb683b0af7a6e5315a7d98e9d082faf5e5",
+ "files": {
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00001-of-00010.gguf": [
+ 5781234,
+ "e88691700c4778890cfa845bc21242cf101faa014c8a44d1315402e3f64e701a"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00002-of-00010.gguf": [
+ 48129640000,
+ "8b9e24ae7322af960c6cceb3cc79623ac79c173fba860da2b10fe5de04e763e2"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00003-of-00010.gguf": [
+ 48129640000,
+ "9b5047e0f84d64f673d638a6d85282bae16b53f39cf12559c4b8605b127baeca"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00004-of-00010.gguf": [
+ 48129640000,
+ "83a630d0dd9f2dc6e1d1223b265e78523a51ba6483ab70e3ae3f188978df2491"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00005-of-00010.gguf": [
+ 48129640000,
+ "90ecd6d773dd0eab5ce1a7bdff626617f8b59360ae53a2cd5d763bde0088b2e8"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00006-of-00010.gguf": [
+ 48129640000,
+ "e688bfa9a31f00ac33ac7cd739fbc0a903ae0d1601e85c485c23147c57a282fd"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00007-of-00010.gguf": [
+ 48129640000,
+ "daf02b75c0ad3562c8ccc505fcd94ce08584f967f485b9e9e3ebdbfcafcaf286"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00008-of-00010.gguf": [
+ 101377628544,
+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00009-of-00010.gguf": [
+ 101380404256,
+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"
+ ],
+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00010-of-00010.gguf": [
+ 9165923392,
+ "82a3958d47e2c220f49d7235256a54c3a93a08f9c1bbf90c7a0c552a64c90d9a"
+ ]
+ }
+ }
+ ]
+ }
+ },
+ "maya": {
+ "_": "Maya's quants of DeepSeek-V4.1-Flash-1017 (tools/maya_quant: per-expert statistics from the model itself, error-feedback rounding of the experts), each one GGUF split with everything in it - the Engram tables too, in Q6_K (made from DeepSeek's released weights). The files are listed here once they are uploaded, with the commit they were uploaded at.",
+ "repo": "peasantsmith/DeepSeek-V4.1-Flash-1017-Maya-GGUF",
+ "revision": "276e0badcfb4a6d8f6c2b9d4da1b468cd339c221",
+ "quants": {
+ "Maya-S": {
+ "about": "experts IQ2_XXS / IQ3_XXS, the first and last four layers IQ2_S / IQ3_S, dense weights Q8_0, the Engram tables Q6_K",
+ "files": {
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00001-of-00006.gguf": [
+ 47887697312,
+ "1f3437f57042e69f94c141ffbcba0d5610dab63feb07244fb468be4940d6763f"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00002-of-00006.gguf": [
+ 47669578400,
+ "66501fee8702ffd54e6f065323ab54bb3150d20b10b01922c589e7462c7b5ba6"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00003-of-00006.gguf": [
+ 47103347360,
+ "3195b23f33337c46a07c48583946d99147c3635412bc4adc6acdc4cf42bf30ad"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00004-of-00006.gguf": [
+ 35532532032,
+ "d60dba35fac9601cd5c6026439dd980c457a5bb1c009192ea539b220108ea0db"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00005-of-00006.gguf": [
+ 80641295472,
+ "c98d46f96a25d17c4c54da5a60782c130255c364a22e621ea28d5c98a692ceb2"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00006-of-00006.gguf": [
+ 80643503412,
+ "98d65ef4f36e0b193935d44c3586241ca09b1b47e90f118e85332cfdb5860263"
+ ]
+ },
+ "earlier": [
+ {
+ "_": "v1.1.16's Maya-S: 4 shards without the Engram tables, which came from unsloth's MXFP8 shard of them beside it in <...-GGUF>/Engram (\"engram\": either of its layouts) - used as they are when both are whole, nothing downloaded again",
+ "revision": "26c6aeaf5ecce88fe47103f780279bcc6f1f0d9b",
+ "files": {
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00001-of-00004.gguf": [
+ 47887697280,
+ "b33d91333b463414fd91fbeb14a0939e3064a1abd820fa06310992da611eeb2e"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00002-of-00004.gguf": [
+ 47669578400,
+ "8198c1a5b9e2f964affacc1141627e2f0715711571c1fe04cb81702f232c5c39"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00003-of-00004.gguf": [
+ 47103347360,
+ "96b9b5e22ed60825794c377301cf2671a7a2ec463c7580b57e101f222c8ee516"
+ ],
+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00004-of-00004.gguf": [
+ 35532532032,
+ "506dd95a2ea4f17c933534d730ec72b2ef1b82eec4847770f4f23c38344f96ca"
+ ]
+ },
+ "engram": [
+ {
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00009.gguf": [
+ 202758032592,
+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"
+ ]
+ },
+ {
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00010.gguf": [
+ 101377628544,
+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"
+ ],
+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00010.gguf": [
+ 101380404256,
+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "Maya-M": {
+ "about": "experts IQ3_S, dense weights Q8_0",
+ "files": {}
+ }
+ },
+ "drafter": {
+ "_": "The DSpark drafter Maya ships: DeepSeek's FP4 experts as released, its FP8 dense weights in Q8_0 (the engine's formats: used as it is, beside the quants' folders as dspark/); the shards + drafter_pack.py path (\"drafter\" below) is the fallback while this is not pinned.",
+ "files": {
+ "dspark/dspark.bin": [
+ 7967616768,
+ "42df65159f980a9189d80597e42b30012c7b10e0dd385a47051f3b2fd659bb7b"
+ ],
+ "dspark/dspark.json": [
+ 96965,
+ "639c3144bca8cfe4db59057047dca40411f0cc732796e2a9321feb98fee2295f"
+ ]
+ }
+ }
+ },
+ "drafter": {
+ "_": "DeepSeek's DSpark drafter (the speculative decoder the GGUFs leave out): the three shards of DeepSeek's own checkpoint that hold only its blocks (mtp.0-2), at the revision the Engram tables come from; tools/ds4/drafter_pack.py makes them the GGUF path's formats (dspark.bin, beside the quants' folders).",
+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",
+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",
+ "files": {
+ "model-00044-of-00048.safetensors": [
+ 2652728736,
+ "3e8ac315c2bcca3e83c5e6e25c5758a75c03732063a6ff0c1b0e24b9ff49f6e6"
+ ],
+ "model-00045-of-00048.safetensors": [
+ 2573998176,
+ "9c73b27c4343236790008979d5e0b565f687aa64440031b9e096fed1ab8039c5"
+ ],
+ "model-00046-of-00048.safetensors": [
+ 2706402896,
+ "1dccc19a3741972c661de734572dc7d4b899e6e99bfbdf3ae4e057c707e5b339"
+ ]
+ }
+ },
+ "vision": {
+ "_": "DeepSeek's vision tower (the GGUFs leave it out; every GGUF of the model shares it, the Maya quants too): shard 1 of DeepSeek's own checkpoint holds only the tower (vision.*) and the aligner (aligner.*), BF16, and config.json their geometry; the picture span's three marker rows sit beside the embedding in shard 2, so only their 30 KB are read from it (\"marks\": an HTTP range request, each row [offset in the file, bytes, sha256]). tools/ds4/vision_pack.py makes them vision.bin + vision.json, beside the quants' folders.",
+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",
+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",
+ "files": {
+ "model-00001-of-00048.safetensors": [
+ 970533624,
+ "13bd121075f0b68cb7e8e8021a1011d42a81a83f557a2602fd9c07d52fddb0ec"
+ ],
+ "config.json": [
+ 3311,
+ "8be45ce0476004a3f529fd896115a4a2e800a129ad2d3ec05b16050f52e21879"
+ ]
+ },
+ "marks": {
+ "file": "model-00002-of-00048.safetensors",
+ "bytes": 1323858272,
+ "rows": {
+ "image_start": [
+ 1323848032,
+ 10240,
+ "5f86461a3a0565f77468091e13c0c81db83614295ad004136b987cbc430470fd"
+ ],
+ "image_newline": [
+ 1323837792,
+ 10240,
+ "50e8ecbca5409f87546c3d9f78b49b1653204cfb2a8f5068dc738cd02b3df69f"
+ ],
+ "image_end": [
+ 1323827552,
+ 10240,
+ "f55bfbcb7e06217c4822d27cf7a6696966c099f2805dbf47fc1a6d6f56a885b6"
+ ]
+ }
+ }
+ }
+}
diff --git a/tools/dsv41/drafter_pack.py b/tools/dsv41/drafter_pack.py
new file mode 100644
index 000000000..0b4394f58
--- /dev/null
+++ b/tools/dsv41/drafter_pack.py
@@ -0,0 +1,216 @@
+"""tools/ds4/drafter_pack.py - DeepSeek-V4.1's DSpark drafter (the speculative decoder the GGUFs leave out) for a GGUF
+install: its three blocks from DeepSeek's own checkpoint (the shards that hold only them, model-00044..46-of-00048),
+in the formats the GGUF path runs, as one file beside them that pack_gguf.py adds to the pack.
+
+ python tools/ds4/drafter_pack.py --dir <folder with the three shards> -> dspark.bin + dspark.json there
+
+Every value is kept exactly: the FP8 (E4M3) matrices with their E8M0 scale a 32 x 32 block become BF16 (an E4M3 value
+times a power of two has at most four significant bits); the routed experts (FP4 E2M1 codes two to a byte, the first in
+the low nibble, an E8M0 scale a 32) become ggml's MXFP4 blocks (the scale byte, then value j's code in byte j's low
+nibble and value j + 16's in its high one); BF16 and F32 tensors are copied. dspark.json lists every tensor (name,
+dtype, offset, bytes, shape) under the checkpoint's names, as tensors.txt does.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import re
+import struct
+import sys
+
+import numpy as np
+
+SHARDS = ("model-00044-of-00048.safetensors", "model-00045-of-00048.safetensors", "model-00046-of-00048.safetensors")
+ALIGN = 256
+
+
+def e4m3_table() -> np.ndarray:
+ t = np.zeros(256, np.float32)
+ for c in range(256):
+ s, e, m = c >> 7, (c >> 3) & 15, c & 7
+ v = np.nan if (e == 15 and m == 7) else (m / 8.0) * 2.0 ** -6 if e == 0 else (1 + m / 8.0) * 2.0 ** (e - 7)
+ t[c] = -v if s else v
+ return t
+
+
+E4M3 = e4m3_table()
+
+
+def shard(path: pathlib.Path):
+ raw = np.memmap(path, np.uint8, mode="r")
+ n = struct.unpack("<Q", bytes(raw[:8]))[0]
+ head = json.loads(bytes(raw[8:8 + n]))
+ head.pop("__metadata__", None)
+ return raw, head, 8 + n
+
+
+def data(raw, base: int, e: dict) -> np.ndarray:
+ a, b = e["data_offsets"]
+ return raw[base + a:base + b]
+
+
+def bf16(x: np.ndarray) -> np.ndarray:
+ """float32 -> BF16 bits, rounded to nearest even (exact here)."""
+ u = x.astype(np.float32).view(np.uint32)
+ return ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16)
+
+
+def fp8_to_bf16(w: np.ndarray, s: np.ndarray, rows: int, cols: int) -> np.ndarray:
+ w = w.reshape(rows, cols)
+ scale = np.exp2(s.reshape((rows + 31) // 32, (cols + 31) // 32).astype(np.float32) - 127.0)
+ v = E4M3[w] * np.repeat(np.repeat(scale, 32, 0), 32, 1)[:rows, :cols]
+ return bf16(v)
+
+
+def fp4_to_mxfp4(w: np.ndarray, s: np.ndarray, rows: int, k: int) -> np.ndarray:
+ nb = k // 32
+ p = w.reshape(rows, nb, 16)
+ codes = np.empty((rows, nb, 32), np.uint8)
+ codes[..., 0::2] = p & 15
+ codes[..., 1::2] = p >> 4
+ out = np.empty((rows, nb, 17), np.uint8)
+ out[..., 0] = s.reshape(rows, nb)
+ out[..., 1:] = codes[..., :16] | (codes[..., 16:] << 4)
+ return out
+
+
+QTYPES = {"Q8_0": (8, 32, 34), "Q2_K": (10, 256, 84), "Q3_K": (11, 256, 110), "Q4_K": (12, 256, 144),
+ "Q5_K": (13, 256, 176), "Q6_K": (14, 256, 210), "IQ2_XXS": (16, 256, 66), "IQ3_XXS": (18, 256, 98),
+ "IQ3_S": (21, 256, 110), "IQ2_S": (22, 256, 82), "IQ4_XS": (23, 256, 136)}
+DENSE_KEEP = ("markov_head", "confidence_head", "norm", "hc_", "ffn.gate.", "attn_sink")
+E2M1 = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
+
+
+def requant(d: pathlib.Path, name: str, experts: str, dense: str, lib: str, jobs: int, out=None) -> int:
+ """dspark.bin's tensors again with the routed experts in `experts` and the dense matrices (the ones the engine reads
+ as matrices) in `dense`, through ggml's quantizers (libggml-base): dspark-<name>.bin / .json beside it - a smaller
+ drafter (pack_gguf.py --drafter dspark-<name>)."""
+ import ctypes
+ from concurrent.futures import ThreadPoolExecutor
+ g = ctypes.CDLL(lib)
+ g.ggml_quantize_chunk.restype = ctypes.c_size_t
+ g.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,
+ ctypes.c_int64, ctypes.c_void_p]
+ src = json.loads((d / "dspark.json").read_text())
+ raw = np.memmap(d / "dspark.bin", np.uint8, mode="r")
+
+ def f32(dt, b):
+ if dt == "BF16":
+ return (np.frombuffer(b, np.uint16).astype(np.uint32) << 16).view(np.float32)
+ if dt == "MXFP4":
+ blk = np.frombuffer(b, np.uint8).reshape(-1, 17)
+ v = np.empty((blk.shape[0], 32), np.float32)
+ v[:, :16] = E2M1[blk[:, 1:] & 15]
+ v[:, 16:] = E2M1[blk[:, 1:] >> 4]
+ return (v * np.ldexp(np.float32(1), blk[:, :1].astype(np.int32) - 128)).reshape(-1)
+ raise ValueError(dt)
+
+ def quant(t, x, rows, ne0): # (one tensor on the calling thread; the pool runs tensors side by side)
+ tid, per, bpb = QTYPES[t]
+ q = np.empty(rows * ne0 // per * bpb, np.uint8)
+ ones = np.ones(ne0, np.float32)
+ g.ggml_quantize_chunk(tid, x.ctypes.data, q.ctypes.data, 0, rows, ne0, ones.ctypes.data)
+ return q
+
+ role_re = re.compile(r"(mtp\.\d+\.ffn\.experts\.)\d+\.(w[123])\.weight$")
+
+ def prep(item):
+ t, dt, so, nb, shape = item
+ b = raw[so:so + nb]
+ m = role_re.match(t)
+ role = (m.group(1), m.group(2)) if m else None
+ want = experts if role else dense
+ if (not role and (dt != "BF16" or len(shape) != 2 or shape[1] % 32 or any(k in t for k in DENSE_KEEP))) \
+ or want in ("keep", dt):
+ return t, dt, np.asarray(b), shape, role
+ if shape[1] % QTYPES[want][1]:
+ raise SystemExit("%s: rows of %d are not whole %s blocks" % (t, shape[1], want))
+ return t, want, quant(want, np.ascontiguousarray(f32(dt, b)), shape[0], shape[1]), shape, role
+
+ for t in {experts, dense} - {"keep"}:
+ g.ggml_quantize_init(QTYPES[t][0])
+ entries, off, prev = [], 0, None
+ o = pathlib.Path(out) if out else d
+ o.mkdir(parents=True, exist_ok=True)
+ items = list(src["tensors"])
+ with (o / ("dspark-%s.bin.part" % name)).open("wb") as f, ThreadPoolExecutor(jobs) as ex:
+ for i0 in range(0, len(items), 2 * jobs):
+ for t, odt, q, shape, role in ex.map(prep, items[i0:i0 + 2 * jobs]):
+ if role is None or role != prev: # (a role's experts back to back: expert e at the first's + e x size)
+ pad = (-off) % ALIGN
+ f.write(b"\0" * pad)
+ off += pad
+ f.write(q.tobytes())
+ entries.append([t, odt, off, q.nbytes, shape])
+ off += q.nbytes
+ prev = role
+ (o / ("dspark-%s.bin.part" % name)).replace(o / ("dspark-%s.bin" % name))
+ (o / ("dspark-%s.json" % name)).write_text(json.dumps({"from": "dspark.bin", "experts": experts, "dense": dense,
+ "tensors": entries}))
+ print("dspark-%s.bin: %d tensors, %.2f GB (experts %s, dense %s)" % (name, len(entries), off / 1e9, experts, dense))
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--dir", required=True, help="the folder with the drafter's three shards")
+ ap.add_argument("--requant", metavar="NAME",
+ help="from dspark.bin there: dspark-NAME.bin with --experts / --dense types (ggml's quantizers)")
+ ap.add_argument("--experts", default="IQ4_XS", help="(--requant) the routed experts' type, or keep")
+ ap.add_argument("--dense", default="Q8_0", help="(--requant) the dense matrices' type, or keep")
+ ap.add_argument("--lib", default=str(pathlib.Path("~/llama.cpp/build/bin/libggml-base.so").expanduser()),
+ help="(--requant) libggml-base")
+ ap.add_argument("--jobs", type=int, default=8)
+ ap.add_argument("--out", help="(--requant) the folder to write to (default: --dir)")
+ a = ap.parse_args()
+ d = pathlib.Path(a.dir)
+ if a.requant:
+ return requant(d, a.requant, a.experts, a.dense, a.lib, a.jobs, a.out)
+ entries, off = [], 0
+ exp = re.compile(r"(mtp\.\d+\.ffn\.experts\.)(\d+)\.(w[123])\.weight$")
+
+ def order(head):
+ """The block's tensors, then each role's experts (w1, w3, w2) back to back in expert order - the engine reads
+ expert e of a role at the first one's offset + e x its size, as the GGUF stacks them."""
+ plain = sorted(t for t in head if t.startswith("mtp.") and not t.endswith(".scale") and not exp.match(t))
+ ex = [exp.match(t) for t in head if exp.match(t)]
+ roles = sorted({(m.group(1), m.group(3)) for m in ex}, key=lambda pr: (pr[0], "w1w3w2".index(pr[1])))
+ n = {pr: 1 + max(int(m.group(2)) for m in ex if (m.group(1), m.group(3)) == pr) for pr in roles}
+ return [(t, False) for t in plain] + [("%s%d.%s.weight" % (p, i, r), i < n[(p, r)] - 1)
+ for p, r in roles for i in range(n[(p, r)])]
+
+ with (d / "dspark.bin.part").open("wb") as f:
+ for name in SHARDS:
+ raw, head, base = shard(d / name)
+ for t, packed in order(head):
+ e, shape = head[t], list(head[t]["shape"])
+ if e["dtype"] == "F8_E4M3":
+ s = head[t[:-len("weight")] + "scale"]
+ out, dt = fp8_to_bf16(data(raw, base, e), data(raw, base, s), shape[0], shape[1]), "BF16"
+ elif e["dtype"] == "I8":
+ s = head[t[:-len("weight")] + "scale"]
+ shape = [shape[0], shape[1] * 2]
+ out, dt = fp4_to_mxfp4(data(raw, base, e), data(raw, base, s), shape[0], shape[1]), "MXFP4"
+ elif e["dtype"] in ("BF16", "F32"):
+ out, dt = data(raw, base, e), e["dtype"]
+ else:
+ raise SystemExit(f"{name}: {t} is {e['dtype']}, not a type the drafter's pack covers")
+ b = out.tobytes()
+ f.write(b)
+ entries.append([t, dt, off, len(b), shape])
+ off += len(b)
+ if packed: # (the next expert of the role follows at once)
+ continue
+ pad = (-off) % ALIGN
+ f.write(b"\0" * pad)
+ off += pad
+ del raw
+ (d / "dspark.bin.part").replace(d / "dspark.bin")
+ (d / "dspark.json").write_text(json.dumps({"from": list(SHARDS), "tensors": entries}))
+ print(f"dspark.bin: {len(entries)} tensors, {off / 1e9:.2f} GB")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/dspark_ref.py b/tools/dsv41/dspark_ref.py
new file mode 100644
index 000000000..e27485813
--- /dev/null
+++ b/tools/dsv41/dspark_ref.py
@@ -0,0 +1,387 @@
+"""tools/ds4/dspark_ref.py - DeepSeek-V4.1's DSpark drafter in torch, straight from DeepSeek's checkpoint, against what
+the engine's drafter computed (the engine's oracle for the drafter; ref.py / ref_gguf.py cover the trunk only).
+
+ DS4_DSPARK_DUMP=d.bin maya-ds4 --pack <pack> ... --n 64 (the engine: its drafter's inputs and outputs)
+ python tools/ds4/dspark_ref.py --dump d.bin --shards <dir with model-00044..46-of-00048> --pack <pack>
+ [--rounds 4] [--head]
+
+It follows DeepSeek's inference/model.py (MIT) - DSparkBlock / DSparkAttention / forward_spec: the taps of target
+layers 37..39 (the streams' mean entering them) through main_proj / main_norm into every block's window ring (wkv,
+kv_norm, rope, FP8 rounding), the block [y, noise x 4] at the positions after the ring's through the three blocks
+(Hyper-Connections, window attention over the ring and the whole block, the 128-expert MoE with 3 picks), the final
+collapse and, with --head, the trunk's head plus the markov bias (the drafts) - with every weight decoded here from
+the checkpoint's own FP8 / FP4 / BF16 tensors (not from drafter_pack.py's dspark.bin), activations F32. Per round
+it prints how far the engine's hidden rows and confidences are from it, and with --head whether the drafts agree.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import pathlib
+import struct
+import sys
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+SHARDS = ("model-00044-of-00048.safetensors", "model-00045-of-00048.safetensors", "model-00046-of-00048.safetensors")
+
+
+def e4m3_table() -> np.ndarray:
+ t = np.zeros(256, np.float32)
+ for c in range(256):
+ s, e, m = c >> 7, (c >> 3) & 15, c & 7
+ v = np.nan if (e == 15 and m == 7) else (m / 8.0) * 2.0 ** -6 if e == 0 else (1 + m / 8.0) * 2.0 ** (e - 7)
+ t[c] = -v if s else v
+ return t
+
+
+E4M3 = e4m3_table()
+E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], np.float32)
+
+
+class Ckpt:
+ """The drafter's shards: each tensor decoded to F32 on request (FP8 with its 32 x 32 E8M0 block scales; FP4 codes
+ two to a byte, the first in the low nibble, an E8M0 scale per 32 along K; BF16; F32)."""
+
+ def __init__(self, d: pathlib.Path):
+ self.t = {}
+ for name in SHARDS:
+ raw = np.memmap(d / name, np.uint8, mode="r")
+ n = struct.unpack("<Q", bytes(raw[:8]))[0]
+ head = json.loads(bytes(raw[8:8 + n]))
+ head.pop("__metadata__", None)
+ for k, e in head.items():
+ self.t[k] = (raw, 8 + n, e)
+
+ def raw(self, k):
+ raw, base, e = self.t[k]
+ a, b = e["data_offsets"]
+ return raw[base + a:base + b], e["dtype"], e["shape"]
+
+ def get(self, k) -> torch.Tensor:
+ w, dt, sh = self.raw(k)
+ if dt == "F32":
+ v = np.frombuffer(w, np.float32).reshape(sh)
+ elif dt == "BF16":
+ v = (np.frombuffer(w, np.uint16).astype(np.uint32) << 16).view(np.float32).reshape(sh)
+ elif dt == "F8_E4M3":
+ s, _, ssh = self.raw(k[:-len("weight")] + "scale")
+ r, c = sh
+ sc = np.exp2(np.asarray(s, np.float32).reshape(ssh) - 127.0)
+ v = E4M3[np.asarray(w).reshape(r, c)] * np.repeat(np.repeat(sc, 32, 0), 32, 1)[:r, :c]
+ elif dt == "I8":
+ s, _, ssh = self.raw(k[:-len("weight")] + "scale")
+ r, c2 = sh
+ p = np.asarray(w).reshape(r, c2)
+ codes = np.empty((r, c2 * 2), np.uint8)
+ codes[:, 0::2] = p & 15
+ codes[:, 1::2] = p >> 4
+ sc = np.exp2(np.asarray(s, np.float32).reshape(ssh) - 127.0)
+ v = E2M1[codes] * np.repeat(sc, 32, 1)
+ else:
+ raise SystemExit("%s: dtype %s" % (k, dt))
+ return torch.from_numpy(np.ascontiguousarray(v, np.float32))
+
+
+def fp8_round(x: torch.Tensor, block: int = 32) -> torch.Tensor:
+ sh = x.shape
+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)
+ amax = v.abs().amax(-1, keepdim=True).clamp_min(1e-4)
+ s = torch.exp2(torch.ceil(torch.log2(amax / 448.0)))
+ y = (v / s).clamp(-448.0, 448.0).to(torch.float8_e4m3fn).float() * s
+ return y.reshape(sh)
+
+
+def rms(x, w, eps):
+ x = x.float()
+ y = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps)
+ return y * w if w is not None else y
+
+
+def freqs(dim, n, base):
+ f = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+ a = torch.outer(torch.arange(n, dtype=torch.float32), f)
+ return torch.polar(torch.ones_like(a), a)
+
+
+def rope(x, fc, inverse=False):
+ c = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2).contiguous())
+ if inverse:
+ fc = fc.conj()
+ if x.ndim == 3:
+ fc = fc[:, None, :]
+ return torch.view_as_real(c * fc).flatten(-2)
+
+
+def pack_conf(pack: pathlib.Path):
+ cfg = {}
+ for line in (pack / "ds4.txt").read_text().splitlines():
+ k, _, v = line.partition(" ")
+ cfg[k] = v
+ files, tens = {}, {}
+ for line in (pack / "tensors.txt").read_text().splitlines():
+ p = line.split()
+ if p[0] == "file":
+ files[int(p[1])] = p[2]
+ elif p[0] == "t":
+ tens[p[1]] = (p[2], int(p[3]), int(p[4]), int(p[5]), [int(x) for x in p[6:]])
+ return cfg, files, tens
+
+
+def gguf_rows(pack, cfg, files, tens, name):
+ dt, fi, off, nb, sh = tens[name]
+ if dt != "BF16":
+ raise SystemExit("%s: %s, the oracle reads a BF16 one" % (name, dt))
+ path = pathlib.Path(cfg["model_dir"]) / files[fi]
+ return np.memmap(path, np.uint16, mode="r", offset=off, shape=tuple(sh))
+
+
+def bf16_rows(m, idx):
+ return torch.from_numpy((np.asarray(m[idx]).astype(np.uint32) << 16).view(np.float32))
+
+
+class Drafter:
+ def __init__(self, ck: Ckpt, cfg: dict):
+ self.ck, self.cfg = ck, cfg
+ self.D = int(cfg["hidden_size"])
+ self.hc = int(cfg["hc_mult"])
+ self.eps = float(cfg["rms_norm_eps"])
+ self.hc_eps = float(cfg["hc_eps"])
+ self.sink_iters = int(cfg["hc_sinkhorn_iters"])
+ self.H, self.HD, self.rd = int(cfg["num_attention_heads"]), int(cfg["head_dim"]), int(cfg["qk_rope_head_dim"])
+ self.G = int(cfg["o_groups"])
+ self.win = int(cfg["sliding_window"])
+ self.k = int(cfg["dspark_num_experts_per_tok"])
+ self.scale = float(cfg["routed_scaling_factor"])
+ self.lim = float(cfg["swiglu_limit"])
+ self.fc = freqs(self.rd, 1 << 16, float(cfg["rope_theta"]))
+ self.nb = 3
+ self.cache = {}
+ self.ring = [dict() for _ in range(self.nb)] # position -> main KV row, per block
+
+ def W(self, k):
+ if k not in self.cache:
+ self.cache[k] = self.ck.get(k)
+ return self.cache[k]
+
+ def seed(self, p0, taps): # taps [n, 3 * D]
+ main_x = rms(taps @ self.W("mtp.0.main_proj.weight").T, self.W("mtp.0.main_norm.weight"), self.eps)
+ n = taps.shape[0]
+ fc = self.fc[p0:p0 + n]
+ for b in range(self.nb):
+ A = "mtp.%d.attn." % b
+ kv = rms(main_x @ self.W(A + "wkv.weight").T, self.W(A + "kv_norm.weight"), self.eps)
+ kv = torch.cat([kv[:, :-self.rd], rope(kv[:, -self.rd:], fc)], -1)
+ kv = fp8_round(kv)
+ for i in range(n):
+ self.ring[b][p0 + i] = kv[i]
+
+ def hc_mixes(self, x, P):
+ hc = self.hc
+ fn, scale, base = self.W(P + "_fn"), self.W(P + "_scale"), self.W(P + "_base")
+ xf = x.flatten(1).float()
+ r = torch.rsqrt(xf.square().mean(-1, keepdim=True) + self.eps)
+ mixes = (xf @ fn.T) * r
+ pre = torch.sigmoid(mixes[:, :hc] * scale[0] + base[:hc]) + self.hc_eps
+ post = 2 * torch.sigmoid(mixes[:, hc:2 * hc] * scale[1] + base[hc:2 * hc])
+ comb = (mixes[:, 2 * hc:] * scale[2] + base[2 * hc:]).view(-1, hc, hc)
+ eps = self.hc_eps
+ comb = comb.softmax(-1) + eps
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
+ for _ in range(self.sink_iters - 1):
+ comb = comb / (comb.sum(-1, keepdim=True) + eps)
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
+ return pre, post, comb
+
+ def attention(self, b, x, p0):
+ A = "mtp.%d.attn." % b
+ T = x.shape[0]
+ H, D, rd = self.H, self.HD, self.rd
+ fc = self.fc[p0:p0 + T]
+ qr = rms(x @ self.W(A + "wq_a.weight").T, self.W(A + "q_norm.weight"), self.eps)
+ q = (qr @ self.W(A + "wq_b.weight").T).view(T, H, D)
+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)
+ kv = rms(x @ self.W(A + "wkv.weight").T, self.W(A + "kv_norm.weight"), self.eps)
+ kv = torch.cat([kv[:, :-rd], rope(kv[:, -rd:], fc)], -1)
+ kv = fp8_round(kv)
+ ring = [self.ring[b][p] for p in range(max(0, p0 - self.win), p0) if p in self.ring[b]]
+ allkv = torch.cat([torch.stack(ring), kv], 0) if ring else kv
+ sink = self.W(A + "attn_sink")
+ s = torch.einsum("thd,kd->thk", q, allkv) * D ** -0.5
+ m = torch.maximum(s.amax(-1), sink[None, :])
+ pr = torch.exp(s - m.unsqueeze(-1))
+ den = pr.sum(-1) + torch.exp(sink[None, :] - m)
+ o = torch.einsum("thk,kd->thd", pr, allkv) / den.unsqueeze(-1)
+ o = torch.cat([o[..., :-rd], rope(o[..., -rd:], fc, inverse=True)], -1)
+ o = fp8_round(o.reshape(T, self.G, -1))
+ wo_a = self.W(A + "wo_a.weight").view(self.G, -1, o.shape[-1])
+ oa = torch.einsum("tgd,grd->tgr", o, wo_a)
+ return oa.flatten(1) @ self.W(A + "wo_b.weight").T
+
+ def moe(self, b, x):
+ M = "mtp.%d.ffn." % b
+ T = x.shape[0]
+ scores = F.softplus(x.float() @ self.W(M + "gate.weight").T).sqrt()
+ idx = (scores + self.W(M + "gate.bias")).topk(self.k, dim=-1)[1]
+ wts = scores.gather(1, idx)
+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * self.scale
+ lim = self.lim
+ y = torch.zeros(T, self.D)
+ for e in torch.unique(idx).tolist():
+ P = M + "experts.%d." % e
+ rows, slot = torch.where(idx == e)
+ xs = x[rows]
+ w1, w3 = self.ck.get(P + "w1.weight"), self.ck.get(P + "w3.weight")
+ h = F.silu((xs @ w1.T).clamp(max=lim)) * (xs @ w3.T).clamp(-lim, lim)
+ del w1, w3
+ w2 = self.ck.get(P + "w2.weight")
+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ w2.T)
+ del w2
+ S = M + "shared_experts."
+ h = F.silu((x @ self.W(S + "w1.weight").T).clamp(max=lim)) * (x @ self.W(S + "w3.weight").T).clamp(-lim, lim)
+ return y + h @ self.W(S + "w2.weight").T, idx
+
+ def block_forward(self, emb, p0): # emb [T, D] -> the collapsed hidden rows [T, D], each block's stream
+ T = emb.shape[0]
+ h = emb.unsqueeze(1).repeat(1, self.hc, 1)
+ pre = torch.zeros(T, self.hc)
+ pre[:, 0] = 1.0
+ routes = []
+ streams = []
+ for b in range(self.nb):
+ B = "mtp.%d." % b
+ res = h
+ a_pre, a_post, a_comb = self.hc_mixes(h, B + "hc_attn")
+ x = rms((pre.unsqueeze(-1) * h).sum(1), self.W(B + "attn_norm.weight"), self.eps)
+ y = self.attention(b, x, p0)
+ h = a_post.unsqueeze(-1) * y.unsqueeze(1) + (a_comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)
+ res = h
+ f_pre, f_post, f_comb = self.hc_mixes(h, B + "hc_ffn")
+ x = rms((a_pre.unsqueeze(-1) * h).sum(1), self.W(B + "ffn_norm.weight"), self.eps)
+ y, idx = self.moe(b, x)
+ routes.append(idx)
+ h = f_post.unsqueeze(-1) * y.unsqueeze(1) + (f_comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)
+ pre = f_pre
+ streams.append(h)
+ return (pre.unsqueeze(-1) * h).sum(1), routes, streams
+
+
+def records(path, nb, hc, win, hd):
+ """The engine's dump: 'S' seeds (first position, the taps), 'D' draft rounds (the block's position, y, the drafts,
+ their confidences, the collapsed hidden rows, then per block the ring it read (BF16) and the stream it left)."""
+ with open(path, "rb") as f:
+ while True:
+ hb = f.read(32)
+ if len(hb) < 32:
+ return
+ kind, a, b, c = struct.unpack("<4q", hb)
+ if kind == ord("S"):
+ n, w = b, c
+ taps = np.frombuffer(f.read(n * w * 4), np.float32).reshape(n, w)
+ yield "S", a, taps
+ elif kind == ord("D"):
+ T = c
+ D = int(DIM[0])
+ toks = np.frombuffer(f.read(T * 4), np.int32)
+ conf = np.frombuffer(f.read(T * 4), np.float32)
+ x = np.frombuffer(f.read(T * 4 * D), np.float32).reshape(T, D)
+ rings, streams = [], []
+ for _ in range(nb):
+ r = np.frombuffer(f.read(win * hd * 2), np.uint16).reshape(win, hd)
+ rings.append((r.astype(np.uint32) << 16).view(np.float32))
+ streams.append(np.frombuffer(f.read(T * hc * D * 4), np.float32).reshape(T, hc, D))
+ yield "D", (a, b, T, toks, conf, x, rings, streams), None
+ else:
+ raise SystemExit("dump: record %d" % kind)
+
+
+DIM = [5120]
+
+
+def rel(a: torch.Tensor, b: torch.Tensor) -> float:
+ return float((a - b).norm() / b.norm().clamp_min(1e-30))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--dump", required=True)
+ ap.add_argument("--shards", required=True, help="the folder with DeepSeek's model-00044..46-of-00048")
+ ap.add_argument("--pack", required=True, help="the maya-ds4 pack (ds4.txt, tensors.txt: the embedding, the head)")
+ ap.add_argument("--rounds", type=int, default=4, help="draft rounds to check (from --skip)")
+ ap.add_argument("--skip", type=int, default=0, help="draft rounds to pass over first")
+ ap.add_argument("--head", action="store_true", help="also the drafts: the trunk's head + the markov bias")
+ ap.add_argument("--threads", type=int, default=4)
+ a = ap.parse_args()
+ torch.set_num_threads(a.threads)
+ pack = pathlib.Path(a.pack)
+ cfg, files, tens = pack_conf(pack)
+ DIM[0] = int(cfg["hidden_size"])
+ ck = Ckpt(pathlib.Path(a.shards))
+ dr = Drafter(ck, cfg)
+ emb = gguf_rows(pack, cfg, files, tens, "embed.weight")
+ head = gguf_rows(pack, cfg, files, tens, "head.weight") if a.head else None
+ noise = int(cfg["dspark_noise_token_id"])
+ seen = done = 0
+ worst_x = worst_c = 0.0
+ agree = total = 0
+ for kind, v, taps in records(a.dump, dr.nb, dr.hc, dr.win, dr.HD):
+ if kind == "S":
+ dr.seed(v, torch.from_numpy(taps.copy()))
+ continue
+ p0, y, T, toks, conf, xd, rings, streams = v
+ seen += 1
+ if seen <= a.skip:
+ continue
+ ex = torch.from_numpy(xd.copy())
+ ids = [y] + [noise] * (T - 1)
+ xh, routes, ref_streams = dr.block_forward(bf16_rows(emb, ids), p0)
+ for b in range(dr.nb): # the ring each block read (the positions the reference holds), the stream it left
+ ps = [p for p in range(max(0, p0 - dr.win), p0) if p in dr.ring[b]]
+ er = torch.from_numpy(rings[b][[p % dr.win for p in ps]].copy())
+ rr = torch.stack([dr.ring[b][p] for p in ps])
+ print(" block %d: ring rel err %.4f (%d rows), stream rel err %.4f, routes %s" % (
+ b, rel(er, rr), len(ps), rel(torch.from_numpy(streams[b].copy()), ref_streams[b]),
+ routes[b].tolist()), flush=True)
+ prev = [y] + [int(t) for t in toks[:-1]]
+ me = dr.W("mtp.2.markov_head.embed.weight")[prev]
+ cw = dr.W("mtp.2.confidence_head.proj.weight")[0]
+ cf = torch.sigmoid(torch.cat([xh, me], -1) @ cw)
+ rerr = ((ex - xh).norm(dim=-1) / xh.norm(dim=-1)).tolist()
+ cos = F.cosine_similarity(ex, xh, dim=-1).tolist()
+ dc = (torch.from_numpy(conf.copy()) - cf).abs().tolist()
+ worst_x = max(worst_x, max(rerr))
+ worst_c = max(worst_c, max(dc))
+ line = "round %d (pos %d, y %d): hidden rel err %s, cos %s; conf engine %s ref %s" % (
+ seen, p0, y, " ".join("%.4f" % r for r in rerr), " ".join("%.5f" % c for c in cos),
+ " ".join("%.3f" % c for c in conf.tolist()), " ".join("%.3f" % c for c in cf.tolist()))
+ if head is not None:
+ xn = rms(xh, dr.W("mtp.2.norm.weight"), dr.eps)
+ lg = torch.zeros(T, head.shape[0])
+ for c0 in range(0, head.shape[0], 16384):
+ c1 = min(head.shape[0], c0 + 16384)
+ lg[:, c0:c1] = xn @ bf16_rows(head, slice(c0, c1)).T
+ mh = dr.W("mtp.2.markov_head.head.weight")
+ ref_t = []
+ p = y
+ for i in range(T): # greedy chain on the reference's own drafts
+ row = lg[i] + mh @ dr.W("mtp.2.markov_head.embed.weight")[p]
+ p = int(row.argmax())
+ ref_t.append(p)
+ same = sum(int(r == int(t)) for r, t in zip(ref_t, toks))
+ agree += same
+ total += T
+ line += "; drafts engine %s ref %s" % (list(map(int, toks)), ref_t)
+ print(line, flush=True)
+ done += 1
+ if done >= a.rounds:
+ break
+ print("checked %d rounds: worst hidden rel err %.4f, worst conf diff %.4f%s" % (
+ done, worst_x, worst_c, (", drafts agree %d/%d" % (agree, total)) if head is not None else ""))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/dumpcmp.py b/tools/dsv41/dumpcmp.py
new file mode 100644
index 000000000..1af24c526
--- /dev/null
+++ b/tools/dsv41/dumpcmp.py
@@ -0,0 +1,184 @@
+"""dumpcmp.py - two maya-ds4 runs' debug dumps side by side (the prompt path's numerics: where two runs part).
+
+ layers A B [--pos P] DS4_DUMP_L files: each layer record's relative error between the runs - the median over
+ the positions, the worst position, and position P's
+ route R --pos P a DS4_DUMP_R file: position P's experts a layer (virtual slot, the expert's rows, P's row)
+ gate R... --pos P DS4_DUMP_R files: position P's router a layer - its picks and the margin between the k-th
+ pick's score and the next expert's (sqrt(softplus(logit)) + bias)
+ sets A B --pos P --route RA RB
+ DS4_DUMP_Y files (and each run's DS4_DUMP_R): each layer's set rows' h / y error between
+ the runs (rows matched by token and expert), the worst row and position P's, with their
+ expert's rows and the piece they were computed in
+"""
+import argparse
+import sys
+
+import numpy as np
+
+
+def read_l(path, dim_hc):
+ out = []
+ with open(path, "rb") as f:
+ while True:
+ h = f.read(24)
+ if len(h) < 24:
+ break
+ il, p0, t = (int(v) for v in np.frombuffer(h, dtype=np.int64))
+ v = np.frombuffer(f.read(t * dim_hc * 4), dtype=np.float32).reshape(t, dim_hc)
+ out.append(((il, p0), v))
+ return out
+
+
+def rel(a, b):
+ a = a.astype(np.float64)
+ b = b.astype(np.float64)
+ return np.linalg.norm(a - b, axis=-1) / np.maximum(np.linalg.norm(b, axis=-1), 1e-30)
+
+
+def read_r(path):
+ out = {}
+ with open(path, "rb") as f:
+ while True:
+ h = f.read(64)
+ if len(h) < 64:
+ break
+ il, p0, t, k, ne, nvs, nsl, kp = (int(v) for v in np.frombuffer(h, dtype=np.int64))
+ ri = np.frombuffer(f.read(t * kp * 4), dtype=np.int32).reshape(t, kp)[:, :k]
+ vs = np.frombuffer(f.read(ne * 4), dtype=np.int32)
+ b = np.frombuffer(f.read((nvs + 1) * 4), dtype=np.int32)
+ rt = np.frombuffer(f.read(t * k * 4), dtype=np.int32)
+ gl = np.frombuffer(f.read(t * ne * 4), dtype=np.float32).reshape(t, ne)
+ gb = np.frombuffer(f.read(ne * 4), dtype=np.float32)
+ out[(il, p0)] = dict(t=t, k=k, ri=ri, vs=vs, b=b, rt=rt, nsl=nsl, glog=gl, bias=gb)
+ return out
+
+
+def read_y(path, ff, dim): # a layer's pieces at a time (the dump holds the layers in order)
+ cur, il0 = [], None
+ with open(path, "rb") as f:
+ while True:
+ h = f.read(48)
+ if len(h) < 48:
+ break
+ il, a, n, ne, ring, mr = (int(v) for v in np.frombuffer(h, dtype=np.int64))
+ b = np.frombuffer(f.read((ne + 1) * 4), dtype=np.int32)
+ hv = np.frombuffer(f.read(n * ff * 2), dtype=np.float16).reshape(n, ff)
+ yv = np.frombuffer(f.read(n * dim * 4), dtype=np.float32).reshape(n, dim)
+ if il0 is not None and il != il0:
+ yield il0, cur
+ cur = []
+ il0 = il
+ cur.append(dict(il=il, a=a, n=n, ne=ne, ring=ring, mr=mr, b=b, h=hv, y=yv))
+ if cur:
+ yield il0, cur
+
+
+def row_keys(r):
+ """A layer's sorted rows -> (token, expert, the expert's rows, the row's place among them)."""
+ keys = {}
+ for e, v in enumerate(r["vs"]):
+ lo, hi = int(r["b"][v]), int(r["b"][v + 1])
+ for g in range(lo, hi):
+ keys[g] = (int(r["rt"][g]), e, hi - lo, g - lo)
+ return keys
+
+
+def sets(a):
+ """Each layer: the set rows' y / h error between run A and run B, a row matched to the other run's by its (token,
+ expert) - DS4_DUMP_R of each run (--route RA RB: the routing may differ) - the median, the worst row, position P's
+ rows, with the expert's rows, the row's place in them and the piece it was computed in (row0+rows/experts x max)."""
+ RA, RB = (read_r(f) for f in a.route)
+ YB = read_y(a.files[1], a.ff, a.dim)
+ print("%-3s %8s | %-52s | %s" % ("L", "median y", "worst: y err, h err, token, expert rows+place, ring, piece",
+ "pos %d: y err, h err, expert rows+place, ring, piece" % a.pos))
+ for il, pa in read_y(a.files[0], a.ff, a.dim):
+ ilb, pb = next(YB)
+ assert ilb == il, (il, ilb)
+ ka = [row_keys(r) for (l2, _), r in RA.items() if l2 == il]
+ kb = [row_keys(r) for (l2, _), r in RB.items() if l2 == il]
+ if len(ka) != 1 or len(kb) != 1: # (one chunk: p0 0)
+ continue
+ ka, kb = ka[0], kb[0]
+ rows = {}
+ for d in pb:
+ for i in range(d["n"]):
+ t, e, _, _ = kb[d["a"] + i]
+ rows[(t, e)] = (d, i)
+ errs = []
+ for d in pa:
+ for i in range(d["n"]):
+ t, e, n, pl = ka[d["a"] + i]
+ if (t, e) not in rows:
+ continue
+ db, ib = rows[(t, e)]
+ eh = float(rel(d["h"][i:i + 1].astype(np.float32), db["h"][ib:ib + 1].astype(np.float32))[0])
+ ey = float(rel(d["y"][i:i + 1], db["y"][ib:ib + 1])[0])
+ errs.append((ey, eh, t, e, n, pl, d["ring"], "%d+%d/%dx%d" % (d["a"], d["n"], d["ne"], d["mr"])))
+ if not errs:
+ continue
+ w = max(errs)
+ mine = sorted((x for x in errs if x[2] == a.pos), reverse=True)
+ print("%-3d %8.1e | %7.1e %7.1e t%-4d e%-3d %3d+%-3d %d %-16s | %s" % (
+ il, float(np.median([x[0] for x in errs])), w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7],
+ "; ".join("%.1e %.1e e%d %d+%d %d %s" % (x[0], x[1], x[3], x[4], x[5], x[6], x[7]) for x in mine)))
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("mode", choices=["layers", "route", "sets", "gate"])
+ ap.add_argument("files", nargs="+")
+ ap.add_argument("--pos", type=int, default=-1)
+ ap.add_argument("--dim", type=int, default=5120)
+ ap.add_argument("--hc", type=int, default=4)
+ ap.add_argument("--ff", type=int, default=2304)
+ ap.add_argument("--route", nargs=2, metavar=("RA", "RB"))
+ a = ap.parse_args()
+ if a.mode == "layers":
+ A = read_l(a.files[0], a.dim * a.hc)
+ B = dict(read_l(a.files[1], a.dim * a.hc))
+ print("%-12s %10s %10s %6s %10s" % ("record", "median", "worst", "at", "pos %d" % a.pos))
+ for key, va in A:
+ if key not in B:
+ continue
+ e = rel(va, B[key])
+ w = int(np.argmax(e))
+ p = a.pos - key[1]
+ ep = "%.3e" % e[p] if 0 <= p < len(e) else "-"
+ print("%-12s %10.3e %10.3e %6d %10s" % ("L%d@%d" % key, float(np.median(e)), float(e[w]), w + key[1], ep))
+ return
+ if a.mode == "gate":
+ for f in a.files: # each run's router at position P: the picks and the margin past the k-th
+ print("== %s" % f)
+ for (il, p0), r in sorted(read_r(f).items()):
+ p = a.pos - p0
+ if not 0 <= p < r["t"]:
+ continue
+ lg = r["glog"][p].astype(np.float64)
+ sc = np.sqrt(np.where(lg > 20, lg, np.log1p(np.exp(np.minimum(lg, 20)))))
+ sel = sc + r["bias"]
+ o = np.argsort(-sel, kind="stable")
+ k = r["k"]
+ print("L%-3d picks %s | k-th %.5f, next e%d %.5f: margin %.2e" % (
+ il, " ".join("e%d" % e for e in o[:k]), sel[o[k - 1]], o[k], sel[o[k]], sel[o[k - 1]] - sel[o[k]]))
+ return
+ if a.mode == "route":
+ R = read_r(a.files[0])
+ for (il, p0), r in sorted(R.items()):
+ p = a.pos - p0
+ if not 0 <= p < r["t"]:
+ continue
+ rows = np.nonzero(r["rt"] == p)[0]
+ parts = []
+ for e in r["ri"][p]:
+ v = int(r["vs"][e])
+ lo, hi = int(r["b"][v]), int(r["b"][v + 1])
+ mine = [int(x) for x in rows if lo <= x < hi]
+ parts.append("e%d v%d%s rows %d (row %s, %d into it)" % (e, v, "" if v < r["nsl"] else "*", hi - lo,
+ mine[0] if mine else "?", (mine[0] - lo) if mine else -1))
+ print("L%d@%d: %s" % (il, p0, "; ".join(parts)))
+ return
+ sets(a)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/dumpscan.py b/tools/dsv41/dumpscan.py
new file mode 100644
index 000000000..2c799cbd7
--- /dev/null
+++ b/tools/dsv41/dumpscan.py
@@ -0,0 +1,42 @@
+"""dumpscan.py DUMP_L [--pos P] [--dim 5120] [--hc 4] - one run's DS4_DUMP_L records: each layer's residual stream at
+position P against the other positions (its norm over the median norm, its largest |value| and where, non-finite
+values) - where a position's stream blows up or goes non-finite."""
+import argparse
+import sys
+
+import numpy as np
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("dump")
+ ap.add_argument("--pos", type=int, default=0)
+ ap.add_argument("--dim", type=int, default=5120)
+ ap.add_argument("--hc", type=int, default=4)
+ a = ap.parse_args()
+ n = a.dim * a.hc
+ print("%-10s %10s %10s %12s %10s %10s %s" % ("record", "|h| pos", "median", "pos / med", "max|v| pos", "max|v| all",
+ "non-finite (pos / all)"))
+ with open(a.dump, "rb") as f:
+ while True:
+ h = f.read(24)
+ if len(h) < 24:
+ break
+ il, p0, t = (int(v) for v in np.frombuffer(h, dtype=np.int64))
+ v = np.frombuffer(f.read(t * n * 4), dtype=np.float32).reshape(t, n)
+ p = a.pos - p0
+ if not 0 <= p < t:
+ continue
+ fin = np.isfinite(v)
+ vv = np.where(fin, v, 0).astype(np.float64)
+ nr = np.linalg.norm(vv, axis=1)
+ med = float(np.median(nr))
+ mx = np.abs(vv[p])
+ j = int(np.argmax(mx))
+ print("L%-4d@%-3d %10.3e %10.3e %12.2f %10.3e %10.3e %d / %d (max at stream %d dim %d)" % (
+ il, p0, nr[p], med, nr[p] / max(med, 1e-30), mx[j], float(np.abs(vv).max()),
+ int((~fin[p]).sum()), int((~fin).sum()), j // a.dim, j % a.dim))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/dumpscan_y.py b/tools/dsv41/dumpscan_y.py
new file mode 100644
index 000000000..bb8cbae5e
--- /dev/null
+++ b/tools/dsv41/dumpscan_y.py
@@ -0,0 +1,48 @@
+"""dumpscan_y.py DUMP_Y DUMP_R --pos P [--layers a-b] - one run's expert set rows (DS4_DUMP_Y, DS4_DUMP_R): position
+P's rows a layer - its experts, their rows and P's place in them, |h| and |y| against the layer's median row."""
+import argparse
+import sys
+
+import numpy as np
+
+sys.path.insert(0, __file__.rsplit("/", 1)[0] if "/" in __file__ else ".")
+from dumpcmp import read_r, read_y, row_keys # noqa: E402
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("dump_y")
+ ap.add_argument("dump_r")
+ ap.add_argument("--pos", type=int, required=True)
+ ap.add_argument("--layers", default="0-99")
+ ap.add_argument("--dim", type=int, default=5120)
+ ap.add_argument("--ff", type=int, default=2304)
+ a = ap.parse_args()
+ l0, l1 = (int(v) for v in a.layers.split("-"))
+ R = read_r(a.dump_r)
+ for il, pieces in read_y(a.dump_y, a.ff, a.dim):
+ if not l0 <= il <= l1:
+ continue
+ r = [v for (l2, _), v in R.items() if l2 == il]
+ if len(r) != 1:
+ continue
+ keys = row_keys(r[0])
+ hn, yn, mine = [], [], []
+ for d in pieces:
+ for i in range(d["n"]):
+ t, e, n, pl = keys[d["a"] + i]
+ h = float(np.linalg.norm(d["h"][i].astype(np.float64)))
+ y = float(np.linalg.norm(d["y"][i].astype(np.float64)))
+ hm = float(np.abs(d["h"][i].astype(np.float32)).max())
+ hn.append(h)
+ yn.append(y)
+ if t == a.pos:
+ mine.append((e, n, pl, h, hm, y, d["ring"], "%d+%d/%dx%d" % (d["a"], d["n"], d["ne"], d["mr"])))
+ mh, my = float(np.median(hn)), float(np.median(yn))
+ print("L%-3d median |h| %.3e |y| %.3e; pos %d: %s" % (il, mh, my, a.pos, "; ".join(
+ "e%d (%d rows, #%d) |h| %.2e max %.2e |y| %.2e (x%.1f) ring %d %s" % (e, n, pl, h, hm, y, y / max(my, 1e-30), rg, pc)
+ for e, n, pl, h, hm, y, rg, pc in mine)))
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/engram_hash.py b/tools/dsv41/engram_hash.py
new file mode 100644
index 000000000..43b861d94
--- /dev/null
+++ b/tools/dsv41/engram_hash.py
@@ -0,0 +1,142 @@
+"""tools/ds4/engram_hash.py - DeepSeek-V4.1's Engram n-gram hashes (after DeepSeek's inference/engram.py, MIT).
+
+A position is hashed as the n-grams (2 .. max_ngram_size) that end there, each over n_heads heads; every (n-gram
+size, head) pair owns a prime-sized bucket range of the layer's table. Token ids first go through a compressed map
+where tokens that normalize alike (NFKC, NFD, accents stripped, lowercased, whitespace runs collapsed) share an id.
+
+ lay = Layout.from_config(text_config)
+ tm, n = compressed_token_map(tokenizer_json_path) # needs the `tokenizers` package
+ h = Hasher(lay, tm, multipliers(lay, n), pad=tm[text_config["engram_pad_token_id"]])
+ h.ids(tokens) -> int64 [T, n_layers, (max_ngram_size - 1) * n_heads], absolute table rows
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+def _is_prime(n: int) -> bool:
+ if n < 2:
+ return False
+ if n % 2 == 0:
+ return n == 2
+ i = 3
+ while i * i <= n:
+ if n % i == 0:
+ return False
+ i += 2
+ return True
+
+
+def _next_prime(start: int, seen: set[int]) -> int:
+ """The smallest prime above `start` that has not been handed out yet (engram.py's find_next_prime)."""
+ c = start + 1
+ while not _is_prime(c) or c in seen:
+ c += 1
+ return c
+
+
+@dataclass(frozen=True)
+class Layout:
+ layer_ids: tuple[int, ...]
+ num_embeddings: tuple[int, ...]
+ max_ngram: int
+ n_heads: int
+ head_dim: int
+ primes: tuple[tuple[tuple[int, ...], ...], ...] # [layer][n-gram size - 2][head]
+
+ @classmethod
+ def from_config(cls, t: dict) -> "Layout":
+ layer_ids = tuple(t["engram_layer_ids"])
+ mx, nh = t["engram_max_ngram_size"], t["engram_n_heads"]
+ primes, seen = [], set()
+ for _ in layer_ids:
+ per = []
+ for _ in range(mx - 1):
+ sizes, cur = [], t["engram_vocab_size"] - 1
+ for _ in range(nh):
+ cur = _next_prime(cur, seen)
+ seen.add(cur)
+ sizes.append(cur)
+ per.append(tuple(sizes))
+ primes.append(tuple(per))
+ lay = cls(layer_ids, tuple(t["engram_num_embeddings"]), mx, nh, t["engram_head_dim"], tuple(primes))
+ for i, n in enumerate(lay.num_embeddings):
+ got = sum(p for per in lay.primes[i] for p in per)
+ if got != n:
+ raise ValueError("engram layer %d: the primes sum to %d, the config says %d rows" % (i, got, n))
+ return lay
+
+ def flat_primes(self) -> np.ndarray:
+ """[n_layers, (max_ngram - 1) * n_heads] int64, in hash-column order."""
+ return np.array([[p for per in layer for p in per] for layer in self.primes], dtype=np.int64)
+
+ def offsets(self) -> np.ndarray:
+ f = self.flat_primes()
+ return np.concatenate([np.zeros((f.shape[0], 1), np.int64), np.cumsum(f, axis=1)[:, :-1]], axis=1)
+
+
+def multipliers(lay: Layout, compressed_vocab: int) -> np.ndarray:
+ """[n_layers, max_ngram] int64, odd, bounded so token * multiplier stays inside int64 (engram.py)."""
+ bound = max(1, (np.iinfo(np.int64).max // compressed_vocab) // 2)
+ rows = []
+ for lid in lay.layer_ids:
+ g = np.random.default_rng(10007 * lid)
+ rows.append(g.integers(low=0, high=bound, size=(lay.max_ngram,), dtype=np.int64) * 2 + 1)
+ return np.stack(rows)
+
+
+def compressed_token_map(tokenizer_json: str) -> tuple[np.ndarray, int]:
+ """engram.py's build_compressed_token_map over the raw Rust tokenizer (no clean-up of spaces)."""
+ from tokenizers import Regex, Tokenizer, normalizers
+
+ tok = Tokenizer.from_file(str(tokenizer_json))
+ sentinel = ""
+ norm = normalizers.Sequence([
+ normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(), normalizers.Lowercase(),
+ normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), normalizers.Replace(Regex(r"^ $"), sentinel),
+ normalizers.Strip(), normalizers.Replace(sentinel, " "),
+ ])
+ n = tok.get_vocab_size(with_added_tokens=True)
+ key_to_new: dict[str, int] = {}
+ lookup = np.zeros(n, dtype=np.int64)
+ for tid in range(n):
+ text = tok.decode([tid], skip_special_tokens=False)
+ if "�" in text:
+ key = tok.id_to_token(tid)
+ else:
+ normalized = norm.normalize_str(text)
+ key = normalized if normalized else text
+ new = key_to_new.get(key)
+ if new is None:
+ new = len(key_to_new)
+ key_to_new[key] = new
+ lookup[tid] = new
+ return lookup, len(key_to_new)
+
+
+class Hasher:
+ def __init__(self, lay: Layout, token_map: np.ndarray, mult: np.ndarray, pad: int):
+ self.lay, self.tm, self.mult, self.pad = lay, token_map, mult, int(pad)
+ self.primes = np.array(lay.primes, dtype=np.int64) # [L, max_ngram - 1, heads]
+ self.off = lay.offsets() # [L, cols]
+
+ def ids(self, tokens) -> np.ndarray:
+ """Absolute table rows [T, n_layers, cols] for a sequence that starts at position 0 (no dead tokens)."""
+ c = self.tm[np.asarray(tokens, dtype=np.int64)]
+ T = len(c)
+ pos = np.arange(T)
+ toks = []
+ blocked = np.zeros(T, dtype=bool)
+ for s in range(self.lay.max_ngram):
+ src = c[np.clip(pos - s, 0, None)]
+ blocked = blocked | (pos < s)
+ toks.append(np.where(blocked, self.pad, src))
+ toks = np.stack(toks, axis=-1) # [T, max_ngram]
+ prod = toks[:, None, :] * self.mult[None] # [T, L, max_ngram]
+ rolling, out = prod[..., 0], []
+ for i in range(1, self.lay.max_ngram):
+ rolling = np.bitwise_xor(rolling, prod[..., i])
+ out.append(rolling[..., None] % self.primes[None, :, i - 1])
+ return np.concatenate(out, axis=-1) + self.off[None]
diff --git a/tools/dsv41/exl3_torch.py b/tools/dsv41/exl3_torch.py
new file mode 100644
index 000000000..848e2e68d
--- /dev/null
+++ b/tools/dsv41/exl3_torch.py
@@ -0,0 +1,146 @@
+"""tools/ds4/exl3_torch.py - EXL3 (exllamav3) matrices decoded on the GPU with torch, for the DeepSeek-V4.1 reference.
+
+The format is include/strata/kernels/exl3.hpp's; this is tools/exl3_pack.py's numpy decoder (exl3_states, exl3_values,
+exl3_rows) on torch tensors. decode(st, prefix) returns the full weight as [n, k] (out x in, F.linear's layout):
+W = diag(suh) H W' H diag(svh), so x W is H(H(x * suh) W') * svh.
+"""
+from __future__ import annotations
+
+import math
+
+import numpy as np
+import torch
+
+_P = torch.arange(256)
+_ROW = (_P >> 3 & 3) * 2 + (_P & 1) + 8 * (_P >> 1 & 1)
+_COL = (_P >> 5) + 8 * (_P >> 2 & 1)
+_PERM = _ROW * 16 + _COL # tile[..., _PERM[p]] = value of step p
+
+
+def _sylvester(n: int) -> torch.Tensor:
+ h = torch.ones(1, 1, dtype=torch.float64)
+ while h.shape[0] < n:
+ h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)], 0)
+ return h / math.sqrt(n)
+
+
+_HAD = {}
+
+
+def had(dev) -> torch.Tensor:
+ if dev not in _HAD:
+ _HAD[dev] = _sylvester(128).to(dev, torch.float32)
+ return _HAD[dev]
+
+
+def _f16(b: int) -> float:
+ return float(np.array([b], dtype=np.uint16).view(np.float16)[0])
+
+
+K_INV, K_BIAS = _f16(0x1EEE), _f16(0xC931)
+
+
+def states(words: torch.Tensor, K: int) -> torch.Tensor:
+ """(..., 8 K) int32 tile words -> (..., 256) 16-bit states (MSB-first stream, tail-biting), int64."""
+ W = 8 * K
+ dev = words.device
+ p = _P.to(dev)
+ b1 = (p + 1 + 256) * K
+ i1, i0 = (b1 - 1) >> 5, (b1 - 16) >> 5
+ sh = ((i1 + 1) << 5) - b1
+ w = words.to(torch.int64) & 0xFFFFFFFF
+ m = (w[..., i0 % W] << 32) | w[..., i1 % W]
+ return (m >> sh) & 0xFFFF
+
+
+def values(st: torch.Tensor, cb: int) -> torch.Tensor:
+ """States -> FP16-rounded codebook values (float32)."""
+ if cb == 2:
+ x = (st * 0x83DCD12D) & 0xFFFFFFFF
+ bs = (x & 0xFF) + (x >> 8 & 0xFF) + (x >> 16 & 0xFF) + (x >> 24 & 0xFF)
+ v = (1024.0 + bs.to(torch.float64)) * K_INV + K_BIAS
+ return v.to(torch.float16).to(torch.float32)
+ x = st * (0xCBAC1FED if cb == 1 else 89226354)
+ if cb == 0:
+ x = x + 64248484
+ x = ((x & 0xFFFFFFFF) & 0x8FFF8FFF) ^ 0x3B603B60
+ lo = (x & 0xFFFF).to(torch.int16).view(torch.float16).to(torch.float64) # bit patterns as FP16
+ hi = (x >> 16).to(torch.int16).view(torch.float16).to(torch.float64)
+ return (lo + hi).to(torch.float16).to(torch.float32)
+
+
+def decode_raw(trellis: torch.Tensor, suh: torch.Tensor, svh: torch.Tensor, cb: int) -> torch.Tensor:
+ """trellis int16 [k/16, n/16, 16 K] (device), suh F16 [k], svh F16 [n] -> W [n, k] float32."""
+ kt, nt, w16 = trellis.shape
+ K = w16 // 16
+ k, n = kt * 16, nt * 16
+ dev = trellis.device
+ words = trellis.contiguous().view(torch.int32) # [kt, nt, 8K]
+ out = torch.empty(n, k, dtype=torch.float32, device=dev)
+ H = had(dev)
+ step = max(1, (1 << 27) // (nt * 256 * 8)) # 16-row slices per chunk (bounded scratch)
+ step = max(8, step - step % 8) # whole 128-row Hadamard blocks
+ perm = _PERM.to(dev)
+ for a in range(0, kt, step):
+ b = min(kt, a + step)
+ v = values(states(words[a:b], K), cb) # [b-a, nt, 256] in step order
+ tile = torch.empty_like(v)
+ tile[..., perm] = v
+ inner = tile.view(b - a, nt, 16, 16).permute(0, 2, 1, 3).reshape((b - a) * 16, n)
+ inner = (inner.view(-1, n // 128, 128) @ H).view(-1, n) # over n, in blocks of 128
+ inner = (H @ inner.view(-1, 128, n)).view(-1, n) # over k, in blocks of 128
+ out[:, a * 16:b * 16] = inner.T
+ out *= svh.to(torch.float32)[:, None]
+ out *= suh.to(torch.float32)[None, :]
+ return out
+
+
+def decode_many(st, prefixes: list[str], dev) -> list[torch.Tensor]:
+ """Several EXL3 matrices of one shape (a layer's experts' w1, say) decoded together: their trellises side by side
+ along n (both Hadamards act inside blocks of 128, so the stacked matrix decodes as each would), then each one's
+ channel scales. Returns [n, k] float32 views, one a prefix."""
+ trs = [tensor(st, p + ".trellis", dev) for p in prefixes]
+ kt, nt, _ = trs[0].shape
+ cb = codebook(st, prefixes[0])
+ ones_k = torch.ones(kt * 16, dtype=torch.float16, device=dev)
+ ones_n = torch.ones(nt * 16 * len(prefixes), dtype=torch.float16, device=dev)
+ inner = decode_raw(torch.cat(trs, dim=1), ones_k, ones_n, cb) # [N n, k]
+ n = nt * 16
+ out = []
+ for i, p in enumerate(prefixes):
+ w = inner[i * n:(i + 1) * n]
+ w *= tensor(st, p + ".svh", dev).to(torch.float32)[:, None]
+ w *= tensor(st, p + ".suh", dev).to(torch.float32)[None, :]
+ out.append(w)
+ return out
+
+
+def tensor(st, name: str, dev) -> torch.Tensor:
+ _, dt, shape, _, _ = st.t[name]
+ r = st.raw(name)
+ np_dt = {"F32": np.float32, "F16": np.float16, "I16": np.int16, "I32": np.int32, "BF16": np.uint16}[dt]
+ a = torch.from_numpy(np.array(r.view(np_dt)).reshape(shape))
+ if dt == "BF16":
+ a = a.view(torch.int16).view(torch.bfloat16)
+ return a.to(dev)
+
+
+def is_exl3(st, prefix: str) -> bool:
+ return st.has(prefix + ".trellis")
+
+
+def codebook(st, prefix: str) -> int:
+ return 2 if st.has(prefix + ".mul1") else 1 if st.has(prefix + ".mcg") else 0
+
+
+def decode(st, prefix: str, dev) -> torch.Tensor:
+ """The EXL3 matrix at `prefix` (.trellis/.suh/.svh) as W [n, k] float32 on dev."""
+ return decode_raw(tensor(st, prefix + ".trellis", dev), tensor(st, prefix + ".suh", dev),
+ tensor(st, prefix + ".svh", dev), codebook(st, prefix))
+
+
+def weight(st, prefix: str, dev) -> torch.Tensor:
+ """A linear weight [n, k] float32: EXL3 decoded, or the stored float tensor (prefix.weight)."""
+ if is_exl3(st, prefix):
+ return decode(st, prefix, dev)
+ return torch.from_numpy(np.array(st.f32(prefix + ".weight"))).to(dev)
diff --git a/tools/dsv41/pack.py b/tools/dsv41/pack.py
new file mode 100644
index 000000000..4ab6edf33
--- /dev/null
+++ b/tools/dsv41/pack.py
@@ -0,0 +1,152 @@
+"""tools/ds4/pack.py - the engine's pack for a DeepSeek-V4.1 EXL3 model, beside the model's files (no weights copied).
+
+ python tools/ds4/pack.py --model ~/models/DSV4
+
+writes <model>/maya-ds4/:
+ ds4.txt the geometry, `key value` lines (lists comma-separated), from config.json's text_config (and its
+ vision_config as vision.<key>, image_token_id)
+ tensors.txt every tensor of the directory's safetensors (the index's 43 shards and the base repo's Engram shards
+ 47 / 48 beside them): `file <i> <name>` lines, then `t <name> <dtype> <file> <offset> <bytes> <dims...>`
+ engram.bin the Engram hash tables (tools/ds4/engram_hash.py): int32 magic 'DSE1', vocab, layers, max_ngram,
+ heads, head_dim, pad (compressed); int32 token_map[vocab]; int64 mult[layers][max_ngram];
+ int64 primes[layers][cols]; int64 offsets[layers][cols] (cols = (max_ngram - 1) * heads)
+ tokenizer/ the server's tokenizer (tools/strata_tokenizer.py's layout, pre "deepseek3") and chat template
+
+The compressed token map needs the `tokenizers` package (it normalizes every token as DeepSeek's training did).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import struct
+import sys
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import engram_hash as EH # noqa: E402
+from st import SafeTensors # noqa: E402
+
+TEMPLATE = HERE / "chat_template.jinja"
+
+
+def need(st: SafeTensors, t: dict) -> list[str]:
+ """The tensors the engine reads; a missing one stops the pack here instead of the engine's start."""
+ miss = []
+ for li in range(t["num_hidden_layers"]):
+ L = "layers.%d." % li
+ for n in ("attn.wq_a.trellis", "attn.wq_b.trellis", "attn.wkv.trellis", "attn.wo_b.trellis",
+ "attn.attn_sink", "attn.q_norm.weight", "attn.kv_norm.weight", "attn_norm.weight", "ffn_norm.weight",
+ "ffn.gate.weight", "ffn.gate.bias", "hc_attn_fn", "hc_ffn_fn", "ffn.shared_experts.w1.trellis"):
+ if not st.has(L + n):
+ miss.append(L + n)
+ if li in t["engram_layer_ids"] and not st.has(L + "engram.embed.weight"):
+ miss.append(L + "engram.embed.weight (shards 47 / 48 of deepseek-ai/DeepSeek-V4.1-Flash-1017)")
+ for n in ("embed.weight", "norm.weight", "head.trellis"):
+ if not st.has(n):
+ miss.append(n)
+ return miss
+
+
+def write_engram(out: pathlib.Path, model: pathlib.Path, t: dict):
+ lay = EH.Layout.from_config(t)
+ tm, n = EH.compressed_token_map(model / "tokenizer.json")
+ if n != t["engram_compressed_vocab_size"]:
+ raise SystemExit("engram: the compressed vocab has %d ids, the config says %d" %
+ (n, t["engram_compressed_vocab_size"]))
+ vocab = t["vocab_size"]
+ full = np.zeros(vocab, dtype=np.int32) # ids past the tokenizer's (none in use) map to 0
+ full[:len(tm)] = tm
+ mult = EH.multipliers(lay, n)
+ cols = (lay.max_ngram - 1) * lay.n_heads
+ with out.open("wb") as f:
+ f.write(struct.pack("<4s6i", b"DSE1", vocab, len(lay.layer_ids), lay.max_ngram, lay.n_heads, lay.head_dim,
+ int(tm[t["engram_pad_token_id"]])))
+ f.write(full.astype("<i4").tobytes())
+ f.write(mult.astype("<i8").tobytes())
+ f.write(lay.flat_primes().astype("<i8").tobytes())
+ f.write(lay.offsets().astype("<i8").tobytes())
+ return len(lay.layer_ids), cols
+
+
+def write_tokenizer(out: pathlib.Path, model: pathlib.Path):
+ """vocab.json (id order), merges.txt, token_type.json, tokenizer.json {pre, special_ids} and the chat template."""
+ tj = json.loads((model / "tokenizer.json").read_text(encoding="utf-8"))
+ vocab = tj["model"]["vocab"]
+ added = {a["id"]: a for a in tj["added_tokens"]}
+ n = max(max(vocab.values()), max(added)) + 1
+ tokens = ["<|unused▁%d|>" % i for i in range(n)] # (ids no token names: none in this vocabulary)
+ types = [1] * n # llama.cpp: 1 normal, 3 control, 4 user-defined
+ for s, i in vocab.items():
+ tokens[i] = s
+ for i, a in added.items():
+ tokens[i] = a["content"]
+ types[i] = 3 if a["special"] else 4
+ merges = tj["model"]["merges"]
+ merges = [m if isinstance(m, str) else " ".join(m) for m in merges]
+ out.mkdir(parents=True, exist_ok=True)
+ if len(set(tokens)) != n:
+ raise SystemExit("tokenizer.json: duplicate token strings")
+ (out / "vocab.json").write_text(json.dumps({t: i for i, t in enumerate(tokens)}, ensure_ascii=False),
+ encoding="utf-8")
+ (out / "merges.txt").write_text("\n".join(merges), encoding="utf-8")
+ (out / "token_type.json").write_text(json.dumps(types))
+ (out / "tokenizer.json").write_text(json.dumps({"pre": "deepseek3", "special_ids": {
+ "bos": 0, "eos": 1, "pad": 2}}, indent=1))
+ if TEMPLATE.exists():
+ (out / "chat_template.jinja").write_text(TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8")
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--model", required=True)
+ ap.add_argument("--out", help="default: <model>/maya-ds4")
+ a = ap.parse_args()
+ model = pathlib.Path(a.model).expanduser().resolve()
+ out = pathlib.Path(a.out).expanduser() if a.out else model / "maya-ds4"
+ cfg = json.loads((model / "config.json").read_text())
+ t = cfg["text_config"]
+ q = cfg.get("quantization_config", {})
+ if q.get("quant_method") != "exl3":
+ raise SystemExit("%s: not an EXL3 model (quant_method %r)" % (model, q.get("quant_method")))
+ st = SafeTensors(model)
+ miss = need(st, t)
+ if miss:
+ raise SystemExit("missing tensors:\n " + "\n ".join(miss[:20]))
+ out.mkdir(parents=True, exist_ok=True)
+
+ with (out / "ds4.txt").open("w") as f:
+ for k, v in t.items():
+ if isinstance(v, bool):
+ v = int(v)
+ if isinstance(v, (int, float)):
+ f.write("%s %s\n" % (k, v))
+ elif isinstance(v, list) and all(isinstance(x, (int, float)) for x in v):
+ f.write("%s %s\n" % (k, ",".join(str(x) for x in v)))
+ elif isinstance(v, dict):
+ for k2, v2 in v.items():
+ if isinstance(v2, (int, float)) and not isinstance(v2, bool):
+ f.write("%s.%s %s\n" % (k, k2, v2))
+ # the vision tower's geometry (vision.<key>) and the token an image's span carries
+ for k, v in (cfg.get("vision_config") or {}).items():
+ if isinstance(v, (int, float)) and not isinstance(v, bool):
+ f.write("vision.%s %s\n" % (k, v))
+ if isinstance(cfg.get("image_token_id"), int):
+ f.write("image_token_id %d\n" % cfg["image_token_id"])
+ f.write("model_dir %s\n" % model)
+ with (out / "tensors.txt").open("w") as f:
+ for i, fn in enumerate(st.files):
+ f.write("file %d %s\n" % (i, fn))
+ for name, (fn, dt, shape, off, nb) in st.t.items():
+ f.write("t %s %s %d %d %d %s\n" % (name, dt, st.files.index(fn), off, nb, " ".join(map(str, shape))))
+ nl, cols = write_engram(out / "engram.bin", model, t)
+ write_tokenizer(out / "tokenizer", model)
+ print("%s: %d tensors in %d files, Engram %d layers x %d hash columns -> %s" %
+ (model.name, len(st.t), len(st.files), nl, cols, out))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
new file mode 100644
index 000000000..f7cd510ba
--- /dev/null
+++ b/tools/dsv41/pack_gguf.py
@@ -0,0 +1,498 @@
+"""tools/ds4/pack_gguf.py - the engine's pack for a DeepSeek-V4.1 GGUF (unsloth's DeepSeek-V4.1-Flash-1017 UD-Q8_K_XL
+and the like: dense BF16, routed experts MXFP4, Engram tables MXFP8 or Q6_K), beside the model's files (no weights
+copied).
+
+ python tools/ds4/pack_gguf.py --model ~/models/DeepSeek-V4.1-Flash-1017-GGUF/UD-Q8_K_XL
+
+writes <model>/maya-ds4/ in pack.py's format, so the engine reads both:
+ ds4.txt the geometry under config.json's names (the GGUF's deepseek41.* keys translated), model_dir, format gguf
+ tensors.txt every tensor under the checkpoint's names: `file <i> <name>` lines, then
+ `t <name> <dtype> <file> <offset> <bytes> <dims...>` - dims outermost first (torch order: a matrix is
+ [out][in]), dtype BF16 / F32 / MXFP4 / MXFP8 (ggml's block layouts, 32 values a block); the routed
+ experts as one tensor a layer and role ([n_exp][out][in])
+ engram.bin the Engram hash tables, from the GGUF's metadata (pack.py's layout; no tokenizers package needed)
+ tokenizer/ the server's tokenizer from the GGUF's vocabulary and merges (pre "deepseek3"), and its chat template
+
+A GGUF has no DSpark drafter: drafter_pack.py makes it from DeepSeek's checkpoint into ../dspark/, and the pack takes it
+from there when it is present. Nor a vision tower: vision_pack.py makes it from DeepSeek's checkpoint into ../vision/
+(--vision: another folder), and the pack takes it and its geometry (vision.<key>, image_token_id) from there when it is
+present; without it the engine reads text only.
+
+A quant that leaves the two Engram tables out (v1.1.16's Maya-S; Maya's quants hold them since) takes them from a set
+of GGUF files that holds them, beside it: --engram, else ../Engram/ (unsloth's MXFP8 shard of them, where setups before
+v1.1.17 put it) when it is whole.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import re
+import struct
+import sys
+
+TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17),
+ 67: ("MXFP8", 32, 33), # ggml type -> (name, values a block, bytes a block); 67: unsloth's fork
+ 43: ("MXFP8", 32, 33), # llama.cpp's own id (unsloth's GGUFs from 2026-10-19 on)
+ # the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)
+ 10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),
+ 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),
+ 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136)}
+# the Engram set a quant without the tables takes them from by default, beside it (or inside its folder): unsloth's
+# MXFP8 shard of them, where setups before v1.1.17 put it for Maya-S (--engram: any other)
+ENGRAM_SETS = ("Engram",)
+
+
+class Gguf:
+ """One GGUF shard's metadata and tensor infos (offsets absolute in the file)."""
+
+ def __init__(self, path: pathlib.Path):
+ self.path = path
+ with path.open("rb") as f:
+ self.f = f
+ magic = f.read(4)
+ if magic != b"GGUF":
+ raise SystemExit("%s: not a GGUF file" % path)
+ self.version, nt, nkv = struct.unpack("<IQQ", f.read(20))
+ self.kv = {}
+ for _ in range(nkv):
+ k = self._str()
+ self.kv[k] = self._val(struct.unpack("<I", f.read(4))[0])
+ self.tensors = []
+ for _ in range(nt):
+ name = self._str()
+ nd = struct.unpack("<I", f.read(4))[0]
+ dims = list(struct.unpack("<%dQ" % nd, f.read(8 * nd)))
+ typ, off = struct.unpack("<IQ", f.read(12))
+ self.tensors.append((name, typ, dims, off))
+ align = int(self.kv.get("general.alignment", 32))
+ pos = f.tell()
+ self.data_start = (pos + align - 1) // align * align
+ del self.f
+
+ def _str(self):
+ n = struct.unpack("<Q", self.f.read(8))[0]
+ return self.f.read(n).decode("utf-8", "replace")
+
+ def _val(self, t):
+ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
+ if t in fmt:
+ n = struct.calcsize(fmt[t])
+ return struct.unpack("<" + fmt[t], self.f.read(n))[0]
+ if t == 8:
+ return self._str()
+ if t == 9:
+ at, n = struct.unpack("<IQ", self.f.read(12))
+ if at in fmt:
+ n_b = struct.calcsize(fmt[at])
+ return list(struct.unpack("<%d%s" % (n, fmt[at]), self.f.read(n * n_b)))
+ return [self._val(at) for _ in range(n)]
+ raise SystemExit("%s: metadata type %d" % (self.path, t))
+
+
+# GGUF name (blk.N. stripped) -> the checkpoint's name under layers.N. (pack.py / the engine)
+LAYER = {
+ "attn_norm.weight": "attn_norm.weight",
+ "attn_sinks.weight": "attn.attn_sink",
+ "attn_q_a.weight": "attn.wq_a.weight",
+ "attn_q_a_norm.weight": "attn.q_norm.weight",
+ "attn_q_b.weight": "attn.wq_b.weight",
+ "attn_kv.weight": "attn.wkv.weight",
+ "attn_kv_a_norm.weight": "attn.kv_norm.weight",
+ "attn_output_a.weight": "attn.wo_a.slice", # one entry a group (the engine's slices)
+ "attn_output_b.weight": "attn.wo_b.weight",
+ "attn_compressor_kv.weight": "attn.compressor.wkv.weight",
+ "attn_compressor_gate.weight": "attn.compressor.wgate.weight",
+ "attn_compressor_norm.weight": "attn.compressor.norm.weight",
+ "indexer.attn_q_b.weight": "attn.indexer.wq_b.weight",
+ "indexer.attn_k.weight": "attn.indexer.wk.weight",
+ "indexer.k_norm.weight": "attn.indexer.k_norm.weight",
+ "indexer.proj.weight": "attn.indexer.weights_proj.weight",
+ "hc_attn_fn.weight": "hc_attn_fn",
+ "hc_attn_base.weight": "hc_attn_base",
+ "hc_attn_scale.weight": "hc_attn_scale",
+ "hc_ffn_fn.weight": "hc_ffn_fn",
+ "hc_ffn_base.weight": "hc_ffn_base",
+ "hc_ffn_scale.weight": "hc_ffn_scale",
+ "ffn_norm.weight": "ffn_norm.weight",
+ "ffn_gate_inp.weight": "ffn.gate.weight",
+ "exp_probs_b.bias": "ffn.gate.bias",
+ "exp_probs_b_vl.bias": "ffn.gate.bias_vl",
+ "ffn_gate_shexp.weight": "ffn.shared_experts.w1.weight",
+ "ffn_up_shexp.weight": "ffn.shared_experts.w3.weight",
+ "ffn_down_shexp.weight": "ffn.shared_experts.w2.weight",
+ "ffn_gate_exps.weight": "ffn.experts.w1", # [n_exp][ff][dim]: one entry an expert
+ "ffn_up_exps.weight": "ffn.experts.w3",
+ "ffn_down_exps.weight": "ffn.experts.w2", # [n_exp][dim][ff]
+ "engram_embd.weight": "engram.embed.weight",
+ "engram_kv.weight": "engram.wkv.weight",
+ "engram_gate.weight": "engram.gate.weight", # q_weight * k_weight, folded by the converter
+}
+GLOBAL = {"token_embd.weight": "embed.weight", "output_norm.weight": "norm.weight", "output.weight": "head.weight"}
+
+
+def geometry(kv: dict, vocab: int) -> dict:
+ """config.json's text_config names (what the engine reads) from the GGUF's deepseek41.* keys."""
+ a = kv["general.architecture"]
+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731
+ nl = g("block_count")
+ ratios = list(g("attention.compress_ratios", []))
+ out = {
+ "vocab_size": vocab,
+ "hidden_size": g("embedding_length"),
+ "num_hidden_layers": nl,
+ "num_attention_heads": g("attention.head_count"),
+ "head_dim": g("attention.key_length"),
+ "qk_rope_head_dim": g("rope.dimension_count"),
+ "q_lora_rank": g("attention.q_lora_rank"),
+ "o_lora_rank": g("attention.output_lora_rank"),
+ "o_groups": g("attention.output_group_count"),
+ "n_routed_experts": g("expert_count"),
+ "num_experts_per_tok": g("expert_used_count"),
+ "moe_intermediate_size": g("expert_feed_forward_length"),
+ "sliding_window": g("attention.sliding_window"),
+ "hc_mult": g("hyper_connection.count"),
+ "hc_sinkhorn_iters": g("hyper_connection.sinkhorn_iterations"),
+ "hc_eps": g("hyper_connection.epsilon"),
+ "index_n_heads": g("attention.indexer.head_count"),
+ "index_head_dim": g("attention.indexer.key_length"),
+ "index_topk": g("attention.indexer.top_k"),
+ "candidate_source_layer_id": g("attention.indexer.candidate_source_layer"),
+ "candidate_topk_blocks": g("attention.indexer.candidate_top_k_blocks"),
+ "candidate_block_size": g("attention.indexer.candidate_block_size"),
+ "rope_scaling.original_max_position_embeddings": g("rope.scaling.original_context_length"),
+ "rope_scaling.factor": g("rope.scaling.factor"),
+ "rope_scaling.beta_fast": g("rope.scaling.yarn_beta_fast"),
+ "rope_scaling.beta_slow": g("rope.scaling.yarn_beta_slow"),
+ "eos_token_id": kv.get("tokenizer.ggml.eos_token_id", 1),
+ "rms_norm_eps": g("attention.layer_norm_rms_epsilon"),
+ "routed_scaling_factor": g("expert_weights_scale"),
+ "swiglu_limit": (g("swiglu_clamp_exp") or [0.0])[0],
+ "rope_theta": g("rope.freq_base"),
+ "compress_rope_theta": g("attention.compress_rope_freq_base"),
+ "compress_ratios": ratios,
+ "kv_source_layer_ids": list(g("attention.kv_source_layers", [])),
+ "index_source_layer_ids": list(g("attention.indexer.source_layers", [])),
+ "engram_layer_ids": list(g("engram.layers", [])),
+ "max_position_embeddings": g("context_length"),
+ "dspark_block_size": g("dspark.block_size", 0),
+ "dspark_noise_token_id": g("dspark.noise_token_id", 0),
+ "dspark_markov_rank": g("dspark.markov_rank", 0),
+ "dspark_n_routed_experts": g("dspark.expert_count", 0),
+ "dspark_num_experts_per_tok": g("dspark.expert_used_count", 0),
+ "dspark_target_layer_ids": list(g("dspark.target_layers", [])),
+ }
+ clamp = g("swiglu_clamp_exp") or []
+ if clamp and any(abs(c - clamp[0]) > 1e-6 for c in clamp):
+ raise SystemExit("swiglu_clamp_exp differs between layers %s: the engine takes one swiglu_limit" % clamp[:8])
+ return out
+
+
+def ple_to_engram(kv: dict):
+ """llama.cpp's own names for the Engram metadata (unsloth's GGUFs from 2026-10-19 on: ple.*, the hash heads' offsets
+ counted across the layers' tables) -> the engram.* keys this pack reads (each layer's offsets from its own table)."""
+ a = kv.get("general.architecture")
+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731
+ if g("engram.layers") or not g("ple.layers"):
+ return
+ layers, ngram, heads = list(g("ple.layers")), g("ple.ngram_size"), g("ple.heads_per_ngram")
+ offs, sizes = list(g("ple.head_offsets")), list(g("ple.head_vocab_sizes"))
+ cols = (ngram - 1) * heads
+ if len(offs) != len(layers) * cols or len(sizes) != len(offs):
+ raise SystemExit("ple: %d head offsets, %d sizes for %d layers x %d columns" %
+ (len(offs), len(sizes), len(layers), cols))
+ per, rows = [], []
+ for i in range(len(layers)):
+ o, s = offs[i * cols:(i + 1) * cols], sizes[i * cols:(i + 1) * cols]
+ per += [x - o[0] for x in o]
+ rows.append(o[-1] + s[-1] - o[0])
+ for k, v in (("layers", layers), ("max_ngram_size", ngram), ("head_count", heads),
+ ("head_dim", g("embedding_length_per_layer_input")), ("multipliers", list(g("ple.layer_multipliers"))),
+ ("primes", sizes), ("offsets", per), ("num_embeddings", rows)):
+ kv["%s.engram.%s" % (a, k)] = v
+
+
+def split_ple(g: Gguf, kv: dict):
+ """llama.cpp's one Engram tensor (per_layer_token_embd: every layer's table, one after the other) -> each layer's
+ own blk.N.engram_embd as the earlier GGUFs had it: the same bytes, read in place."""
+ a = kv["general.architecture"]
+ ple = [t for t in g.tensors if t[0] == "per_layer_token_embd.weight"]
+ if not ple:
+ return
+ name, typ, dims, off = ple[0]
+ layers, rows = kv.get("%s.engram.layers" % a, []), kv.get("%s.engram.num_embeddings" % a, [])
+ if len(dims) != 2 or typ not in TYPES or len(rows) != len(layers) or sum(rows) != dims[1]:
+ raise SystemExit("%s: %s %s does not split in the Engram layers %s of %s rows" % (g.path.name, name, dims, layers,
+ rows))
+ _, per, bpb = TYPES[typ]
+ row = dims[0] // per * bpb
+ parts, start = [], 0
+ for il, n in zip(layers, rows):
+ parts.append(("blk.%d.engram_embd.weight" % il, typ, [dims[0], n], off + start * row))
+ start += n
+ g.tensors = [t for t in g.tensors if t[0] != name] + parts
+
+
+def short(g: Gguf):
+ """Why a GGUF file is shorter than its own tensor directory says (a download still running), or None."""
+ need = g.data_start
+ for _, typ, dims, off in g.tensors:
+ if typ in TYPES:
+ _, per, bpb = TYPES[typ]
+ nel = 1
+ for d in dims:
+ nel *= d
+ need = max(need, g.data_start + off + nel // per * bpb)
+ have = g.path.stat().st_size
+ return None if have >= need else "%s: %d of %d bytes" % (g.path.name, have, need)
+
+
+def engram_set(model: pathlib.Path, explicit: str | None):
+ """The Engram tables' GGUF set for a quant that leaves them out: (its folder, its files read), or (None, []).
+ explicit (--engram): that folder, whole; else the first whole one of ENGRAM_SETS beside the quant or in its
+ folder - one not whole yet (its download still running) is passed over."""
+ dirs = ([pathlib.Path(explicit).expanduser().resolve()] if explicit else
+ [d for n in ENGRAM_SETS for d in (model / n, model.parent / n)])
+ for d in dirs:
+ paths = sorted(d.glob("*.gguf")) if d.is_dir() else []
+ if not paths:
+ if explicit:
+ raise SystemExit("%s: no .gguf files (--engram: the folder of the Engram tables' GGUF set)" % d)
+ continue
+ try:
+ egs = [Gguf(p) for p in paths]
+ why = next((w for w in map(short, egs) if w), None)
+ except (Exception, SystemExit) as e: # noqa: BLE001 - a header cut short raises anything
+ why = str(e) or type(e).__name__
+ if why is None:
+ return d, egs
+ if explicit:
+ raise SystemExit("%s: not whole (%s)" % (d, why))
+ print("%s: not whole (%s) - not used" % (d, why))
+ return None, []
+
+
+def write_engram(out: pathlib.Path, kv: dict, vocab: int):
+ a = kv["general.architecture"]
+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731
+ layers = list(g("engram.layers", []))
+ if not layers:
+ return 0, 0
+ ngram, heads, hd = g("engram.max_ngram_size"), g("engram.head_count"), g("engram.head_dim")
+ tm = list(g("engram.token_map"))
+ mult, primes, offs = list(g("engram.multipliers")), list(g("engram.primes")), list(g("engram.offsets"))
+ cols = (ngram - 1) * heads
+ nl = len(layers)
+ if len(mult) != nl * ngram or len(primes) != nl * cols or len(offs) != nl * cols:
+ raise SystemExit("engram: %d multipliers, %d primes, %d offsets for %d layers x %d columns (ngram %d)" %
+ (len(mult), len(primes), len(offs), nl, cols, ngram))
+ pad = int(tm[g("engram.pad_token_id")])
+ full = tm + [0] * max(0, vocab - len(tm))
+ with out.open("wb") as f:
+ f.write(struct.pack("<4s6i", b"DSE1", vocab, nl, ngram, heads, hd, pad))
+ f.write(struct.pack("<%di" % vocab, *full[:vocab]))
+ f.write(struct.pack("<%dq" % len(mult), *mult))
+ f.write(struct.pack("<%dq" % len(primes), *primes))
+ f.write(struct.pack("<%dq" % len(offs), *offs))
+ return nl, cols
+
+
+def write_tokenizer(out: pathlib.Path, kv: dict, template: str | None):
+ tokens = kv["tokenizer.ggml.tokens"]
+ types = kv.get("tokenizer.ggml.token_type") or [1] * len(tokens)
+ merges = kv.get("tokenizer.ggml.merges") or []
+ if len(set(tokens)) != len(tokens):
+ raise SystemExit("the GGUF's vocabulary has duplicate token strings")
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "vocab.json").write_text(json.dumps({t: i for i, t in enumerate(tokens)}, ensure_ascii=False),
+ encoding="utf-8")
+ (out / "merges.txt").write_text("\n".join(merges), encoding="utf-8")
+ (out / "token_type.json").write_text(json.dumps([int(t) for t in types]))
+ special = {"bos": int(kv.get("tokenizer.ggml.bos_token_id", 0)), "eos": int(kv.get("tokenizer.ggml.eos_token_id", 1)),
+ "pad": int(kv.get("tokenizer.ggml.padding_token_id", 2))}
+ (out / "tokenizer.json").write_text(json.dumps({"pre": "deepseek3", "special_ids": special}, indent=1))
+ if template:
+ (out / "chat_template.jinja").write_text(template, encoding="utf-8")
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--model", required=True, help="the directory of the GGUF's shards")
+ ap.add_argument("--out", help="default: <model>/maya-ds4")
+ ap.add_argument("--vision", help="vision_pack.py's folder (default: <model>/../vision; taken when it is there)")
+ ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")
+ ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out "
+ "(default: <model>/../Engram, when it is whole)")
+ ap.add_argument("--shards", nargs="+", help="the GGUF's files in --model (default: every .gguf there - one "
+ "model's; the setup names them: a folder may hold an earlier layout)")
+ a = ap.parse_args()
+ model = pathlib.Path(a.model).expanduser().resolve()
+ out = pathlib.Path(a.out).expanduser() if a.out else model / "maya-ds4"
+ shards = [model / n for n in a.shards] if a.shards else sorted(model.glob("*.gguf"))
+ if not shards or not all(p.is_file() for p in shards):
+ raise SystemExit("%s: no .gguf files" % model if not shards else
+ "%s: not there" % next(p for p in shards if not p.is_file()))
+ gs = [Gguf(p) for p in shards]
+ kv = next((g.kv for g in gs if "general.architecture" in g.kv), None)
+ if kv is None or kv["general.architecture"] not in ("deepseek41",):
+ raise SystemExit("%s: not a DeepSeek-V4.1 GGUF (architecture %r)" % (model, kv and kv.get("general.architecture")))
+ split = int(kv.get("split.count", 1))
+ if split != len(shards):
+ counts = sorted({m.group(1) for m in (re.search(r"-of-(\d{5})\.gguf$", p.name) for p in shards) if m})
+ raise SystemExit("%s: %d of the %d shards are here" % (model, len(shards), split) if len(counts) < 2 else
+ "%s: the shards of more than one GGUF split (-of-%s): name the model's own with --shards" %
+ (model, ", -of-".join(counts)))
+ ple_to_engram(kv)
+ for g in gs:
+ split_ple(g, kv)
+ # the Engram tables may come from a shared set of GGUF files beside the quant (the same tensor names: engram_set) -
+ # its engram tensors fill in the ones the quant's shards leave out, its engram metadata too. v1.1.16's Maya-S:
+ # unsloth's UD-Q8_K_XL shard that holds only the tables, as it is (00008 of 9; before 2026-10-19 shards 00008 and
+ # 00009 of 10, a table each), so no shard count is checked - every Engram layer's tables must be found, below.
+ # Looked for only when the quant's shards leave some out.
+ have = {t[0] for g in gs for t in g.tensors}
+ arch = kv["general.architecture"]
+ layers = kv.get("%s.engram.layers" % arch, [])
+ lacking = not layers or any("blk.%d.%s.weight" % (il, part) not in have
+ for il in layers for part in ("engram_embd", "engram_kv", "engram_gate"))
+ eg_dir, egs = engram_set(model, a.engram) if lacking or a.engram else (None, [])
+ eg_from = {}
+ if eg_dir is not None:
+ eshards = [g.path for g in egs]
+ for g in egs:
+ for k, v in g.kv.items():
+ if ".engram." in k or ".ple." in k or k.endswith(".embedding_length_per_layer_input"):
+ kv.setdefault(k, v)
+ ple_to_engram(kv)
+ for g in egs:
+ split_ple(g, kv)
+ for p, g in zip(eshards, egs):
+ picked = [t for t in g.tensors if re.match(r"blk\.\d+\.engram_", t[0]) and t[0] not in have]
+ if picked:
+ g.tensors = picked
+ eg_from[p] = g
+ for il in kv.get("%s.engram.layers" % arch, []):
+ for part in ("engram_embd", "engram_kv", "engram_gate"):
+ nm = "blk.%d.%s.weight" % (il, part)
+ if nm not in have and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):
+ raise SystemExit("%s: missing - not in the quant's shards nor in an Engram set (--engram, or "
+ "<model>/../Engram/: unsloth's UD-Q8_K_XL shard of the Engram tables)" % nm)
+ rel = lambda p: os.path.relpath(p, model).replace(os.sep, "/") # noqa: E731
+ gs_all = gs + list(eg_from.values())
+ names = [g.path.name for g in gs] + [rel(p) for p in eg_from]
+
+ tensors, unknown = [], []
+ for fi, g in enumerate(gs_all):
+ for name, typ, dims, off in g.tensors:
+ if typ not in TYPES:
+ raise SystemExit("%s: %s has ggml type %d, which the engine does not read" % (g.path.name, name, typ))
+ tname, per, bpb = TYPES[typ]
+ nel = 1
+ for d in dims:
+ nel *= d
+ if dims[0] % per:
+ raise SystemExit("%s: %s rows of %d are not whole %s blocks" % (g.path.name, name, dims[0], tname))
+ nbytes = nel // per * bpb
+ m = re.match(r"blk\.(\d+)\.(.+)$", name)
+ if m and m.group(2) in LAYER:
+ ck = "layers.%s.%s" % (m.group(1), LAYER[m.group(2)])
+ elif name in GLOBAL:
+ ck = GLOBAL[name]
+ else:
+ unknown.append(name)
+ continue
+ shape = list(reversed(dims)) # ggml ne0 innermost -> torch order, outermost first
+ base = g.data_start + off
+ if ck.endswith(".attn.wo_a.slice"):
+ # [o_groups * o_lora][in] -> o_groups entries of [o_lora][in] (group g: rows g * o_lora ..)
+ n = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])
+ if len(shape) != 2 or shape[0] % n:
+ raise SystemExit("%s: %s %s does not split in %d groups" % (g.path.name, name, shape, n))
+ part = nbytes // n
+ for i in range(n):
+ tensors.append(("%s.%d.weight" % (ck, i), tname, fi, base + i * part, part,
+ [shape[0] // n, shape[1]]))
+ continue
+ if ".ffn.experts." in ck:
+ # [n_exp][out][in] -> an entry an expert: layers.N.ffn.experts.E.w1.weight
+ n, part = shape[0], nbytes // shape[0]
+ pre, role = ck.rsplit(".", 1)
+ for i in range(n):
+ tensors.append(("%s.%d.%s.weight" % (pre, i, role), tname, fi, base + i * part, part, shape[1:]))
+ continue
+ tensors.append((ck, tname, fi, base, nbytes, shape))
+ if unknown:
+ raise SystemExit("tensors the pack does not map (%d): %s" % (len(unknown), ", ".join(unknown[:12])))
+ # the DSpark drafter (drafter_pack.py's dspark.bin beside the quants' folders, from DeepSeek's checkpoint): its
+ # three blocks as mtp.0-2, so the engine drafts with them
+ files = names
+ if eg_from:
+ print("Engram tables from %s (%d shards)" % (eg_dir, len(eg_from)))
+ drafter = pathlib.Path(a.drafter).expanduser().resolve() if a.drafter else model.parent / "dspark" / "dspark.json"
+ if a.drafter and not drafter.is_file():
+ raise SystemExit("%s: no drafter there" % drafter)
+ drafter_in = drafter.is_file() and drafter.with_suffix(".bin").is_file()
+ if drafter_in:
+ files.append(rel(drafter.with_suffix(".bin")))
+ groups = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])
+ for name, dt, off, nb, shape in json.loads(drafter.read_text())["tensors"]:
+ if name.endswith(".attn.wo_a.weight"): # (one entry a group, as the trunk's)
+ for i in range(groups):
+ tensors.append(("%s.slice.%d.weight" % (name[:-len(".weight")], i), dt, len(files) - 1,
+ off + i * (nb // groups), nb // groups, [shape[0] // groups, shape[1]]))
+ continue
+ tensors.append((name, dt, len(files) - 1, off, nb, shape))
+ # the vision tower (vision_pack.py's vision.bin, from DeepSeek's checkpoint): its tensors under the checkpoint's
+ # names, the picture span's marker rows, and the geometry config.json gives them
+ vdir = pathlib.Path(a.vision).expanduser().resolve() if a.vision else model.parent / "vision"
+ vision = {}
+ if (vdir / "vision.json").is_file() and (vdir / "vision.bin").is_file():
+ try:
+ rel = os.path.relpath(vdir / "vision.bin", model).replace(os.sep, "/")
+ except ValueError: # (Windows: another drive)
+ raise SystemExit("%s: the vision folder must be on the model's drive" % vdir)
+ if " " in rel:
+ raise SystemExit("%s: the vision folder's path relative to the model has a space" % vdir)
+ files.append(rel)
+ info = json.loads((vdir / "vision.json").read_text())
+ for name, dt, off, nb, shape in info["tensors"]:
+ tensors.append((name, dt, len(files) - 1, off, nb, shape))
+ vision = info["config"]
+ vocab = next(t[5][0] for t in tensors if t[0] == "embed.weight")
+ geo = geometry(kv, vocab)
+
+ out.mkdir(parents=True, exist_ok=True)
+ with (out / "ds4.txt").open("w") as f:
+ for k, v in geo.items():
+ if v is None:
+ continue
+ if isinstance(v, bool):
+ v = int(v)
+ f.write("%s %s\n" % (k, ",".join(str(x) for x in v) if isinstance(v, list) else v))
+ for k, v in vision.items():
+ f.write("%s %s\n" % (k, v))
+ f.write("format gguf\n")
+ f.write("model_dir %s\n" % model)
+ with (out / "tensors.txt").open("w") as f:
+ for i, name in enumerate(files):
+ f.write("file %d %s\n" % (i, name))
+ for name, dt, fi, off, nb, shape in tensors:
+ f.write("t %s %s %d %d %d %s\n" % (name, dt, fi, off, nb, " ".join(map(str, shape))))
+ nl, cols = write_engram(out / "engram.bin", kv, vocab)
+ # the server's chat template (pack.py's, so both formats chat alike); the GGUF's own when it is not there
+ template = pathlib.Path(__file__).resolve().parent / "chat_template.jinja"
+ write_tokenizer(out / "tokenizer", kv,
+ template.read_text(encoding="utf-8") if template.exists() else kv.get("tokenizer.chat_template"))
+ by = {}
+ for t in tensors:
+ by[t[1]] = by.get(t[1], 0) + t[4]
+ extras = [w for w, there in (("the DSpark drafter", drafter_in), ("the vision tower", bool(vision))) if there]
+ print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %
+ (model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),
+ nl, cols, "".join(", " + w for w in extras), out))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/recipe_maya_s_test.json b/tools/dsv41/recipe_maya_s_test.json
new file mode 100644
index 000000000..320ff5e75
--- /dev/null
+++ b/tools/dsv41/recipe_maya_s_test.json
@@ -0,0 +1,13 @@
+{
+ "comment": "a quick Maya-S-like test model (~200 GB): ggml's IQ2_XXS quantizer runs at ~1 M weights/s a thread (a full Maya-S takes ~8 h on 12 threads), so most layers take Q2_K / Q3_K (20-80 M/s) and a few layers each low-bit type the engine reads, on real weights",
+ "experts": {"gate": "Q2_K", "up": "Q2_K", "down": "Q3_K"},
+ "layers": [
+ {"layers": "10", "gate": "IQ2_XXS", "up": "IQ2_XXS", "down": "IQ3_XXS"},
+ {"layers": "20", "gate": "IQ4_XS", "up": "IQ4_XS", "down": "Q4_K"},
+ {"layers": "25", "gate": "IQ3_XXS", "up": "IQ3_XXS", "down": "IQ3_S"},
+ {"layers": "30", "gate": "IQ2_S", "up": "IQ2_S", "down": "IQ3_S"}
+ ],
+ "dense": "Q8_0",
+ "dense_keep": ["hc_", "engram", "norm", "attn_sinks", "exp_probs_b", "ffn_gate_inp"],
+ "engram": "skip"
+}
diff --git a/tools/dsv41/ref.py b/tools/dsv41/ref.py
new file mode 100644
index 000000000..9659aa8b7
--- /dev/null
+++ b/tools/dsv41/ref.py
@@ -0,0 +1,422 @@
+"""tools/ds4/ref.py - a whole-sequence DeepSeek-V4.1 forward in torch over an EXL3 checkpoint (the engine's oracle).
+
+ python tools/ds4/ref.py --model ~/models/DSV4 --text "The capital of France is" [--dump out.npz] [--gen 8]
+
+It follows DeepSeek's inference/model.py (MIT) at start_pos 0 - the prompt path, every position at once - with each
+layer's weights decoded from EXL3 when the layer runs (tools/ds4/exl3_torch.py) and freed after it, so one GPU of
+24 GB carries the 211 GB model: Hyper-Connections with the lagged pre-mix, sliding-window plus compressed sparse
+attention with the two-level indexer and its candidate blocks, the cross-layer KV / indexer sharing, Engram, the
+sqrt-softplus router, and the FP8 / FP4 activation rounding of the reference kernels (simulated, as their `inplace`
+paths do). Activations are F32 throughout (the reference keeps BF16 between ops).
+
+--dump writes every layer's residual stream ([T, hc, dim] F32 after each block), the logits and the token ids, for
+the engine's parity checks.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import pathlib
+import sys
+import time
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import engram_hash as EH # noqa: E402
+import exl3_torch as ET # noqa: E402
+from st import SafeTensors # noqa: E402
+
+
+# ------------------------------------------------------------------ the reference kernels' roundings
+def fp8_round(x: torch.Tensor, block: int = 32) -> torch.Tensor:
+ """act_quant(..., scale_fmt='ue8m0', inplace=True): per `block` values a power-of-two scale, E4M3 values."""
+ sh = x.shape
+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)
+ amax = v.abs().amax(-1, keepdim=True).clamp_min(1e-4)
+ s = torch.exp2(torch.ceil(torch.log2(amax / 448.0)))
+ y = (v / s).clamp(-448.0, 448.0).to(torch.float8_e4m3fn).float() * s
+ return y.reshape(sh)
+
+
+_FP4 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
+
+
+def _e2m1(v: torch.Tensor) -> torch.Tensor:
+ """Round to the E2M1 grid, nearest, ties to the even mantissa (values already in [-6, 6])."""
+ a = v.abs()
+ g = _FP4.to(v.device)
+ d = (a.unsqueeze(-1) - g).abs()
+ best = d.argmin(-1)
+ # ties: argmin takes the lower index; the even mantissa is the lower one at 0.25, 1.25, 2.5, 5 and the upper one
+ # at 0.75, 1.75, 3.5 (grid indices: mantissa bit = index & 1 for index >= 1, 0 at index 0)
+ lo = best
+ hi = (best + 1).clamp(max=7)
+ tie = (d.gather(-1, lo.unsqueeze(-1)).squeeze(-1) == d.gather(-1, hi.unsqueeze(-1)).squeeze(-1)) & (hi != lo)
+ take_hi = tie & ((lo & 1) == 1)
+ idx = torch.where(take_hi, hi, lo)
+ return torch.copysign(g[idx], v)
+
+
+def fp4_round(x: torch.Tensor, block: int, e4m3_scale: bool) -> torch.Tensor:
+ """fp4_act_quant(..., inplace=True): E2M1 values with an E8M0 (power of two) or an E4M3 scale per block."""
+ sh = x.shape
+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)
+ amax = v.abs().amax(-1, keepdim=True)
+ if e4m3_scale:
+ amax = amax.clamp_min(6 * 2.0 ** -9)
+ s = (amax / 6.0).to(torch.float8_e4m3fn).float()
+ else:
+ amax = amax.clamp_min(6 * 2.0 ** -126)
+ s = torch.exp2(torch.ceil(torch.log2(amax / 6.0)))
+ y = _e2m1((v / s).clamp(-6.0, 6.0)) * s
+ return y.reshape(sh)
+
+
+def rms(x: torch.Tensor, w: torch.Tensor | None, eps: float) -> torch.Tensor:
+ x = x.float()
+ y = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps)
+ return y * w if w is not None else y
+
+
+def freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow, dev) -> torch.Tensor:
+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+ if original_seq_len > 0:
+ def corr(rot):
+ return dim * math.log(original_seq_len / (rot * 2 * math.pi)) / (2 * math.log(base))
+ low = max(math.floor(corr(beta_fast)), 0)
+ high = min(math.ceil(corr(beta_slow)), dim - 1)
+ ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1)
+ smooth = 1 - ramp
+ freqs = freqs / factor * (1 - smooth) + freqs * smooth
+ f = torch.outer(torch.arange(seqlen, dtype=torch.float32), freqs)
+ return torch.polar(torch.ones_like(f), f).to(dev)
+
+
+def rope(x: torch.Tensor, fc: torch.Tensor, inverse: bool = False) -> torch.Tensor:
+ """Adjacent pairs as complex numbers. x [T, d] or [T, h, d]; fc [T, d/2]."""
+ c = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2).contiguous())
+ if inverse:
+ fc = fc.conj()
+ if x.ndim == 3:
+ fc = fc[:, None, :]
+ return torch.view_as_real(c * fc).flatten(-2)
+
+
+# ------------------------------------------------------------------ the model
+class Ref:
+ def __init__(self, model: pathlib.Path, dev: str = "cuda:0", max_seq: int = 8192):
+ self.dir = model
+ self.cfg = json.loads((model / "config.json").read_text())
+ self.t = t = self.cfg["text_config"]
+ self.st = SafeTensors(model)
+ self.dev = dev
+ self.dim, self.hc = t["hidden_size"], t["hc_mult"]
+ self.eps = t["rms_norm_eps"]
+ self.nl = t["num_hidden_layers"]
+ self.ratios = t["compress_ratios"]
+ self.kv_src = set(t["kv_source_layer_ids"])
+ self.ix_src = set(t["index_source_layer_ids"])
+ self.cand_src = t["candidate_source_layer_id"]
+ rs = t["rope_scaling"]
+ rd = t["qk_rope_head_dim"]
+ self.fc_window = freqs_cis(rd, max_seq, 0, t["rope_theta"], rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)
+ self.fc_comp = freqs_cis(rd, max_seq, rs["original_max_position_embeddings"], t["compress_rope_theta"],
+ rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)
+ self.lay = EH.Layout.from_config(t)
+ tm, n = EH.compressed_token_map(model / "tokenizer.json")
+ if n != t["engram_compressed_vocab_size"]:
+ raise SystemExit("engram: compressed vocab %d, the config says %d" % (n, t["engram_compressed_vocab_size"]))
+ self.hasher = EH.Hasher(self.lay, tm, EH.multipliers(self.lay, n), tm[t["engram_pad_token_id"]])
+ self.timing: dict[str, float] = {}
+
+ # weights
+ def W(self, prefix: str) -> torch.Tensor:
+ return ET.weight(self.st, prefix, self.dev)
+
+ def V(self, name: str) -> torch.Tensor:
+ return torch.from_numpy(np.array(self.st.f32(name))).to(self.dev)
+
+ # pieces
+ def hc_mixes(self, x, fn, scale, base):
+ hc = self.hc
+ xf = x.flatten(1).float()
+ r = torch.rsqrt(xf.square().mean(-1, keepdim=True) + self.eps)
+ mixes = (xf @ fn.T) * r # [T, (2 + hc) hc]
+ pre = torch.sigmoid(mixes[:, :hc] * scale[0] + base[:hc]) + self.t["hc_eps"]
+ post = 2 * torch.sigmoid(mixes[:, hc:2 * hc] * scale[1] + base[hc:2 * hc])
+ comb = (mixes[:, 2 * hc:] * scale[2] + base[2 * hc:]).view(-1, hc, hc)
+ eps = self.t["hc_eps"]
+ comb = comb.softmax(-1) + eps
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
+ for _ in range(self.t["hc_sinkhorn_iters"] - 1):
+ comb = comb / (comb.sum(-1, keepdim=True) + eps)
+ comb = comb / (comb.sum(-2, keepdim=True) + eps)
+ return pre, post, comb
+
+ @staticmethod
+ def hc_pre(x, mix): # [T, hc, d], [T, hc] -> [T, d]
+ return (mix.unsqueeze(-1) * x).sum(1)
+
+ @staticmethod
+ def hc_post(y, res, post, comb): # y [T, d], res [T, hc, d]
+ return post.unsqueeze(-1) * y.unsqueeze(1) + (comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)
+
+ def engram(self, li: int, x: torch.Tensor, hash_ids: np.ndarray) -> torch.Tensor:
+ """x [T, hc, d]; hash_ids [T, cols] absolute rows."""
+ P = "layers.%d.engram." % li
+ T, cols = hash_ids.shape
+ flat = hash_ids.reshape(-1)
+ wt = self.st.raw(P + "embed.weight").view(np.uint8).reshape(-1, self.lay.head_dim)
+ sc = self.st.raw(P + "embed.scale").view(np.uint8).reshape(-1, self.lay.head_dim // 32)
+ order = np.argsort(flat) # sequential reads
+ rows = np.empty((flat.size, self.lay.head_dim), np.uint8)
+ srow = np.empty((flat.size, self.lay.head_dim // 32), np.uint8)
+ rows[order] = wt[flat[order]]
+ srow[order] = sc[flat[order]]
+ v = torch.from_numpy(rows).to(self.dev).view(torch.float8_e4m3fn).float()
+ s = torch.from_numpy(srow).to(self.dev).view(torch.float8_e8m0fnu).float()
+ e = (v.view(-1, self.lay.head_dim // 32, 32) * s.unsqueeze(-1)).view(T, cols * self.lay.head_dim)
+ e = e.to(torch.bfloat16).float() # ParallelEngramEmbedding returns BF16
+ kv = e @ self.W(P + "wkv").T # [T, (hc + 1) d]
+ d, hc = self.dim, self.hc
+ key, value = kv[:, :hc * d].view(T, hc, d), kv[:, hc * d:]
+ w = self.V(P + "q_weight") * self.V(P + "k_weight") # [hc, d]
+ h = x.float()
+ rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)
+ dot = (h * w * key).sum(-1) * rstd * d ** -0.5
+ gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))
+ return h + gate.unsqueeze(-1) * value.unsqueeze(1)
+
+ def sparse_attn(self, q, kv, sink, idx, scale):
+ """q [T, H, D], kv [N, D], sink [H], idx [T, K] int (-1 none) -> [T, H, D]."""
+ T, H, D = q.shape
+ out = torch.empty_like(q)
+ for a in range(0, T, 256):
+ b = min(T, a + 256)
+ ii = idx[a:b]
+ valid = ii >= 0
+ g = kv[ii.clamp_min(0)] # [t, K, D]
+ s = torch.einsum("thd,tkd->thk", q[a:b], g) * scale
+ s = s.masked_fill(~valid.unsqueeze(1), float("-inf"))
+ m = torch.maximum(s.amax(-1), sink[None, :]) # [t, H]
+ m = torch.maximum(m, torch.full_like(m, -1e30))
+ p = torch.exp(s - m.unsqueeze(-1))
+ den = p.sum(-1) + torch.exp(sink[None, :] - m)
+ out[a:b] = torch.einsum("thk,tkd->thd", p, g) / den.unsqueeze(-1)
+ return out
+
+ @staticmethod
+ def candidates(logits, compress_lens, topk_blocks, block):
+ width = logits.size(-1)
+ scores = F.pad(logits, (0, -width % block), value=float("-inf"))
+ scores = scores.unflatten(-1, (-1, block)).amax(-1)
+ nb = scores.size(-1)
+ last = (compress_lens - 1) // block
+ scores = scores.masked_fill(torch.arange(nb, device=logits.device) == last, float("inf"))
+ top = scores.topk(min(topk_blocks, nb), dim=-1)
+ keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > float("-inf"))
+ return keep.repeat_interleave(block, dim=-1)[..., :width]
+
+ def attention(self, li: int, x: torch.Tensor, S: dict) -> torch.Tensor:
+ t = self.t
+ A = ("layers.%d.attn." % li) if li < self.nl else ("mtp.%d.attn." % (li - self.nl))
+ T = x.shape[0]
+ H, D, rd = t["num_attention_heads"], t["head_dim"], t["qk_rope_head_dim"]
+ ratio = self.ratios[li]
+ fc = (self.fc_comp if ratio else self.fc_window)[:T]
+ qr = rms(x @ self.W(A + "wq_a").T, self.V(A + "q_norm.weight"), self.eps)
+ q = (qr @ self.W(A + "wq_b").T).view(T, H, D)
+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)
+ kv = rms(x @ self.W(A + "wkv").T, self.V(A + "kv_norm.weight"), self.eps)
+ kv = torch.cat([kv[:, :-rd], rope(kv[:, -rd:], fc)], -1)
+ kv = fp8_round(kv)
+ win = t["sliding_window"]
+ end = torch.arange(T, device=self.dev).unsqueeze(1)
+ widx = (end - win + 1).clamp(0) + torch.arange(min(T, win), device=self.dev)
+ widx = torch.where(widx > end, -1, widx)
+ all_kv, all_idx = kv, widx
+ if ratio:
+ is_src = li < self.nl and li in self.kv_src
+ latent = None
+ if is_src:
+ C = A + "compressor."
+ if ratio == 1:
+ latent = rms(x @ self.W(C + "wkv").T, self.V(C + "norm.weight"), self.eps)
+ elif T >= ratio:
+ cut = T - T % ratio
+ kvc = x[:cut] @ self.W(C + "wkv").T
+ sc = x[:cut] @ self.W(C + "wgate").T
+ kvc, sc = kvc.view(-1, ratio, D), sc.view(-1, ratio, D)
+ latent = rms((kvc * sc.softmax(1)).sum(1), self.V(C + "norm.weight"), self.eps)
+ # the indexer (index sources) before the latent is rotated
+ if li < self.nl and li in self.ix_src:
+ S["topk"] = self.indexer(li, A, x, qr, latent, ratio, T, fc, S)
+ if latent is not None:
+ lat = torch.cat([latent[:, :-rd], rope(latent[:, -rd:], fc[:T - T % ratio:ratio])], -1)
+ S["ckv"] = fp4_round(lat, 16, True)
+ ckv = S["ckv"][:T // ratio] if S.get("ckv") is not None else kv.new_zeros(0, D)
+ all_kv = torch.cat([kv, ckv], 0)
+ all_idx = torch.cat([widx, S["topk"]], -1)
+ o = self.sparse_attn(q, all_kv, self.V(A + "attn_sink"), all_idx, D ** -0.5)
+ o = torch.cat([o[..., :-rd], rope(o[..., -rd:], fc, inverse=True)], -1)
+ G = t["o_groups"]
+ o = fp8_round(o.reshape(T, G, -1))
+ parts = [o[:, g] @ self.W(A + "wo_a.slice.%d" % g).T for g in range(G)]
+ return torch.cat(parts, -1) @ self.W(A + "wo_b").T
+
+ def indexer(self, li, A, x, qr, latent, ratio, T, fc, S):
+ t = self.t
+ I = A + "indexer."
+ nh, hd, rd = t["index_n_heads"], t["index_head_dim"], t["qk_rope_head_dim"]
+ clen = T // ratio
+ if clen == 0:
+ return torch.empty(T, 0, dtype=torch.long, device=self.dev)
+ if li in self.kv_src and latent is not None: # this layer owns the index keys
+ k = rms(latent @ self.W(I + "wk").T, self.V(I + "k_norm.weight"), self.eps)
+ k = torch.cat([k[:, :-rd], rope(k[:, -rd:], fc[:T - T % ratio:ratio])], -1)
+ S["ik"] = fp4_round(k, 32, False)
+ q = (qr @ self.W(I + "wq_b").T).view(T, nh, hd)
+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)
+ q = fp4_round(q, 32, False)
+ ik = S["ik"][:clen]
+ w = (x @ self.V(I + "weights_proj.weight").T) * (hd ** -0.5 * nh ** -0.5) # [T, nh]
+ score = torch.zeros(T, clen, device=self.dev)
+ for a in range(0, T, 512):
+ b = min(T, a + 512)
+ s = torch.einsum("thd,kd->thk", q[a:b], ik).relu_()
+ score[a:b] = (s * w[a:b].unsqueeze(-1)).sum(1)
+ lens = (torch.arange(1, T + 1, device=self.dev) // ratio).unsqueeze(-1)
+ score = score.masked_fill(torch.arange(clen, device=self.dev) >= lens, float("-inf"))
+ if li == self.cand_src:
+ S["cand"] = self.candidates(score, lens, t["candidate_topk_blocks"], t["candidate_block_size"])
+ elif 0 <= self.cand_src < li:
+ score = score.masked_fill(~S["cand"], float("-inf"))
+ topk = min(t["index_topk"], clen)
+ idx = score.topk(topk, dim=-1, sorted=False).indices.sort(-1).values
+ return torch.where(idx < lens, idx + T, -1)
+
+ def moe(self, li: int, x: torch.Tensor) -> torch.Tensor:
+ t = self.t
+ M = "layers.%d.ffn." % li
+ T = x.shape[0]
+ gw = self.V(M + "gate.weight")
+ scores = F.softplus(x.float() @ gw.T).sqrt()
+ idx = (scores + self.V(M + "gate.bias")).topk(t["num_experts_per_tok"], dim=-1)[1]
+ wts = scores.gather(1, idx)
+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * t["routed_scaling_factor"]
+ lim = t["swiglu_limit"]
+ y = torch.zeros(T, self.dim, device=self.dev)
+
+ def expert(P, xs):
+ g = (xs @ self.W(P + "w1").T).clamp(max=lim)
+ u = (xs @ self.W(P + "w3").T).clamp(-lim, lim)
+ return F.silu(g) * u, P
+
+ uniq = torch.unique(idx).tolist()
+ for a in range(0, len(uniq), 24): # 24 experts decoded at once
+ batch = uniq[a:a + 24]
+ Ps = [M + "experts.%d." % e for e in batch]
+ w1 = ET.decode_many(self.st, [P + "w1" for P in Ps], self.dev)
+ w3 = ET.decode_many(self.st, [P + "w3" for P in Ps], self.dev)
+ w2 = ET.decode_many(self.st, [P + "w2" for P in Ps], self.dev)
+ for i, e in enumerate(batch):
+ rows, slot = torch.where(idx == e)
+ xs = x[rows]
+ h = F.silu((xs @ w1[i].T).clamp(max=lim)) * (xs @ w3[i].T).clamp(-lim, lim)
+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ w2[i].T)
+ del w1, w2, w3
+ h, _ = expert(M + "shared_experts.", x)
+ y += h @ self.W(M + "shared_experts.w2").T
+ self.last_route = (idx, wts)
+ return y
+
+ @torch.inference_mode()
+ def forward(self, ids: list[int], dump: list | None = None) -> torch.Tensor:
+ T = len(ids)
+ hashes = self.hasher.ids(ids) # [T, n_engram, cols]
+ emb = self.st.raw("embed.weight").view(np.uint16).reshape(-1, self.dim)
+ h = torch.from_numpy(np.array(emb[np.asarray(ids)])).to(self.dev).view(torch.bfloat16).float()
+ h = h.unsqueeze(1).repeat(1, self.hc, 1)
+ pre = torch.zeros(T, self.hc, device=self.dev)
+ pre[:, 0] = 1.0
+ S: dict = {}
+ for li in range(self.nl):
+ t0 = time.time()
+ B = "layers.%d." % li
+ if li in self.lay.layer_ids:
+ h = self.engram(li, h, hashes[:, self.lay.layer_ids.index(li)])
+ res = h
+ a_pre, a_post, a_comb = self.hc_mixes(h, self.V(B + "hc_attn_fn"), self.V(B + "hc_attn_scale"),
+ self.V(B + "hc_attn_base"))
+ x = rms(self.hc_pre(h, pre), self.V(B + "attn_norm.weight"), self.eps)
+ h = self.hc_post(self.attention(li, x, S), res, a_post, a_comb)
+ res = h
+ f_pre, f_post, f_comb = self.hc_mixes(h, self.V(B + "hc_ffn_fn"), self.V(B + "hc_ffn_scale"),
+ self.V(B + "hc_ffn_base"))
+ x = rms(self.hc_pre(h, a_pre), self.V(B + "ffn_norm.weight"), self.eps)
+ h = self.hc_post(self.moe(li, x), res, f_post, f_comb)
+ pre = f_pre
+ if dump is not None:
+ dump.append(h.cpu().numpy())
+ self.timing[li] = time.time() - t0
+ x = rms(self.hc_pre(h, pre), self.V("norm.weight"), self.eps)
+ head = self.W("head")
+ return x @ head.T # [T, vocab]
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--model", default=str(pathlib.Path("~/models/DSV4").expanduser()))
+ ap.add_argument("--text", help="raw text (tokenized as is, no BOS added)")
+ ap.add_argument("--ids", help="comma-separated token ids")
+ ap.add_argument("--chat", help="a user message, as DeepSeek-V4.1's chat (non-thinking) prompt")
+ ap.add_argument("--gen", type=int, default=0, help="greedy tokens to append (a whole forward each)")
+ ap.add_argument("--dump", help="write the per-layer residual streams, the logits and the ids (.npz)")
+ ap.add_argument("--no-hs", action="store_true", help="--dump without the per-layer residual streams")
+ ap.add_argument("--dev", default="cuda:0")
+ a = ap.parse_args()
+ model = pathlib.Path(a.model).expanduser()
+ from tokenizers import Tokenizer
+ tok = Tokenizer.from_file(str(model / "tokenizer.json"))
+ if a.ids:
+ ids = [int(x) for x in a.ids.split(",")]
+ elif a.chat:
+ ids = tok.encode("<|begin▁of▁sentence|><|User|>" + a.chat + "<|Assistant|></think>",
+ add_special_tokens=False).ids
+ else:
+ ids = tok.encode(a.text or "The capital of France is", add_special_tokens=False).ids
+ r = Ref(model, a.dev)
+ print("tokens:", len(ids), ids[:32], flush=True)
+ dump = [] if a.dump and not a.no_hs else None
+ t0 = time.time()
+ logits = r.forward(ids, dump)
+ print("forward %.1fs" % (time.time() - t0), flush=True)
+ lp = torch.log_softmax(logits.float(), -1)
+ if len(ids) > 1:
+ tgt = torch.tensor(ids[1:], device=lp.device)
+ nll = -lp[:-1].gather(1, tgt[:, None]).squeeze(1)
+ print("ppl over %d tokens: %.4f" % (len(ids) - 1, math.exp(nll.mean().item())))
+ top = lp[-1].topk(8)
+ print("next:", [(tok.decode([i]), round(v, 3)) for v, i in zip(top.values.tolist(), top.indices.tolist())])
+ if a.dump:
+ np.savez(a.dump, ids=np.array(ids), logits=logits.cpu().numpy(),
+ hs=np.stack(dump) if dump else np.zeros(0))
+ out = list(ids)
+ for _ in range(a.gen):
+ nxt = int(lp[-1].argmax())
+ out.append(nxt)
+ print("gen:", repr(tok.decode(out[len(ids):], skip_special_tokens=False)), flush=True)
+ if nxt == 1:
+ break
+ logits = r.forward(out)
+ lp = torch.log_softmax(logits.float(), -1)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/ref_gguf.py b/tools/dsv41/ref_gguf.py
new file mode 100644
index 000000000..60e8abf85
--- /dev/null
+++ b/tools/dsv41/ref_gguf.py
@@ -0,0 +1,301 @@
+"""tools/ds4/ref_gguf.py - tools/ds4/ref.py's oracle over a GGUF (tools/ds4/pack_gguf.py's pack): the same forward,
+each weight dequantised in torch from the GGUF's own blocks (BF16, Q8_0, MXFP4, MXFP8) - a check of maya-ds4's GGUF
+path that shares none of its code.
+
+ python tools/ds4/ref_gguf.py --pack <model>/maya-ds4 --ids 0,671,6102 [--dump out.npz] [--gen 8]
+
+The token ids come from the server's tokenizer (maya-ds4 --serve, or the engine's own --ids runs); --dump writes
+what tools/ds4/compare.py reads.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import os
+import pathlib
+import struct
+import sys
+import time
+from types import SimpleNamespace
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import ref as R # noqa: E402
+
+BLOCK = {"Q8_0": 34, "MXFP4": 17, "MXFP8": 33}
+E2M1 = torch.tensor([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=torch.float32)
+
+
+class Pack:
+ """pack_gguf.py's tensors.txt over the GGUF's files (memory-mapped), and ds4.txt's geometry."""
+
+ def __init__(self, d: pathlib.Path):
+ self.kv = {}
+ for line in (d / "ds4.txt").read_text().splitlines():
+ k, _, v = line.partition(" ")
+ self.kv[k] = v
+ model = pathlib.Path(self.kv["model_dir"])
+ self.mm, self.t = {}, {}
+ for line in (d / "tensors.txt").read_text().splitlines():
+ f = line.split()
+ if f[0] == "file":
+ self.mm[int(f[1])] = np.memmap(model / f[2], dtype=np.uint8, mode="r")
+ elif f[0] == "t":
+ self.t[f[1]] = (f[2], int(f[3]), int(f[4]), int(f[5]), tuple(int(x) for x in f[6:]))
+ self._embed = None
+
+ def has(self, name):
+ return name in self.t
+
+ def raw(self, name) -> np.ndarray:
+ dt, fi, off, nb, _ = self.t[name]
+ return self.mm[fi][off:off + nb]
+
+ def num(self, k, d=0.0):
+ return float(self.kv.get(k, d))
+
+ def ints(self, k):
+ return [int(x) for x in self.kv.get(k, "").split(",") if x]
+
+
+def dequant(p: Pack, name: str, dev) -> torch.Tensor:
+ """A tensor as F32 on dev, shaped as stored ([out][in] for a matrix)."""
+ dt, _, _, _, shape = p.t[name]
+ r = torch.from_numpy(np.array(p.raw(name))).to(dev)
+ if dt == "F32":
+ v = r.view(torch.float32)
+ elif dt == "F16":
+ v = r.view(torch.float16).float()
+ elif dt == "BF16":
+ v = r.view(torch.bfloat16).float()
+ elif dt == "Q8_0":
+ b = r.view(-1, 34)
+ v = (b[:, :2].contiguous().view(torch.float16).float() * b[:, 2:].view(torch.int8).float()).reshape(-1)
+ elif dt == "MXFP4":
+ b = r.view(-1, 17)
+ s = torch.ldexp(torch.ones(1, device=dev), b[:, :1].int() - 128) # 2^(e - 128)
+ q = b[:, 1:].long()
+ lut = E2M1.to(dev)
+ v = (torch.cat([lut[q & 15], lut[q >> 4]], 1) * s).reshape(-1) # values 0..15, then 16..31
+ elif dt == "MXFP8":
+ b = r.view(-1, 33)
+ s = torch.ldexp(torch.ones(1, device=dev), b[:, :1].int() - 127)
+ v = (b[:, 1:].contiguous().view(torch.float8_e4m3fn).float() * s).reshape(-1)
+ else: # the low-bit expert types (IQ2_XXS .. Q4_K): gguf-py's numpy dequantisers (ggml's dequantize_row_*)
+ v = torch.from_numpy(gq_values(np.array(p.raw(name)), dt, name)).to(dev)
+ return v.reshape(shape)
+
+
+def gq_values(raw: np.ndarray, dt: str, name: str) -> np.ndarray:
+ """ggml blocks of type dt as F32 (flat) through gguf-py's dequantisers - the installed package, else a llama.cpp
+ checkout's ($GGUF_PY, ~/llama.cpp/gguf-py)."""
+ try:
+ try:
+ import gguf
+ except ImportError:
+ for d in (os.environ.get("GGUF_PY", ""), os.path.expanduser("~/llama.cpp/gguf-py")):
+ if d and os.path.isdir(d):
+ sys.path.insert(0, d)
+ import gguf
+ from gguf.quants import dequantize as gq_dequant
+ qt = gguf.GGMLQuantizationType[dt]
+ except (ImportError, KeyError):
+ raise SystemExit("%s: dtype %s (no gguf-py with a dequantiser for it: GGUF_PY=<llama.cpp>/gguf-py)" % (name, dt))
+ return np.ascontiguousarray(gq_dequant(raw, qt), dtype=np.float32).reshape(-1)
+
+
+class Hasher:
+ """engram_hash.Hasher from the pack's engram.bin (the token map, multipliers, primes and offsets)."""
+
+ def __init__(self, path: pathlib.Path):
+ b = path.read_bytes()
+ vocab, nl, ngram, heads, hd, pad = struct.unpack_from("<6i", b, 4)
+ cols = (ngram - 1) * heads
+ o = 28
+ self.tm = np.frombuffer(b, np.int32, vocab, o).astype(np.int64)
+ o += 4 * vocab
+ self.mult = np.frombuffer(b, np.int64, nl * ngram, o).reshape(nl, ngram)
+ o += 8 * nl * ngram
+ self.primes = np.frombuffer(b, np.int64, nl * cols, o).reshape(nl, ngram - 1, heads)
+ o += 8 * nl * cols
+ self.off = np.frombuffer(b, np.int64, nl * cols, o).reshape(nl, cols)
+ self.ngram, self.pad, self.hd = ngram, pad, hd
+
+ def ids(self, tokens) -> np.ndarray: # engram_hash.Hasher.ids
+ c = self.tm[np.asarray(tokens, dtype=np.int64)]
+ T = len(c)
+ pos = np.arange(T)
+ toks, blocked = [], np.zeros(T, dtype=bool)
+ for s in range(self.ngram):
+ src = c[np.clip(pos - s, 0, None)]
+ blocked = blocked | (pos < s)
+ toks.append(np.where(blocked, self.pad, src))
+ toks = np.stack(toks, axis=-1)
+ prod = toks[:, None, :] * self.mult[None]
+ rolling, out = prod[..., 0], []
+ for i in range(1, self.ngram):
+ rolling = np.bitwise_xor(rolling, prod[..., i])
+ out.append(rolling[..., None] % self.primes[None, :, i - 1])
+ return np.concatenate(out, axis=-1) + self.off[None]
+
+
+def q6k_rows(rows: np.ndarray, name: str) -> np.ndarray:
+ """Engram rows of Q6_K superblocks as F32 (ggml's dequantize_row_q6_K, through gguf-py)."""
+ return gq_values(rows.reshape(-1), "Q6_K", name)
+
+
+class RefGguf(R.Ref):
+ def __init__(self, pack: pathlib.Path, dev: str = "cuda:0", max_seq: int = 8192):
+ self.dir = pack
+ self.st = p = Pack(pack)
+ g = p.num
+ t = {k: int(g(k)) for k in ("hidden_size", "hc_mult", "num_hidden_layers", "candidate_source_layer_id",
+ "qk_rope_head_dim", "num_experts_per_tok", "sliding_window", "o_groups",
+ "num_attention_heads", "index_topk", "index_n_heads", "index_head_dim", "head_dim",
+ "candidate_topk_blocks", "candidate_block_size")}
+ t.update(hc_eps=g("hc_eps"), hc_sinkhorn_iters=int(g("hc_sinkhorn_iters")),
+ rms_norm_eps=g("rms_norm_eps"), routed_scaling_factor=g("routed_scaling_factor"),
+ swiglu_limit=g("swiglu_limit"), rope_theta=g("rope_theta"), compress_rope_theta=g("compress_rope_theta"),
+ compress_ratios=p.ints("compress_ratios"), kv_source_layer_ids=p.ints("kv_source_layer_ids"),
+ index_source_layer_ids=p.ints("index_source_layer_ids"),
+ rope_scaling={"factor": g("rope_scaling.factor"), "beta_fast": g("rope_scaling.beta_fast"),
+ "beta_slow": g("rope_scaling.beta_slow"),
+ "original_max_position_embeddings": int(g("rope_scaling.original_max_position_embeddings"))})
+ self.cfg = {"text_config": t}
+ self.t = t
+ self.dev = dev
+ self.dim, self.hc = t["hidden_size"], t["hc_mult"]
+ self.eps = t["rms_norm_eps"]
+ self.nl = t["num_hidden_layers"]
+ self.ratios = t["compress_ratios"]
+ self.kv_src = set(t["kv_source_layer_ids"])
+ self.ix_src = set(t["index_source_layer_ids"])
+ self.cand_src = t["candidate_source_layer_id"]
+ rs, rd = t["rope_scaling"], t["qk_rope_head_dim"]
+ self.fc_window = R.freqs_cis(rd, max_seq, 0, t["rope_theta"], rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)
+ self.fc_comp = R.freqs_cis(rd, max_seq, rs["original_max_position_embeddings"], t["compress_rope_theta"],
+ rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)
+ self.hasher = Hasher(pack / "engram.bin")
+ self.lay = SimpleNamespace(layer_ids=tuple(p.ints("engram_layer_ids")), head_dim=self.hasher.hd)
+ self.timing = {}
+ self._embed = None
+
+ def W(self, prefix: str) -> torch.Tensor:
+ return dequant(self.st, prefix + ".weight", self.dev)
+
+ def V(self, name: str) -> torch.Tensor:
+ return dequant(self.st, name, self.dev).float()
+
+ def engram(self, li, x, hash_ids):
+ P = "layers.%d.engram." % li
+ T, cols = hash_ids.shape
+ hd = self.lay.head_dim
+ dt = self.st.t[P + "embed.weight"][0]
+ rb = hd // 256 * 210 if dt == "Q6_K" else hd // 32 * 33 # (a row's bytes: Q6_K or MXFP8 blocks)
+ tab = self.st.raw(P + "embed.weight").reshape(-1, rb)
+ flat = hash_ids.reshape(-1)
+ order = np.argsort(flat)
+ rows = np.empty((flat.size, rb), np.uint8)
+ rows[order] = tab[flat[order]]
+ if dt == "Q6_K":
+ e = torch.from_numpy(q6k_rows(rows, P)).to(self.dev).view(T, cols * hd)
+ else:
+ b = torch.from_numpy(rows).to(self.dev).view(-1, 33)
+ s = torch.ldexp(torch.ones(1, device=self.dev), b[:, :1].int() - 127)
+ e = (b[:, 1:].contiguous().view(torch.float8_e4m3fn).float() * s).view(T, cols * hd)
+ e = e.to(torch.bfloat16).float()
+ kv = e @ self.W(P + "wkv").T
+ d, hc = self.dim, self.hc
+ key, value = kv[:, :hc * d].view(T, hc, d), kv[:, hc * d:]
+ w = self.V(P + "gate.weight") # q_weight * k_weight (folded)
+ h = x.float()
+ rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)
+ dot = (h * w * key).sum(-1) * rstd * d ** -0.5
+ gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))
+ return h + gate.unsqueeze(-1) * value.unsqueeze(1)
+
+ def moe(self, li, x):
+ t = self.t
+ M = "layers.%d.ffn." % li
+ T = x.shape[0]
+ gw = self.V(M + "gate.weight")
+ scores = F.softplus(x.float() @ gw.T).sqrt()
+ idx = (scores + self.V(M + "gate.bias")).topk(t["num_experts_per_tok"], dim=-1)[1]
+ wts = scores.gather(1, idx)
+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * t["routed_scaling_factor"]
+ lim = t["swiglu_limit"]
+ y = torch.zeros(T, self.dim, device=self.dev)
+ for e in torch.unique(idx).tolist():
+ P = M + "experts.%d." % e
+ rows, slot = torch.where(idx == e)
+ xs = x[rows]
+ h = F.silu((xs @ self.W(P + "w1").T).clamp(max=lim)) * (xs @ self.W(P + "w3").T).clamp(-lim, lim)
+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ self.W(P + "w2").T)
+ S = M + "shared_experts."
+ h = F.silu((x @ self.W(S + "w1").T).clamp(max=lim)) * (x @ self.W(S + "w3").T).clamp(-lim, lim)
+ y += h @ self.W(S + "w2").T
+ self.last_route = (idx, wts)
+ return y
+
+ @torch.inference_mode()
+ def forward(self, ids, dump=None):
+ # ref.py's forward reads the embedding rows as BF16: a Q8_0 table is dequantised for the rows asked
+ dt = self.st.t["embed.weight"][0]
+ if dt != "BF16":
+ full = self.st.raw("embed.weight").reshape(-1, self.dim // 32 * 34)
+ sel = torch.from_numpy(np.array(full[np.asarray(ids)])).to(self.dev).view(-1, 34)
+ v = (sel[:, :2].contiguous().view(torch.float16).float() * sel[:, 2:].view(torch.int8).float())
+ rows = v.reshape(len(ids), self.dim).to(torch.bfloat16).view(torch.uint16).cpu().numpy()
+ raw0 = self.st.raw
+ emb = np.zeros((max(ids) + 1, self.dim), np.uint16)
+ emb[np.asarray(ids)] = rows
+ self.st.raw = lambda n, e=emb, r=raw0: e.view(np.uint8) if n == "embed.weight" else r(n)
+ try:
+ return super().forward(ids, dump)
+ finally:
+ self.st.raw = raw0
+ return super().forward(ids, dump)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--pack", required=True, help="pack_gguf.py's pack (<model>/maya-ds4)")
+ ap.add_argument("--ids", required=True, help="comma-separated token ids")
+ ap.add_argument("--gen", type=int, default=0, help="greedy tokens to append (a whole forward each)")
+ ap.add_argument("--dump", help="write the per-layer residual streams, the logits and the ids (.npz)")
+ ap.add_argument("--no-hs", action="store_true", help="--dump without the per-layer residual streams")
+ ap.add_argument("--dev", default="cuda:0")
+ a = ap.parse_args()
+ ids = [int(x) for x in a.ids.split(",")]
+ r = RefGguf(pathlib.Path(a.pack).expanduser(), a.dev)
+ print("tokens:", len(ids), ids[:32], flush=True)
+ dump = [] if a.dump and not a.no_hs else None
+ t0 = time.time()
+ logits = r.forward(ids, dump)
+ print("forward %.1fs" % (time.time() - t0), flush=True)
+ lp = torch.log_softmax(logits.float(), -1)
+ if len(ids) > 1:
+ tgt = torch.tensor(ids[1:], device=lp.device)
+ nll = -lp[:-1].gather(1, tgt[:, None]).squeeze(1)
+ print("ppl over %d tokens: %.4f" % (len(ids) - 1, math.exp(nll.mean().item())))
+ top = lp[-1].topk(8)
+ print("next:", list(zip(top.indices.tolist(), [round(v, 3) for v in top.values.tolist()])))
+ if a.dump:
+ np.savez(a.dump, ids=np.array(ids), logits=logits.cpu().numpy(), hs=np.stack(dump) if dump else np.zeros(0))
+ out = list(ids)
+ for _ in range(a.gen):
+ nxt = int(lp[-1].argmax())
+ out.append(nxt)
+ print("gen ids:", out[len(ids):], flush=True)
+ if nxt == 1:
+ break
+ lp = torch.log_softmax(r.forward(out).float(), -1)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/requant_gguf.py b/tools/dsv41/requant_gguf.py
new file mode 100644
index 000000000..0740c7492
--- /dev/null
+++ b/tools/dsv41/requant_gguf.py
@@ -0,0 +1,449 @@
+"""tools/ds4/requant_gguf.py - a DeepSeek-V4.1 GGUF requantized by a recipe (the routed experts in ggml's low-bit types,
+the dense matrices in Q8_0), through ggml's own quantizers (libggml-base, ctypes) - a quick test model for maya-ds4's
+low-bit expert kernels until the real Maya quants exist.
+
+ python tools/ds4/requant_gguf.py --src ~/models/DeepSeek-V4.1-Flash-1017-GGUF/UD-Q8_K_XL \\
+ --out ~/ds4test/Maya-S-test --recipe recipe.json --lib ~/llama.cpp/build/bin/libggml-base.so --jobs 10
+
+The recipe (JSON):
+ {"experts": {"gate": "IQ2_XXS", "up": "IQ2_XXS", "down": "IQ3_XXS"},
+ "layers": [{"layers": "0-1,38-39", "gate": "IQ2_S", "up": "IQ2_S", "down": "IQ3_S"}],
+ "dense": "Q8_0",
+ "dense_keep": ["hc_", "engram", "norm", "attn_sinks", "exp_probs_b", "ffn_gate_inp"],
+ "engram": "skip"}
+- experts: each role's type (gate = ffn_gate_exps, up = ffn_up_exps, down = ffn_down_exps); "layers" entries override it
+ for the listed blocks (later entries win). A role's type may be "keep".
+- dense: the type of the other float matrices (2-D or more, F32 / F16 / BF16, rows a multiple of 32), except names
+ containing a dense_keep string, which stay as they are; "keep" keeps them all.
+- engram: "keep" copies the Engram tables (engram_*) into the output, "skip" leaves them out (a shared Engram shard set
+ beside the quant: pack_gguf.py reads <model>/Engram/*.gguf or <model>/../Engram/*.gguf); --engram-link then links the
+ source's shards there as that set.
+
+The output has one shard a source shard (the same tensors in the same order, the metadata copied; split.tensors.count
+updated), tensors at the source's alignment. Workers (processes) read the source with O_DIRECT, so the page cache that
+running engines rely on is neither evicted nor filled, and drop what they wrote from it; they run at SCHED_IDLE.
+
+IQ2_XXS / IQ2_XS / IQ1_* need an importance matrix: --imatrix (llama.cpp's legacy .dat) or all ones.
+"""
+from __future__ import annotations
+
+import argparse
+import ctypes
+import json
+import mmap
+import multiprocessing as mp
+import os
+import pathlib
+import re
+import struct
+import sys
+import time
+
+import numpy as np
+
+# ggml type id -> (name, values a block, bytes a block)
+GT = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110),
+ 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176), 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66),
+ 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50), 20: ("IQ4_NL", 32, 18),
+ 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
+ 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 67: ("MXFP8", 32, 33)}
+BYNAME = {v[0]: k for k, v in GT.items()}
+NEED_IMATRIX = {16, 17, 19, 29}
+ROLE = {"ffn_gate_exps.weight": "gate", "ffn_up_exps.weight": "up", "ffn_down_exps.weight": "down"}
+ALIGN_IO = 4096
+
+
+def nbytes_of(typ: int, dims: list[int]) -> int:
+ _, per, bpb = GT[typ]
+ n = 1
+ for d in dims:
+ n *= d
+ return n // per * bpb
+
+
+class Shard:
+ """A GGUF shard: its KV records (raw bytes each, in order) and tensor infos (absolute data offsets)."""
+
+ def __init__(self, path: pathlib.Path):
+ self.path = path
+ with path.open("rb") as f:
+ hdr = f.read(24)
+ if hdr[:4] != b"GGUF":
+ raise SystemExit("%s: not a GGUF file" % path)
+ self.version, nt, nkv = struct.unpack("<IQQ", hdr[4:])
+ buf = f.read(64 << 20) # (the metadata: the vocabulary is a few MB)
+ pos = 0
+
+ def take(n):
+ nonlocal pos, buf
+ while pos + n > len(buf):
+ more = f.read(64 << 20)
+ if not more:
+ raise SystemExit("%s: truncated metadata" % path)
+ buf += more
+ v = buf[pos:pos + n]
+ pos += n
+ return v
+
+ def s():
+ return take(struct.unpack("<Q", take(8))[0]).decode("utf-8", "replace")
+
+ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
+
+ def val(t):
+ if t in fmt:
+ return struct.unpack("<" + fmt[t], take(struct.calcsize(fmt[t])))[0]
+ if t == 8:
+ return s()
+ if t == 9:
+ at, n = struct.unpack("<IQ", take(12))
+ if at in fmt:
+ return list(struct.unpack("<%d%s" % (n, fmt[at]), take(n * struct.calcsize(fmt[at]))))
+ return [val(at) for _ in range(n)]
+ raise SystemExit("%s: metadata type %d" % (path, t))
+
+ self.kv, self.kv_raw = {}, []
+ for _ in range(nkv):
+ p0 = pos
+ k = s()
+ self.kv[k] = val(struct.unpack("<I", take(4))[0])
+ self.kv_raw.append((k, bytes(buf[p0:pos])))
+ self.tensors = []
+ for _ in range(nt):
+ name = s()
+ nd = struct.unpack("<I", take(4))[0]
+ dims = list(struct.unpack("<%dQ" % nd, take(8 * nd)))
+ typ, off = struct.unpack("<IQ", take(12))
+ self.tensors.append([name, typ, dims, off])
+ self.align = int(self.kv.get("general.alignment", 32))
+ start = 24 + pos
+ self.data_start = (start + self.align - 1) // self.align * self.align
+ for t in self.tensors:
+ if t[1] not in GT:
+ raise SystemExit("%s: %s has ggml type %d, which this tool does not read" % (path.name, t[0], t[1]))
+ t[3] += self.data_start
+
+
+def kv_int(raw: bytes, v: int) -> bytes:
+ """An integer KV record with a new value, in its own type."""
+ kl = struct.unpack("<Q", raw[:8])[0]
+ t = struct.unpack("<I", raw[8 + kl:12 + kl])[0]
+ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 10: "Q", 11: "q"}[t]
+ return raw[:12 + kl] + struct.pack("<" + fmt, v)
+
+
+def layer_set(spec: str) -> set[int]:
+ out = set()
+ for part in str(spec).split(","):
+ part = part.strip()
+ if not part:
+ continue
+ if "-" in part:
+ a, b = part.split("-")
+ out.update(range(int(a), int(b) + 1))
+ else:
+ out.add(int(part))
+ return out
+
+
+def plan_type(recipe: dict, name: str, typ: int, dims: list[int]) -> int | None:
+ """The output type of a tensor (None: copied as it is, -1: left out)."""
+ m = re.match(r"blk\.(\d+)\.(.+)$", name)
+ short = m.group(2) if m else name
+ if short.startswith("engram_"):
+ return None if recipe.get("engram", "skip") == "keep" else -1
+ if m and short in ROLE:
+ role = ROLE[short]
+ t = recipe.get("experts", {}).get(role, "keep")
+ for ov in recipe.get("layers", []):
+ if int(m.group(1)) in layer_set(ov["layers"]) and role in ov:
+ t = ov[role]
+ if t == "keep":
+ return None
+ if t not in BYNAME:
+ raise SystemExit("recipe: unknown type %r" % t)
+ q = BYNAME[t]
+ if dims[0] % GT[q][1]:
+ raise SystemExit("%s: rows of %d are not whole %s blocks" % (name, dims[0], t))
+ return q
+ dense = recipe.get("dense", "keep")
+ if dense == "keep" or typ not in (1, 30) or len(dims) < 2 or dims[0] % 32: # (F32: the router, mHC - kept)
+ return None
+ if any(k in short for k in recipe.get("dense_keep", [])):
+ return None
+ q = BYNAME[dense]
+ if dims[0] % GT[q][1]:
+ return None
+ return q
+
+
+# ---------------------------------------------------------------- workers
+_lib = None
+_fds = {}
+_imat = None
+
+
+def _init(lib_path: str, imatrix: str | None):
+ global _lib, _imat
+ try:
+ os.sched_setscheduler(0, os.SCHED_IDLE, os.sched_param(0))
+ except (AttributeError, OSError):
+ try:
+ os.nice(19)
+ except OSError:
+ pass
+ _lib = ctypes.CDLL(lib_path)
+ _lib.ggml_quantize_chunk.restype = ctypes.c_size_t
+ _lib.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,
+ ctypes.c_int64, ctypes.c_void_p]
+ _lib.ggml_quantize_init.argtypes = [ctypes.c_int]
+ _imat = load_imatrix(imatrix) if imatrix else None
+
+
+def load_imatrix(path: str) -> dict:
+ """llama.cpp's legacy imatrix .dat: {tensor name: float32 values (ne0, or ne0 * n_expert for experts)}."""
+ out = {}
+ with open(path, "rb") as f:
+ n = struct.unpack("<i", f.read(4))[0]
+ for _ in range(n):
+ ln = struct.unpack("<i", f.read(4))[0]
+ name = f.read(ln).decode()
+ ncall, nval = struct.unpack("<ii", f.read(8))
+ v = np.frombuffer(f.read(4 * nval), dtype=np.float32).copy()
+ out[name] = v / max(1, ncall)
+ return out
+
+
+def _fd(path: str, direct: bool):
+ key = (path, direct)
+ if key not in _fds:
+ flags = os.O_RDONLY | (getattr(os, "O_DIRECT", 0) if direct else 0)
+ try:
+ _fds[key] = os.open(path, flags)
+ except OSError:
+ _fds[key] = os.open(path, os.O_RDONLY)
+ return _fds[key]
+
+
+def read_src(path: str, off: int, n: int) -> memoryview:
+ a0 = off // ALIGN_IO * ALIGN_IO
+ a1 = (off + n + ALIGN_IO - 1) // ALIGN_IO * ALIGN_IO
+ buf = mmap.mmap(-1, a1 - a0)
+ try:
+ got = os.preadv(_fd(path, True), [buf], a0)
+ except OSError: # (no O_DIRECT here: a plain read)
+ got = os.preadv(_fd(path, False), [buf], a0)
+ if got < off - a0 + n:
+ raise IOError("%s: short read at %d" % (path, off))
+ return memoryview(buf)[off - a0:off - a0 + n]
+
+
+E2M1 = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)
+
+
+def to_f32(typ: int, raw: memoryview, n: int) -> np.ndarray:
+ b = np.frombuffer(raw, dtype=np.uint8)
+ if typ == 0:
+ return np.frombuffer(raw, dtype=np.float32).copy()
+ if typ == 1:
+ return np.frombuffer(raw, dtype=np.float16).astype(np.float32)
+ if typ == 30:
+ return (np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16).view(np.float32)
+ if typ == 8:
+ blk = b.reshape(-1, 34)
+ d = blk[:, :2].copy().view(np.float16).astype(np.float32)
+ q = blk[:, 2:].view(np.int8).astype(np.float32)
+ return (q * d).reshape(-1)
+ if typ == 39: # ggml's MXFP4: [e8m0][16 code bytes], value j = kvalues[code] * 2^(e - 128)
+ blk = b.reshape(-1, 17)
+ e = blk[:, 0].astype(np.int32)
+ d = np.ldexp(np.float32(1.0), e - 128).astype(np.float32)
+ q = blk[:, 1:]
+ v = np.empty((blk.shape[0], 32), dtype=np.float32)
+ v[:, :16] = E2M1[q & 15]
+ v[:, 16:] = E2M1[q >> 4]
+ return (v * d[:, None]).reshape(-1)
+ raise ValueError("source type %d" % typ)
+
+
+def _job(j: dict):
+ """One piece of a tensor: rows [r0, r1) (of ne0 values each) quantized, or bytes copied; written in place."""
+ t0 = time.time()
+ out_fd = os.open(j["out"], os.O_WRONLY)
+ try:
+ if j["q"] is None: # a copy, in pieces of 256 MB
+ n, done = j["n"], 0
+ while done < n:
+ k = min(256 << 20, n - done)
+ os.pwrite(out_fd, read_src(j["src"], j["src_off"] + done, k), j["out_off"] + done)
+ done += k
+ else:
+ ne0, r0, r1, st, q = j["ne0"], j["r0"], j["r1"], j["typ"], j["q"]
+ srb = GT[st][2] * ne0 // GT[st][1]
+ raw = read_src(j["src"], j["src_off"] + r0 * srb, (r1 - r0) * srb)
+ x = np.ascontiguousarray(to_f32(st, raw, (r1 - r0) * ne0), dtype=np.float32)
+ del raw
+ qrb = GT[q][2] * ne0 // GT[q][1]
+ dst = np.empty((r1 - r0) * qrb, dtype=np.uint8)
+ imat = None
+ if q in NEED_IMATRIX or (_imat is not None and j["iname"] in _imat):
+ w = _imat.get(j["iname"]) if _imat is not None else None
+ if w is None or w.size % ne0:
+ w = np.ones(ne0, dtype=np.float32)
+ imat = w
+ rows_per = j["rows_per_expert"]
+ r = r0
+ while r < r1: # (one expert's rows at a time: an expert's own importance row)
+ n = min(r1 - r, rows_per - r % rows_per if rows_per else r1 - r)
+ im = None
+ if imat is not None:
+ e = r // rows_per if rows_per else 0
+ im = imat[e * ne0:(e + 1) * ne0] if imat.size >= (e + 1) * ne0 else imat[:ne0]
+ im = np.ascontiguousarray(im, dtype=np.float32)
+ got = _lib.ggml_quantize_chunk(q, x[(r - r0) * ne0:].ctypes.data, dst[(r - r0) * qrb:].ctypes.data,
+ 0, n, ne0, im.ctypes.data if im is not None else None)
+ if got != n * qrb:
+ raise RuntimeError("%s: ggml_quantize_chunk wrote %d bytes, expected %d" % (j["name"], got, n * qrb))
+ r += n
+ os.pwrite(out_fd, memoryview(dst), j["out_off"] + r0 * qrb)
+ j["n"] = (r1 - r0) * qrb
+ j["out_off"] += r0 * qrb
+ os.fdatasync(out_fd)
+ try:
+ os.posix_fadvise(out_fd, j["out_off"], j["n"], os.POSIX_FADV_DONTNEED)
+ except (AttributeError, OSError):
+ pass
+ finally:
+ os.close(out_fd)
+ return j["name"], j["n"], time.time() - t0
+
+
+# ---------------------------------------------------------------- main
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--src", required=True, help="the source GGUF's shards (a directory)")
+ ap.add_argument("--out", required=True, help="the output directory")
+ ap.add_argument("--recipe", required=True, help="the recipe (JSON file)")
+ ap.add_argument("--lib", default=os.path.expanduser("~/llama.cpp/build/bin/libggml-base.so"),
+ help="libggml-base (ggml_quantize_chunk)")
+ ap.add_argument("--imatrix", help="llama.cpp's legacy imatrix .dat (default: all ones where a type needs one)")
+ ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 2) - 2))
+ ap.add_argument("--experts-per-job", type=int, default=4)
+ ap.add_argument("--engram-link", action="store_true",
+ help="(engram skip) link the source's shards as <out>/Engram/, the shared Engram set")
+ ap.add_argument("--dry-run", action="store_true", help="print the plan and the output size")
+ ap.add_argument("--only", help="regex: quantize only the matching tensors (the rest zero-filled: tests)")
+ ap.add_argument("--first", help="regex: the matching tensors' pieces first (e.g. a few layers for a quick test)")
+ ap.add_argument("--resume", action="store_true",
+ help="keep the output files of an earlier run (same plan) and skip the pieces already written")
+ a = ap.parse_args()
+
+ src = pathlib.Path(a.src).expanduser().resolve()
+ out = pathlib.Path(a.out).expanduser()
+ recipe = json.loads(pathlib.Path(a.recipe).read_text())
+ shards = sorted(src.glob("*.gguf"))
+ if not shards:
+ raise SystemExit("%s: no .gguf files" % src)
+ ss = [Shard(p) for p in shards]
+
+ jobs, plan_sizes, outs = [], {}, []
+ for si, sh in enumerate(ss):
+ kept = []
+ for name, typ, dims, off in sh.tensors:
+ q = plan_type(recipe, name, typ, dims)
+ if q == -1:
+ continue
+ kept.append((name, typ, dims, off, typ if q is None else q, q))
+ # the output's layout: the KV records (split.tensors.count updated), the tensor infos, the data
+ kv = []
+ for k, raw in sh.kv_raw:
+ if k == "split.tensors.count":
+ total = sum(1 for s2 in ss for t in s2.tensors if plan_type(recipe, t[0], t[1], t[2]) != -1)
+ raw = kv_int(raw, total)
+ kv.append(raw)
+ tinfo, data_off = [], 0
+ for name, typ, dims, off, otyp, q in kept:
+ nb = nbytes_of(otyp, dims)
+ tinfo.append((name, dims, otyp, data_off, nb))
+ data_off += (nb + sh.align - 1) // sh.align * sh.align
+ plan_sizes[GT[otyp][0]] = plan_sizes.get(GT[otyp][0], 0) + nb
+ head = struct.pack("<4sIQQ", b"GGUF", sh.version, len(tinfo), len(kv)) + b"".join(kv)
+ for name, dims, otyp, doff, nb in tinfo:
+ nm = name.encode()
+ head += struct.pack("<Q", len(nm)) + nm + struct.pack("<I", len(dims))
+ head += struct.pack("<%dQ" % len(dims), *dims) + struct.pack("<IQ", otyp, doff)
+ dstart = (len(head) + sh.align - 1) // sh.align * sh.align
+ opath = out / sh.path.name.replace("UD-Q8_K_XL", "requant")
+ outs.append((opath, head, dstart + data_off))
+ for (name, typ, dims, off, otyp, q), (_, _, _, doff, nb) in zip(kept, tinfo):
+ if a.only and not re.search(a.only, name):
+ continue
+ base = {"name": name, "src": str(sh.path), "src_off": off, "out": str(opath), "out_off": dstart + doff}
+ if q is None:
+ jobs.append(dict(base, q=None, n=nb))
+ continue
+ ne0 = dims[0]
+ rows = 1
+ for d in dims[1:]:
+ rows *= d
+ rpe = dims[1] if len(dims) == 3 else 0 # rows an expert (3-D: [n_exp][rows][ne0])
+ step = rpe * a.experts_per_job if rpe else max(1, (64 << 20) // (4 * ne0))
+ m = re.match(r"blk\.(\d+)\.(.+)$", name)
+ iname = name
+ for r0 in range(0, rows, step):
+ jobs.append(dict(base, q=q, typ=typ, ne0=ne0, r0=r0, r1=min(rows, r0 + step), rows_per_expert=rpe,
+ iname=iname, n=0))
+ total = sum(o[2] for o in outs)
+ print("plan: %d shards, %.1f GB (%s), %d jobs" % (len(outs), total / 1e9,
+ ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(plan_sizes.items())), len(jobs)), flush=True)
+ if a.dry_run:
+ return 0
+ out.mkdir(parents=True, exist_ok=True)
+ for opath, head, size in outs:
+ if a.resume and opath.is_file() and opath.stat().st_size == size:
+ with open(opath, "r+b") as f:
+ if f.read(len(head)) != head:
+ raise SystemExit("%s: not this plan's file (--resume)" % opath)
+ continue
+ with open(opath, "wb") as f:
+ f.write(head)
+ f.truncate(size)
+ if a.resume: # a piece is written when the first and last 256 bytes of its output are not all zero
+ def written(j):
+ n = j["n"] if j["q"] is None else (j["r1"] - j["r0"]) * GT[j["q"]][2] * j["ne0"] // GT[j["q"]][1]
+ o = j["out_off"] + (0 if j["q"] is None else j["r0"] * GT[j["q"]][2] * j["ne0"] // GT[j["q"]][1])
+ with open(j["out"], "rb") as f:
+ f.seek(o)
+ h = f.read(min(256, n))
+ f.seek(o + max(0, n - 256))
+ t = f.read(min(256, n))
+ return any(h) and any(t)
+ before = len(jobs)
+ jobs = [j for j in jobs if not written(j)]
+ print("resume: %d of %d pieces already written" % (before - len(jobs), before), flush=True)
+ if a.engram_link and recipe.get("engram", "skip") == "skip":
+ eg = out / "Engram"
+ eg.mkdir(exist_ok=True)
+ for p in shards:
+ link = eg / p.name
+ if not link.exists():
+ link.symlink_to(p)
+ # the biggest pieces first (the experts), so the last minutes are not one long tensor
+ jobs.sort(key=lambda j: -(j.get("r1", 0) - j.get("r0", 0)) * j.get("ne0", 0) if j["q"] is not None else 0)
+ if a.first:
+ jobs.sort(key=lambda j: 0 if re.search(a.first, j["name"]) else 1)
+ t0, done_b, done_j = time.time(), 0, 0
+ with mp.get_context("fork").Pool(a.jobs, initializer=_init, initargs=(os.path.expanduser(a.lib), a.imatrix)) as pool:
+ for name, nb, dt in pool.imap_unordered(_job, jobs):
+ done_b += nb
+ done_j += 1
+ if done_j % 50 == 0 or done_j == len(jobs):
+ el = time.time() - t0
+ print("%d / %d jobs, %.1f GB written, %.0f s (%.0f min left)" %
+ (done_j, len(jobs), done_b / 1e9, el, el / done_j * (len(jobs) - done_j) / 60), flush=True)
+ print("done: %s (%.1f GB) in %.1f min" % (out, total / 1e9, (time.time() - t0) / 60))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/st.py b/tools/dsv41/st.py
new file mode 100644
index 000000000..de70f0ba2
--- /dev/null
+++ b/tools/dsv41/st.py
@@ -0,0 +1,64 @@
+"""tools/ds4/st.py - every tensor of a DeepSeek-V4.1 model directory, read in place.
+
+ st = SafeTensors(Path("~/models/DSV4").expanduser())
+ st.t["layers.0.attn.wq_a.trellis"] -> (file, dtype, shape, absolute offset, bytes)
+ st.raw(name), st.f32(name) -> numpy views / float32 copies
+
+The index's own files come first; any other .safetensors in the directory (the Engram shards 47 / 48 of the base
+repo, model-0004[78]-of-00048.safetensors) only adds names the index does not have.
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+import struct
+
+import numpy as np
+
+DTYPE_BYTES = {"F32": 4, "F16": 2, "BF16": 2, "I32": 4, "I16": 2, "F8_E4M3": 1, "F8_E8M0": 1, "U8": 1, "I8": 1,
+ "I64": 8}
+
+
+class SafeTensors:
+ def __init__(self, d: pathlib.Path):
+ self.dir = pathlib.Path(d)
+ ix = json.loads((self.dir / "model.safetensors.index.json").read_text())["weight_map"]
+ primary = sorted(set(ix.values()))
+ others = sorted(p.name for p in self.dir.glob("*.safetensors") if p.name not in primary)
+ self.files: list[str] = []
+ self.t: dict[str, tuple] = {}
+ self.mm: dict[str, np.memmap] = {}
+ for fn in primary + others:
+ p = self.dir / fn
+ with p.open("rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ h = json.loads(f.read(n))
+ base = 8 + n
+ self.files.append(fn)
+ for k, v in h.items():
+ if k == "__metadata__" or k in self.t:
+ continue
+ a, b = v["data_offsets"]
+ self.t[k] = (fn, v["dtype"], tuple(v["shape"]), base + a, b - a)
+ self.mm[fn] = np.memmap(p, dtype=np.uint8, mode="r")
+
+ def has(self, name: str) -> bool:
+ return name in self.t
+
+ def raw(self, name: str) -> np.ndarray:
+ fn, _, _, off, nb = self.t[name]
+ return self.mm[fn][off:off + nb]
+
+ def f32(self, name: str) -> np.ndarray:
+ """The tensor as float32 (exact for F32/F16/BF16), shaped as stored."""
+ _, dt, shape, _, _ = self.t[name]
+ r = self.raw(name)
+ if dt == "F32":
+ a = r.view("<f4")
+ elif dt == "F16":
+ a = r.view("<f2").astype(np.float32)
+ elif dt == "BF16":
+ a = (r.view("<u2").astype(np.uint32) << 16).view(np.float32)
+ else:
+ raise ValueError("%s is %s, not a float tensor" % (name, dt))
+ return a.reshape(shape)
diff --git a/tools/dsv41/tok.py b/tools/dsv41/tok.py
new file mode 100644
index 000000000..b26e133bc
--- /dev/null
+++ b/tools/dsv41/tok.py
@@ -0,0 +1,36 @@
+"""tools/ds4/tok.py - token ids for maya-ds4's --ids, and text back from its output (DeepSeek-V4.1's tokenizer).
+
+ python tools/ds4/tok.py --model ~/models/DSV4 --chat "Why is the sky blue?" [--think] -> 0,128803,...
+ python tools/ds4/tok.py --model ~/models/DSV4 --decode 18033,1 -> text
+"""
+from __future__ import annotations
+
+import argparse
+import pathlib
+import sys
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--model", default=str(pathlib.Path("~/models/DSV4").expanduser()))
+ ap.add_argument("--chat")
+ ap.add_argument("--think", action="store_true", help="thinking mode (the prompt ends with <think>)")
+ ap.add_argument("--text")
+ ap.add_argument("--decode")
+ a = ap.parse_args()
+ from tokenizers import Tokenizer
+ tok = Tokenizer.from_file(str(pathlib.Path(a.model).expanduser() / "tokenizer.json"))
+ if a.decode:
+ ids = [int(x) for x in a.decode.replace(" ", "").strip(",").split(",") if x]
+ print(tok.decode(ids, skip_special_tokens=False))
+ return 0
+ if a.chat is not None:
+ s = "<|begin▁of▁sentence|><|User|>" + a.chat + "<|Assistant|>" + ("<think>" if a.think else "</think>")
+ else:
+ s = a.text or ""
+ print(",".join(str(i) for i in tok.encode(s, add_special_tokens=False).ids))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/dsv41/vision_pack.py b/tools/dsv41/vision_pack.py
new file mode 100644
index 000000000..6144b3501
--- /dev/null
+++ b/tools/dsv41/vision_pack.py
@@ -0,0 +1,208 @@
+"""tools/ds4/vision_pack.py - DeepSeek-V4.1's vision tower (the part of the model that reads pictures, which the GGUFs
+leave out) for a GGUF install: shard 1 of DeepSeek's own checkpoint (model-00001-of-00048, the tower's vision.* and the
+aligner's aligner.* tensors, BF16, nothing else) and config.json (their geometry) from download_gguf.json's "vision"
+entry, plus the picture span's three marker rows (image_start, image_newline, image_end: BF16 [5120]), which sit beside
+the embedding in shard 2 - only their 30 KB are read from it, with an HTTP range request, never the 1.3 GB file.
+
+ python tools/ds4/vision_pack.py --dir <folder with shard 1 and config.json> -> vision.bin + vision.json there
+
+vision.bin holds every tensor exactly as the checkpoint has it (copied, 256-byte aligned) and then the three marker
+rows; it is read back and compared with the shard bit for bit before it is kept. vision.json lists every tensor (name,
+dtype, offset, bytes, shape) as tensors.txt does, and the geometry the engine reads (vision.<key> from config.json's
+vision_config, image_token_id); pack_gguf.py adds both to the pack.
+
+The marker rows: the shard's header (a range request), then the rows (one range request), each row's sha256 checked
+against the manifest's. A server that answers a range request with the whole file (status 200, not 206) is cut off at
+once, the request is tried once more, then this stops with the reason - nothing large is ever downloaded here.
+--marks-url takes another URL, or a local file (a mirror, the tests).
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import pathlib
+import struct
+import sys
+import time
+import urllib.request
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+MANIFEST = HERE / "download_gguf.json"
+HF = "https://huggingface.co/{repo}/resolve/{revision}/{path}"
+ALIGN = 256
+MARKS = ("image_start", "image_newline", "image_end") # (the order in vision.bin)
+TRIES = 2 # a range request and one retry
+
+
+class RangeError(Exception):
+ pass
+
+
+def manifest_entry(path: pathlib.Path = MANIFEST) -> dict:
+ v = json.loads(path.read_text(encoding="utf-8")).get("vision") or {}
+ if not v.get("files") or not v.get("marks"):
+ raise SystemExit(f"{path}: no \"vision\" entry")
+ return v
+
+
+def read_range(url: str, start: int, n: int, total: int | None = None, timeout: int = 60) -> bytes:
+ """Bytes start .. start + n - 1 of a file: a URL with a range request (exactly those bytes, status 206 - anything
+ else is cut off unread: RangeError), or a local file / file:// URL read at the offset."""
+ if not url.startswith(("http://", "https://")):
+ p = pathlib.Path(url[7:] if url.startswith("file://") else url)
+ with p.open("rb") as f:
+ f.seek(start)
+ b = f.read(n)
+ if len(b) != n:
+ raise RangeError(f"{p}: {len(b)} of {n} bytes at {start}")
+ return b
+ end = start + n - 1
+ req = urllib.request.Request(url, headers={"User-Agent": "maya-setup", "Range": f"bytes={start}-{end}",
+ "Accept-Encoding": "identity"})
+ try:
+ with urllib.request.urlopen(req, timeout=timeout) as r:
+ cr = r.headers.get("Content-Range", "")
+ want = f"bytes {start}-{end}/"
+ if r.status != 206 or not cr.startswith(want) or (total and cr != f"{want}{total}"):
+ # (the whole file, or other bytes: the connection is closed here without reading the body)
+ raise RangeError(f"the server answered status {r.status} ({cr or 'no Content-Range'}) to a request for "
+ f"bytes {start}-{end}, not just those bytes")
+ b = r.read(n + 1)
+ except OSError as e:
+ raise RangeError(str(e)) from e
+ if len(b) != n:
+ raise RangeError(f"{len(b)} bytes arrived for a range of {n}")
+ return b
+
+
+def fetch_marks(url: str, marks: dict, tries: int = TRIES) -> dict:
+ """The marker rows {name: bytes} from shard 2 at `url`: its header (where they are), then the rows, every row's
+ sha256 checked. Each step is tried `tries` times; then SystemExit with the reason."""
+ rows = marks["rows"]
+ total = marks.get("bytes")
+ why = ""
+ for attempt in range(tries):
+ try:
+ n = struct.unpack("<Q", read_range(url, 0, 8, total))[0]
+ if not 0 < n < 1 << 24:
+ raise RangeError(f"a safetensors header of {n} bytes")
+ head = json.loads(read_range(url, 8, n, total))
+ base = 8 + n
+ at = {}
+ for name, (off, nb, _sha) in rows.items():
+ e = head.get(name) or {}
+ a, b = e.get("data_offsets") or (None, None)
+ if e.get("dtype") != "BF16" or a is None or base + a != off or b - a != nb:
+ raise RangeError(f"{name} is not where the manifest says ({e or 'not in the header'})")
+ at[name] = (off, nb)
+ lo = min(o for o, _ in at.values())
+ hi = max(o + nb for o, nb in at.values())
+ if hi - lo > 1 << 20:
+ raise RangeError(f"the marker rows span {hi - lo} bytes: not read in one piece")
+ blob = read_range(url, lo, hi - lo, total)
+ out = {}
+ for name, (off, nb) in at.items():
+ out[name] = blob[off - lo:off - lo + nb]
+ if hashlib.sha256(out[name]).hexdigest() != rows[name][2]:
+ raise RangeError(f"{name}: the sha256 does not match the manifest's")
+ return out
+ except (RangeError, ValueError) as e:
+ why = str(e)
+ if attempt + 1 < tries:
+ print(f"vision: the marker rows: {why}; trying once more ...", file=sys.stderr)
+ time.sleep(2)
+ raise SystemExit(f"vision: the picture markers (30 KB of {marks['file']}) could not be read: {why}. Nothing large "
+ "was downloaded. Check the connection and run the setup again (or --no-vision: text only).")
+
+
+def shard(path: pathlib.Path):
+ raw = np.memmap(path, np.uint8, mode="r")
+ n = struct.unpack("<Q", bytes(raw[:8]))[0]
+ head = json.loads(bytes(raw[8:8 + n]))
+ head.pop("__metadata__", None)
+ return raw, head, 8 + n
+
+
+def geometry(cfg: dict) -> dict:
+ """pack.py's vision keys: vision.<key> for config.json's numeric vision_config entries, and image_token_id."""
+ out = {}
+ for k, v in (cfg.get("vision_config") or {}).items():
+ if isinstance(v, (int, float)) and not isinstance(v, bool):
+ out["vision.%s" % k] = v
+ if isinstance(cfg.get("image_token_id"), int):
+ out["image_token_id"] = cfg["image_token_id"]
+ if not out.get("vision.num_hidden_layers") or "image_token_id" not in out:
+ raise SystemExit("config.json: no vision_config / image_token_id")
+ return out
+
+
+def pack(d: pathlib.Path, man: dict, marks_url: str | None = None) -> dict:
+ names = [n for n in man["files"] if n.endswith(".safetensors")]
+ if len(names) != 1:
+ raise SystemExit("the manifest's vision entry: one safetensors shard expected")
+ src = d / names[0]
+ if not src.is_file() or src.stat().st_size != man["files"][names[0]][0]:
+ raise SystemExit(f"{src}: missing or not whole")
+ geo = geometry(json.loads((d / "config.json").read_text(encoding="utf-8")))
+ mk = man["marks"]
+ url = marks_url or HF.format(repo=man["repo"], revision=man["revision"], path=mk["file"])
+ rows = fetch_marks(url, mk)
+ raw, head, base = shard(src)
+ order = sorted(head, key=lambda t: head[t]["data_offsets"][0])
+ for t in order:
+ if not t.startswith(("vision.", "aligner.")):
+ raise SystemExit(f"{src.name}: {t} is not the vision tower's")
+ entries, off = [], 0
+ part = d / "vision.bin.part"
+ with part.open("wb") as f:
+ def put(name, dtype, shape, b):
+ nonlocal off
+ f.write(b)
+ entries.append([name, dtype, off, len(b), list(shape)])
+ off += len(b)
+ pad = (-off) % ALIGN
+ f.write(b"\0" * pad)
+ off += pad
+ for t in order:
+ e = head[t]
+ a, b = e["data_offsets"]
+ put(t, e["dtype"], e["shape"], raw[base + a:base + b].tobytes())
+ for name in MARKS:
+ put(name, "BF16", [len(rows[name]) // 2], rows[name])
+ # read back: every tensor as the checkpoint has it, bit for bit
+ back = np.memmap(part, np.uint8, mode="r")
+ for name, _dt, o, nb, _shape in entries:
+ if name in rows:
+ ok = back[o:o + nb].tobytes() == rows[name]
+ else:
+ a, b = head[name]["data_offsets"]
+ ok = np.array_equal(back[o:o + nb], raw[base + a:base + b])
+ if not ok:
+ raise SystemExit(f"vision.bin: {name} differs from the checkpoint after the copy")
+ del back, raw
+ part.replace(d / "vision.bin")
+ info = {"from": {"repo": man["repo"], "revision": man["revision"], "files": sorted(man["files"]),
+ "marks": mk["file"]}, "config": geo, "tensors": entries}
+ (d / "vision.json").write_text(json.dumps(info))
+ return info
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--dir", required=True, help="the folder with shard 1 and config.json")
+ ap.add_argument("--manifest", default=str(MANIFEST))
+ ap.add_argument("--marks-url", help="shard 2 elsewhere (a URL or a local file; default: Hugging Face, pinned)")
+ a = ap.parse_args()
+ d = pathlib.Path(a.dir)
+ info = pack(d, manifest_entry(pathlib.Path(a.manifest)), a.marks_url)
+ nb = sum(e[3] for e in info["tensors"])
+ print(f"vision.bin: {len(info['tensors'])} tensors ({nb / 1e9:.2f} GB), every one equal to the checkpoint's; "
+ f"{info['config']['vision.num_hidden_layers']} blocks, image token {info['config']['image_token_id']}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())