Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: import Project Maya's DeepSeek-V4.1 engine unmodified (v1.1.17, 8d9a9b5d)
mw00bbd0a2c67300
diff --git a/include/strata/dsv41/exl3.hpp b/include/strata/dsv41/exl3.hppnew file mode 100644index 000000000..12f862aaf--- /dev/null+++ b/include/strata/dsv41/exl3.hpp@@ -0,0 +1,185 @@+// include/strata/kernels/exl3.hpp - EXL3 (exllamav3) quantized weights on the device: the decode GEMV, the routed+// experts, and the full reconstruction for the prompt path.+//+// The format (exllamav3, MIT, turboderp). A weight with k inputs and n outputs (y = x W) is stored as+// trellis int16 [k/16][n/16][16 K]: each 16x16 tile is a tail-biting trellis of 256 K-bit steps read MSB-first+// from its 8 K uint32 words (little-endian pairs of the int16s). Step p's state is the 16-bit window that+// ends at bit (p + 1) K, modulo the tile's 256 K bits; the codebook turns the state into a value:+// mul1 (cb 2): fp16(1024 + bytesum(state * 0x83DCD12D)) * fp16(0x1eee) + fp16(0xc931), one FP16 FMA+// mcg (cb 1): x = state * 0xCBAC1FED, 3inst (cb 0): x = state * 89226354 + 64248484; then+// x = (x & 0x8fff8fff) ^ 0x3b603b60 and the value is the FP16 sum of x's two halves.+// Step p sits at tile row (p/8 % 4) * 2 + (p & 1) + 8 (p/2 & 1), column p/32 + 8 (p/4 & 1) (the tensor-+// core fragment order).+// suh, svh FP16 [k], [n]: the input and output channel scales (signs and magnitudes).+// With H the 128-point Sylvester Hadamard over sqrt(128), applied to each 128 consecutive values:+// y = H(H(x * suh) W') * svh, W' the decoded k x n matrix.+//+// Activations enter as F32 or as the engine's q8_1 blocks; the kernels rotate them on the fly (each block rotates+// the slice it reads), so callers keep their existing buffers.+#pragma once++#include <cstddef>+#include <cstdint>+#include <cuda_runtime.h>++namespace strata::kernels::exl3 {++/// The weight type the engine's MvJob / WSlot carry for EXL3: their weight pointer is a (host) const Mat*.+constexpr int kTypeEXL3 = 200;++/// One EXL3 matrix, or a column range of one (a view: trellis/svh advanced to its first column, ld the row stride).+struct Mat {+ const uint32_t* trellis = nullptr; // device; 4-byte aligned+ const uint16_t* suh = nullptr; // device FP16 bits [k]+ const uint16_t* svh = nullptr; // device FP16 bits [n]+ int k = 0, n = 0;+ int K = 0; // bits per weight, 1..8+ int cb = 2; // codebook: 0 3inst, 1 mcg, 2 mul1+ int ld = 0; // tiles per 16-row slice in memory (the full matrix's n / 16); 0: n / 16+ // a GGUF's plain matrix instead (DeepSeek-V4.1's maya-ds4: fmt 1 BF16): raw = [n][ld] (row r = output r, ld+ // values a row; 0: k), the EXL3 fields unused - the engine sends it to its own kernels, never to these+ int fmt = 0;+ const void* raw = nullptr;+};++/// Trellis bytes of a k x n matrix at K bits.+inline size_t trellis_bytes(int64_t k, int64_t n, int K) { return (size_t) (k / 16) * (size_t) (n / 16) * 32u * K; }+/// The columns [c0, c0 + nc) of m (c0 and nc multiples of 128: the output rotation's blocks stay whole).+Mat view(const Mat& m, int c0, int nc);+/// Whether the kernels take this matrix (k, n multiples of 128, K 1..8, cb 0..2).+bool supported(const Mat& m);++/// y[r] = alpha * (x W)[r] + (bias ? bias[r] : 0). x: xf (F32) when given, else xq (q8_1 blocks of k values).+/// ncol > 1: as many activations (a verify window's tokens), activation c at xf + c xf_cs / xq + c xq_cs bytes, its+/// y at y + c y_cs - the weights read and decoded once for up to 8 of them, and each y bit for bit its own job's.+struct MvJob {+ const Mat* m = nullptr;+ const float* xf = nullptr;+ const void* xq = nullptr;+ float* y = nullptr;+ const float* bias = nullptr;+ float alpha = 1.0f;+ int ncol = 1;+ size_t xf_cs = 0, xq_cs = 0, y_cs = 0;+};+constexpr int kMaxMvJobs = 8;+constexpr int kMaxMvCols = 16; // a job's activations at most+bool mv(const MvJob* jobs, int n, cudaStream_t s);+/// The calling thread's launches of mv(), moe_gate_up() and moe_down() read `flag` (a device int) first and do no work+/// while it is non-zero (glmf::set_skip's, which sets it: a pipelined window's tokens past a wrong draft); nullptr (the+/// default): no check. The flag changes only between launches (skip_gate, on the same stream). (v1.1.8's kernels+/// before sm_80 do not read it.)+void set_skip(const int* flag);+/// How many pieces mv cuts m's inputs into on the current device (0: not covered) - one piece, and every column's+/// sum is the same whatever columns the job holds (a view's, or the whole matrix's).+int mv_splits(const Mat& m);+/// How many activations a lone job of m can take with its weights read once on the current device (the input split's+/// partials of them in the scratch; 0: not covered) - a caller with more cuts them in runs of this many.+int mv_cols(const Mat& m);++/// Allocates the per-stream scratch of mv / moe / win_experts on the current device (call once per stream before a+/// graph capture; the first use does it otherwise).+bool prepare(cudaStream_t s);++/// One routed expert's blob: three parts (gate, up, down) at byte offsets from the blob base, each holding the FP16+/// suh, the FP16 svh and the trellis of that matrix.+struct ExpertLayout {+ size_t suh[3] = {0, 0, 0}, svh[3] = {0, 0, 0}, trellis[3] = {0, 0, 0};+ int K[3] = {0, 0, 0};+ int cb = 2;+ int fmt = 0; // 2: a GGUF's MXFP4 expert (maya-ds4's own kernels): trellis[r] = matrix r's offset, [out][in / 32]+};+/// The routed experts of one token (the engine's device plan: plan_ptr[i] = the blob in VRAM, 0 = not on the device;+/// plan_w[i] its weight). gate/up over x (n_embd F32) -> clamped swiglu -> h (F32 [k][n_ff], scratch the caller+/// owns), then down over h, the weighted sum in plan order, + cpu_part when *cpu_flag, + sh_out when given -> out.+struct MoeArgs {+ const unsigned long long* plan_ptr = nullptr;+ const float* plan_w = nullptr;+ const float* cpu_part = nullptr;+ const int* cpu_flag = nullptr;+ ExpertLayout lay;+ int k = 8, n_embd = 4096, n_ff = 2048;+ float limit = 0.0f; // 0: no clamp+};+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);++/// A verify window's routed experts, each distinct expert read once for all the tokens it serves (after exllamav3's+/// fused MoE, which runs the slots that picked one expert as the rows of one tile): T tokens' plans [T][8] grouped by+/// expert (glmf::moe_win_group: group g's blob at grp_ptr[g], its entries e = t * 8 + i at grp_ent[grp_start[g] ..+/// grp_start[g + 1])), token t's F32 input at x + t x_stride, its output at out + t out_stride (plus the CPU lane's+/// part when cpu_flag[t], plus sh_out's row t) - every token's output is moe_gate_up + moe_down's for it alone, bit for+/// bit. h [T * 8][n_ff] and part [T * 8][n_embd] are the caller's scratch.+struct WinArgs {+ const unsigned long long* grp_ptr = nullptr;+ const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;+ const unsigned long long* plan_ptr = nullptr; // [T][8]+ const float* plan_w = nullptr; // [T][8]+ const float* cpu_part = nullptr; // [T][n_embd] (rows cpu_stride apart when given)+ size_t cpu_stride = 0;+ const int* cpu_flag = nullptr; // [T]+ const float* x = nullptr;+ size_t x_stride = 0;+ float* h = nullptr;+ float* part = nullptr;+ const float* sh_out = nullptr; // [T][n_embd] or null+ float* out = nullptr;+ size_t out_stride = 0;+ ExpertLayout lay;+ int T = 0, k = 8, n_embd = 4096, n_ff = 2048;+ float limit = 0.0f;+ // (given) the combine first waits on the device until *wait_ptr == wait_val (the host's answer to the CPU lane's+ // part arrives while the device computes the experts); cpu_flag may be null (no CPU part)+ const unsigned* wait_ptr = nullptr;+ unsigned wait_val = 0;+};+bool win_experts(const WinArgs& a, cudaStream_t s);+/// Whether the current device runs the several-activation kernels (win_experts; mv's ncol > 1): every device does+/// (they take ~10 KB of shared memory a block).+bool several_ok();++/// The prompt path's tensor-core GEMMs (sm_80 and later; mma_supported() says whether this device runs them).+bool mma_supported();+/// A set of n_exp routed experts in VRAM, expert e's blob at wbase + e * stride (the layout in lay), owning the sorted+/// rows [bounds[e], bounds[e + 1]) (device, n_exp + 1 entries); a sorted row's LOCAL row is bounds[e] - row0 + r.+struct SetArgs {+ const uint8_t* wbase = nullptr;+ size_t stride = 0;+ ExpertLayout lay;+ int n_exp = 0;+ const int* bounds = nullptr;+ int row0 = 0;+ int max_rows = 0; // the most rows one expert of the set has+ int n_embd = 4096, n_ff = 2048;+ float limit = 0.0f;+};+/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).+void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);+/// y[local row] = down(h[local row]) (F32 rows, ldy floats apart).+void set_down(const SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);+/// Y[r] = beta Y[r] + (X[r] W) for r < rows: one matrix over FP16 rows (ldx), F32 output rows (ldy).+void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s);++/// The full weight in the engine's row layout: dst[r][c] = W[c][r] (n rows of k values), BF16 or F16 bits.+void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);+/// The decoded W' alone in that layout: dst[c][r] = W'[r][c], F16 bits (exactly the decoded values) - for a product+/// that rotates its activations and outputs instead of the weight (x W = H(H(x * suh) W') * svh): rows_rot_f16 before+/// the GEMM, rows_rot_out after it. No Hadamard over the k x n weight: a plain decode, at the memory's speed.+void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s);+/// out[r][c] = FP16(H(x_r * suh)[c]) for r < nrows, c < ncols (ncols % 128 == 0), x_r = x[idx ? idx[r] : r] (F32, ldx+/// floats a row): an EXL3 matrix's rotated input rows, as the decode kernels rotate theirs.+void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,+ cudaStream_t s);+/// y[r][c] = H(y_r)[c] * svh[c] in place for r < nrows, c < ncols (each 128 values one block; ldy floats a row).+void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s);+/// The decoded W' alone, row-major [k][n] F16 bits (tests).+void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s);++/// out[r][c] = FP16(x[idx ? idx[r] : r][c]) for r < nrows, c < ncols (x: ldx floats a row): the prompt path's rows of+/// one expert, gathered for its GEMMs.+void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s);++/// Launch failures of this family since the process started (each is also printed).+int launch_errors();++} // namespace strata::kernels::exl3diff --git a/include/strata/dsv41/exl3_cpu.hpp b/include/strata/dsv41/exl3_cpu.hppnew file mode 100644index 000000000..f72b674c4--- /dev/null+++ b/include/strata/dsv41/exl3_cpu.hpp@@ -0,0 +1,67 @@+// include/strata/kernels/cpu/exl3_cpu.hpp - EXL3 routed experts on the CPU (the fast path's CPU lane).+//+// The mul1 codebook is affine in the byte sum of state * 0x83DCD12D: w = fp16(1024 + s) * k_inv + k_bias (one FP16+// FMA on the device). The CPU keeps the affine form without the FP16 rounding, so a 128-output band is+// y' = k_inv * sum_k xh_k s_kc + (1024 k_inv + k_bias) * sum_k xh_k, xh = H(x * suh)+// and y = H(y') * svh. The byte sums come from AVX-512 (permutes and funnel shifts pull 16 columns' states at once,+// vpmaddubsw / vpmaddwd add the bytes up, an F32 FMA takes the activation), or from AVX2 + FMA the same way 8 columns+// at a time - the same FMAs in the same order, so the two give the same bits; a CPU with neither runs a scalar loop.+// The format: include/strata/kernels/exl3.hpp.+#pragma once++#include <cstddef>+#include <cstdint>++namespace strata::kernels::cpu {++/// One EXL3 matrix (mul1 codebook): k inputs, n outputs, K bits (1..8), tiles in exllamav3's natural layout.+struct Exl3Part {+ const uint32_t* trellis = nullptr;+ const uint16_t* suh = nullptr; // FP16 bits [k]+ const uint16_t* svh = nullptr; // FP16 bits [n]+ int k = 0, n = 0, K = 0;+};+/// Whether a vector kernel runs here (AVX-512F/BW or AVX2 + FMA); the scalar one otherwise.+bool exl3_cpu_vector();+/// The kernel the functions below run here: 2 AVX-512 (F, BW), 1 AVX2 + FMA (+ F16C), 0 scalar. STRATA_FORCE_AVX2=1+/// keeps an AVX-512 CPU on AVX2 (to compare the two; they give the same bits).+int exl3_cpu_tier();+/// xh = H(x * suh) (k values, k % 128 == 0); returns the sum of xh.+float exl3_cpu_prepare(const Exl3Part& m, const float* x, float* xh);+/// One 128-value block of that: xh[i] = H(x * suh)[i] for i < 128 (suh: the block's 128 scales); returns its sum.+float exl3_cpu_prepare_block(const uint16_t* suh, const float* x, float* xh);+/// The 128 outputs from column c0 (a multiple of 128) of y = H(x W') * svh, from exl3_cpu_prepare's xh and sum.+void exl3_cpu_band(const Exl3Part& m, const float* xh, float xsum, int c0, float* y);+/// The same in parts: the byte-sum products of the 16-row slices [ks0, ks1) (xh from slice 0) ...+void exl3_cpu_band_acc(const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc);+/// exl3_cpu_band_acc for n activations of the same matrix at once (xh[i] -> acc[i], each 128 floats): each tile row is+/// decoded once for up to kExl3CpuMaxN of them, and acc[i] is exl3_cpu_band_acc's for xh[i] alone, bit for bit.+constexpr int kExl3CpuMaxN = 8;+void exl3_cpu_band_acc_n(const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,+ float* const* acc);+/// The byte-sum products of the slices [ks0, ks1) for EVERY column (acc: m.n floats, overwritten) - a split-K piece,+/// whose tile rows lie in memory order where a band's are 1 KB runs a whole tile row apart (from DRAM, ~1.5x the+/// bands' speed). Each column's sums are exl3_cpu_band_acc's over the same slices, bit for bit.+void exl3_cpu_rows_acc(const Exl3Part& m, const float* xh, int ks0, int ks1, float* acc);+/// ... for n activations at once (xh[i] -> acc[i]), each tile row decoded once; acc[i] is the call for xh[i] alone.+void exl3_cpu_rows_acc_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, float* const* acc);+/// ... over the columns [c0, c1) alone (multiples of 128; acc[i] indexed by column, only those overwritten): a piece cut+/// by columns too, for more jobs than pieces - each column's sums are the whole piece's, bit for bit.+void exl3_cpu_rows_acc_cols_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int c0, int c1,+ float* const* acc);+/// The byte-sum products of nb bands from c0 (128 nb columns; acc[i]: 128 nb floats) over every input for any n+/// activations, as a GEMM: each 256-input chunk of a band decoded once into F32 in `wc` (exl3_cpu_gemm_scratch()+/// floats, 64-byte aligned, the caller's), then multiplied for all n, the chunk of the activations kept in L2 across+/// the nb bands - for many tokens an expert (the prompt path), where the band kernels decode a tile row again every+/// kExl3CpuMaxN. The same sums as exl3_cpu_band_acc, in its order.+size_t exl3_cpu_gemm_scratch();+void exl3_cpu_band_gemm(const Exl3Part& m, const float* const* xh, int n, int c0, int nb, float* const* acc,+ float* wc);+/// exl3_cpu_band_acc_n on a given tier (0, 1, 2 as exl3_cpu_tier) - tests and benchmarks; false (nothing computed)+/// where this CPU cannot run it.+bool exl3_cpu_band_acc_on(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,+ float* const* acc);+/// ... and, with the parts' acc summed over every slice, the outputs (the codebook's affine form, H, svh).+void exl3_cpu_band_out(const Exl3Part& m, const float* acc, float xsum, int c0, float* y);++} // namespace strata::kernels::cpudiff --git a/include/strata/dsv41/sampler.hpp b/include/strata/dsv41/sampler.hppnew file mode 100644index 000000000..a7779c92d--- /dev/null+++ b/include/strata/dsv41/sampler.hpp@@ -0,0 +1,92 @@+// include/strata/kernels/sampler.hpp - the sampler chain, host-callable (P2.S2).+//+// penalties -> top_k -> top_p -> min_p -> temperature -> pick+//+// llama.cpp's default chain (issue #53): ONE penalties stage, first, and temperature AFTER the truncation filters;+// top_p cuts before min_p. `src/kernels/cuda/sampler.cu` has the details and `sampler_parity` pins them.+#pragma once++#include <algorithm>+#include <cstddef>+#include <cstdint>+#include <vector>++namespace strata::kernels {++struct SamplerParams {+ int top_k = 20; // sampled path: 1..64 as given; 0 (off) or more than 64 keep the widest list, 64+ float top_p = 0.95f; // 1.0 disables the filter+ float min_p = 0.0f; // 0 disables; keeps tokens with p >= min_p * p_max (a prefix of the top_k list)+ float temperature = 1.0f; // <= 0 means greedy+ int min_keep = 1; // top_p keeps at least this many+ int penalty_last_n = 0; // 0 disables; the window over `history` to count occurrences in+ float penalty_repeat = 1.0f;+ float penalty_freq = 0.0f;+ float penalty_present = 0.0f;+ uint64_t seed = 0; // drives Philox, which is counter-based on (seed, token index)+ uint64_t counter = 0; // absolute draw index of row 0; advance across decode calls+ bool greedy = false;+ // a request's logit_bias: a DEVICE array of n_vocab floats on the sampling device, added to every row's logits+ // before the penalties and the filters (-inf bans a token); nullptr = none+ const float* logit_bias = nullptr;+ // the GLM engine's penalty history, on the HOST (never read by a kernel): every token the request's state has been+ // fed so far, oldest first - its prompt, then each token it wrote. Glm5Model::pick_tokens hands each pick the last+ // penalty_last_n of them (a verify window's row t: these, then the window's drafts 1..t). nullptr = none+ const std::vector<int32_t>* host_history = nullptr;+};++// logits (n_tokens, n_vocab) -> one sampled token id per row in `out`.+//+// **`logits` AND `out` ARE DEVICE POINTERS.** This is a CUDA kernel launch, not a host function, and nothing+// in the parameter names or the types says so - `const float* logits` reads exactly like a host buffer.+// Measured: passing host memory for `logits` faults inside the kernel with an ILLEGAL MEMORY ACCESS that is+// reported by whatever synchronising call happens NEXT, which will be somewhere else entirely and will name a+// buffer that has nothing to do with it. A sticky async fault does not know where it came from.+//+// `history` is (n_tokens, history_len) int32 - ONE ROW PER TOKEN ROW, at a stride of `history_len` - the most+// recent tokens that row's pick follows, with any unused slots set to -1; only the last `p.penalty_last_n` of+// each row are counted, and ids outside [0, n_vocab) are ignored. Pass nullptr and 0 when no penalties apply.+// A verify window's rows need DIFFERENT histories: row t follows the window's drafts 1..t (`penalty_rows`).+void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,+ const SamplerParams& p, int* out, void* stream);++// SPECULATIVE SAMPLING (Leviathan et al. 2023, Chen et al. 2023) for the pipelined NextN decode - opt-in+// (STRATA_GLM_SPEC_SAMPLE=1). The draft is DRAWN from the draft block's own distribution q (this chain on its logits,+// a Philox stream of its own) and its kept list is stored; the model's token is then the draft with probability+// min(1, p(d) / q(d)), else a draw from the residual max(0, p - q) renormalised - so it is distributed exactly as the+// plain draw from p, and a draft stands whenever the token is the draft (sum over x of min(p(x), q(x)) of the time+// instead of p(argmax q)). Same (seed, counter): the same tokens - but not the plain draw's tokens.+struct SpecList {+ int n = 0; // the kept list's length (1..64)+ int ids[64]; // descending, as the selection produced them+ double p[64]; // q over the kept list (sums to 1)+};++// draft rows: logits (n_tokens, n_vocab) -> out[t] drawn through this chain, q[t] its kept list. DEVICE pointers,+// like sample_tokens; a sampled request without penalties only (greedy keeps the argmax draft).+void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,+ void* stream);+// verify rows: the model's logits (n_tokens, n_vocab), the draft draft[t] drawn from q[t] (same p.seed / p.counter as+// its sample_draft) -> out[t], distributed as sample_tokens' draw. A draft q[t] does not hold gets the plain draw.+void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,+ const int* draft, int* out, void* stream);++// The penalty-history rows of a verify window, on the host: row t of `out` (T rows of `h` slots) is the last `h`+// tokens of `tail[0..n_tail)` followed by `window[0..t]`, most recent LAST, -1 in the unused front slots.+// `tail` is what the state consumed before the window, `window[0]` the fed-back token and `window[1..]` the+// drafts: row t is exactly the history plain decode counts when it picks the token after `window[t]`, so drafting+// cannot change which tokens are penalised. Row 0 alone is the single row the engine staged before 0.1.19.+inline void penalty_rows(const int32_t* tail, int64_t n_tail, const int32_t* window, int T, int h, int32_t* out) {+ for (int t = 0; t < T; ++t) {+ int32_t* row = out + (size_t) t * (size_t) h;+ const int64_t avail = n_tail + t + 1; // tail + window[0..t]+ const int take = (int) std::min<int64_t>(h, avail);+ std::fill(row, row + (h - take), -1);+ for (int j = 0; j < take; ++j) {+ const int64_t i = avail - take + j; // index into (tail, window)+ row[h - take + j] = i < n_tail ? tail[i] : window[i - n_tail];+ }+ }+}++} // namespace strata::kernelsdiff --git a/src/dsv41/ds4_engram_io.hpp b/src/dsv41/ds4_engram_io.hppnew file mode 100644index 000000000..1e44c41fb--- /dev/null+++ b/src/dsv41/ds4_engram_io.hpp@@ -0,0 +1,233 @@+// src/ds4/ds4_engram_io.hpp - DeepSeek's Engram (n-gram memory) rows read off the decode's critical path.+//+// A token's Engram rows depend only on the token ids, so they are asked for as soon as the window's tokens are known+// and collected only right before the layer that adds them (layer 1's hide behind the embedding and layer 0, layer+// 14's behind layers 0-13). The reads run on the reader's own threads, each with its own handles to the model's files+// (on Windows the ReadFile calls on one synchronous handle run one at a time), FIFO: the first Engram layer's rows+// first. A bounded row cache (clock eviction) keeps rows already fetched: a row that recurs (the same n-gram again,+// in the prompt or later) is a copy, not a read. The rows are the file's bytes either way, so the output is the same.+#pragma once++#include "ds4_os.hpp"++#include <algorithm>+#include <atomic>+#include <chrono>+#include <condition_variable>+#include <cstdint>+#include <cstring>+#include <deque>+#include <memory>+#include <mutex>+#include <string>+#include <thread>+#include <unordered_map>+#include <utility>+#include <vector>++namespace strata::ds4 {++inline int64_t now_ns() {+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())+ .count();+}++/// Rows already fetched: key -> `bytes` of payload, at most `cap` rows (grown on demand in blocks), clock eviction.+/// Not thread-safe (the caller locks).+class EgRowCache {+public:+ void init(size_t cap, size_t bytes) {+ cap_ = cap;+ bytes_ = bytes;+ map_.reserve(std::min<size_t>(cap, 1u << 20));+ }+ size_t capacity() const { return cap_; }+ size_t size() const { return keys_.size(); }+ /// The row's payload, or nullptr (a hit marks it recently used).+ const uint8_t* find(uint64_t key) {+ const auto it = map_.find(key);+ if (it == map_.end()) return nullptr;+ ref_[it->second] = 1;+ return at(it->second);+ }+ /// Room for key's payload (the caller fills it): a new slot while below capacity, else the clock's victim.+ uint8_t* put(uint64_t key) {+ if (cap_ == 0 || bytes_ == 0) return nullptr;+ const auto it = map_.find(key);+ if (it != map_.end()) return at(it->second);+ uint32_t s;+ if (keys_.size() < cap_) {+ s = (uint32_t) keys_.size();+ if (s % kBlock == 0) blocks_.emplace_back(new uint8_t[(size_t) kBlock * bytes_]);+ keys_.push_back(key);+ ref_.push_back(0);+ } else {+ while (ref_[hand_]) { // (a second chance for rows used since the hand last passed)+ ref_[hand_] = 0;+ hand_ = (hand_ + 1) % (uint32_t) cap_;+ }+ s = hand_;+ hand_ = (hand_ + 1) % (uint32_t) cap_;+ map_.erase(keys_[s]);+ keys_[s] = key;+ ref_[s] = 0;+ }+ map_.emplace(key, s);+ return at(s);+ }++private:+ static constexpr uint32_t kBlock = 4096;+ uint8_t* at(uint32_t s) { return blocks_[s / kBlock].get() + (size_t) (s % kBlock) * bytes_; }+ size_t cap_ = 0, bytes_ = 0;+ uint32_t hand_ = 0;+ std::unordered_map<uint64_t, uint32_t> map_;+ std::vector<uint64_t> keys_;+ std::vector<uint8_t> ref_;+ std::vector<std::unique_ptr<uint8_t[]>> blocks_;+};++/// One row to fetch: its codes to dst, its scales to scales. mx8: n bytes at off of a file, [scale, 32 codes] blocks+/// (a GGUF's MXFP8 row); else n code bytes at off and n2 scale bytes at off2 of file2 (two reads; n2 0: none - a+/// GGUF's Q6_K row, its scales inside its n bytes).+struct EgRead {+ int file = -1;+ uint64_t off = 0;+ uint32_t n = 0;+ uint8_t* dst = nullptr;+ uint8_t* scales = nullptr;+ bool mx8 = false;+ int file2 = -1;+ uint64_t off2 = 0;+ uint32_t n2 = 0;+ uint64_t key = 0;+};++/// A batch of rows (one Engram layer of a window): reads, the other places a row read once goes, and its state.+struct EgBatch {+ std::vector<EgRead> reads;+ std::vector<std::pair<int, std::pair<uint8_t*, uint8_t*>>> dups; // read i -> another (codes, scales)+ std::vector<uint8_t> ok; // per read: 1 read whole+ std::atomic<int> left{0}; // reads not yet done+ std::atomic<int> post{0}; // 0 pending, 1 finishing, 2 finished+ int64_t t0 = 0; // submitted (now_ns())+ std::atomic<int64_t> t1{0}; // the last read done+ bool live = false; // submitted and not yet finished+ void clear() {+ reads.clear();+ dups.clear();+ ok.clear();+ left.store(0);+ post.store(0);+ live = false;+ }+};++/// The reader's threads: each read on whichever is free, in submission order, with that thread's own file handles.+class EgReader {+public:+ ~EgReader() { stop(); }+ void start(const std::vector<std::string>& paths, int threads) {+ paths_ = paths;+ for (int i = 0; i < threads; ++i) th_.emplace_back([this] { loop(); });+ }+ void stop() {+ {+ std::lock_guard<std::mutex> lk(mu_);+ quit_ = true;+ }+ cv_.notify_all();+ for (auto& t : th_) t.join();+ th_.clear();+ }+ int threads() const { return (int) th_.size(); }+ void submit(EgBatch& b) {+ b.ok.assign(b.reads.size(), 0);+ b.post.store(0);+ b.t0 = now_ns();+ b.t1.store(b.t0);+ b.live = true;+ b.left.store((int) b.reads.size(), std::memory_order_release);+ const int n = (int) b.reads.size();+ if (n == 0) return;+ // a token's rows one job each; a prompt chunk's in runs (fewer trips through the queue's lock)+ const int step = std::max(1, std::min(32, n / (int) (th_.size() * 4)));+ int jobs = 0;+ {+ std::lock_guard<std::mutex> lk(mu_);+ for (int i = 0; i < n; i += step, ++jobs) q_.push_back({&b, i, std::min(n, i + step)});+ }+ if (jobs >= (int) th_.size()) cv_.notify_all();+ else+ for (int i = 0; i < jobs; ++i) cv_.notify_one();+ }+ /// Until every read of b is done.+ void wait(EgBatch& b) {+ if (b.left.load(std::memory_order_acquire) == 0) return;+ std::unique_lock<std::mutex> lk(done_mu_);+ done_cv_.wait(lk, [&] { return b.left.load(std::memory_order_acquire) == 0; });+ }++private:+ void loop() {+ std::vector<os::fd_t> fds(paths_.size(), os::kBadFd); // this thread's own handles, opened on first use+ std::vector<uint8_t> buf;+ for (;;) {+ Job job;+ {+ std::unique_lock<std::mutex> lk(mu_);+ cv_.wait(lk, [&] { return quit_ || !q_.empty(); });+ if (quit_ && q_.empty()) break;+ job = q_.front();+ q_.pop_front();+ }+ EgBatch& b = *job.b;+ for (int i = job.i0; i < job.i1; ++i) b.ok[(size_t) i] = read(fds, b.reads[(size_t) i], buf) ? 1 : 0;+ const int64_t t = now_ns();+ for (int64_t prev = b.t1.load(); t > prev && !b.t1.compare_exchange_weak(prev, t);) {+ }+ if (b.left.fetch_sub(job.i1 - job.i0, std::memory_order_acq_rel) == job.i1 - job.i0) { // (b is the waiter's again)+ std::lock_guard<std::mutex> lk(done_mu_);+ done_cv_.notify_all();+ }+ }+ for (os::fd_t f : fds) os::close_fd(f);+ }+ bool read(std::vector<os::fd_t>& fds, const EgRead& r, std::vector<uint8_t>& buf) {+ auto fd = [&](int file) {+ if (file < 0 || file >= (int) fds.size()) return os::kBadFd;+ if (fds[(size_t) file] == os::kBadFd) fds[(size_t) file] = os::open_ro(paths_[(size_t) file]);+ return fds[(size_t) file];+ };+ const os::fd_t f = fd(r.file);+ if (f == os::kBadFd) return false;+ if (!r.mx8) {+ if (!os::pread_full(f, r.dst, r.n, r.off)) return false;+ if (r.n2 == 0) return true;+ const os::fd_t f2 = fd(r.file2);+ return f2 != os::kBadFd && os::pread_full(f2, r.scales, r.n2, r.off2);+ }+ buf.resize(r.n);+ if (!os::pread_full(f, buf.data(), r.n, r.off)) return false;+ for (uint32_t k = 0; k < r.n / 33; ++k) { // [scale, 32 codes] -> codes | scales+ r.scales[k] = buf[(size_t) k * 33];+ std::memcpy(r.dst + (size_t) k * 32, buf.data() + (size_t) k * 33 + 1, 32);+ }+ return true;+ }++ std::vector<std::string> paths_;+ std::vector<std::thread> th_;+ std::mutex mu_;+ std::condition_variable cv_;+ struct Job {+ EgBatch* b = nullptr;+ int i0 = 0, i1 = 0; // reads [i0, i1)+ };+ std::deque<Job> q_;+ bool quit_ = false;+ std::mutex done_mu_;+ std::condition_variable done_cv_;+};++} // namespace strata::ds4diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cunew file mode 100644index 000000000..379d0766d--- /dev/null+++ b/src/dsv41/ds4_gguf.cu@@ -0,0 +1,1461 @@+// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).+#include "ds4_gguf.cuh"++#include "ds4_iq.cuh"+#include "ds4_numerics.cuh"++#include <cuda_fp16.h>+#if !defined(STRATA_USE_HIP)+#include <mma.h>+#endif++#include <algorithm>+#include <cstdlib>+#include <atomic>+#include <cstdio>+#include <cstring>+#include <mutex>+#include <unordered_map>++namespace strata::ds4::gguf {+namespace {++std::atomic<int> g_errors{0};++void check(const char* what) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "ds4 gguf %s: %s\n", what, cudaGetErrorString(e));+ g_errors.fetch_add(1, std::memory_order_relaxed);+ }+}+void fail(const char* what) {+ std::fprintf(stderr, "ds4 gguf %s\n", what);+ g_errors.fetch_add(1, std::memory_order_relaxed);+}++constexpr int kWarps = 8, kThreads = kWarps * 32;+constexpr int kMaxEnt = 16; // a group's entries in one pass (a window's tokens at most)++__device__ __forceinline__ float warp_sum(float v) {+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);+ return v;+}++// ---------------------------------------------------------------- BF16 matrices+struct BfJob {+ const void* w;+ size_t ld; // values a row (BF16), bytes a row (MXFP4)+ const float* x;+ size_t xcs;+ float* y;+ size_t ycs;+ const float* bias;+ float alpha;+ int k, n, ncol;+ const int8_t* xq; // (Q8_0: the activations rounded to int8, q8_quant's)+ const float* xd;+};+struct BfJobs {+ BfJob j[x3::kMaxMvJobs];+};++// grid (max rows / 8, jobs): a warp a row, each of its activations' dot in F32 - a lane 8 BF16 weights (16 bytes) and+// 8 activations a step (VEC), or one of each (rows or activations not 16-byte aligned)+template <int NC, bool VEC>+__global__ void __launch_bounds__(kThreads) bf16_mv_kernel(const BfJobs J) {+ const BfJob& b = J.j[blockIdx.y];+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (row >= b.n) return;+ const uint16_t* w = (const uint16_t*) b.w + (size_t) row * b.ld;+ float acc[NC];+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;+ if (VEC) {+ for (int i = lane * 8; i < b.k; i += 256) {+ const uint4 q = *(const uint4*) (w + i);+ const float wf[8] = {__uint_as_float(q.x << 16), __uint_as_float(q.x & 0xffff0000u),+ __uint_as_float(q.y << 16), __uint_as_float(q.y & 0xffff0000u),+ __uint_as_float(q.z << 16), __uint_as_float(q.z & 0xffff0000u),+ __uint_as_float(q.w << 16), __uint_as_float(q.w & 0xffff0000u)};+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float4* xp = (const float4*) (b.x + c * b.xcs + i);+ const float4 x0 = __ldg(xp), x1 = __ldg(xp + 1);+ float a = acc[c];+ a = fmaf(wf[0], x0.x, a);+ a = fmaf(wf[1], x0.y, a);+ a = fmaf(wf[2], x0.z, a);+ a = fmaf(wf[3], x0.w, a);+ a = fmaf(wf[4], x1.x, a);+ a = fmaf(wf[5], x1.y, a);+ a = fmaf(wf[6], x1.z, a);+ a = fmaf(wf[7], x1.w, a);+ acc[c] = a;+ }+ }+ } else {+ for (int i = lane; i < b.k; i += 32) {+ const float wf = __uint_as_float((uint32_t) w[i] << 16);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) acc[c] = fmaf(wf, b.x[c * b.xcs + i], acc[c]);+ }+ }+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float v = warp_sum(acc[c]);+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}+++__global__ void q8_rows_kernel(const uint8_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,+ bool bf16) {+ const size_t total = (size_t) n * (size_t) k;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {+ const size_t r = i / (size_t) k, c = i - r * (size_t) k;+ const uint8_t* p = w + r * ld + (c / 32) * kQ8Bytes;+ const float v = __half2float(__ushort_as_half(*(const uint16_t*) p)) * (float) (int8_t) p[2 + c % 32];+ dst[i] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));+ }+}++__global__ void bf16_rows_kernel(const uint16_t* __restrict__ w, size_t ld, int k, int n, uint16_t* __restrict__ dst,+ bool bf16) {+ const size_t total = (size_t) n * (size_t) k;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {+ const size_t r = i / (size_t) k, c = i - r * (size_t) k;+ const uint16_t b = w[r * ld + c];+ dst[i] = bf16 ? b : __half_as_ushort(__float2half_rn(__uint_as_float((uint32_t) b << 16)));+ }+}++// ---------------------------------------------------------------- MXFP4 experts+__device__ __forceinline__ float e8m0_half(uint32_t e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF+ return __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);+}+__device__ __forceinline__ void lut_init(float* lut) {+ if (threadIdx.x < 16) {+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};+ lut[threadIdx.x] = threadIdx.x < 8 ? v[threadIdx.x] : -v[threadIdx.x - 8];+ }+}+// block p's 32 values, exactly (a code times a power of two)+__device__ __forceinline__ void mx_block(const uint8_t* __restrict__ p, const float* lut, float* w) {+ const float d = e8m0_half(p[0]);+#pragma unroll+ for (int j = 0; j < 16; ++j) {+ const uint32_t q = p[1 + j];+ w[j] = lut[q & 15u] * d;+ w[j + 16] = lut[q >> 4] * d;+ }+}+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {+ if (limit > 0.0f) {+ gv = fminf(gv, limit);+ uv = fminf(fmaxf(uv, -limit), limit);+ }+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);+}+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {+ float m = 0.0f;+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);+ return __fadd_rn(m, sh);+}++// a Q8_0 block's 32 values, exactly (an FP16 scale times an int8; the block is only 2-byte aligned)+__device__ __forceinline__ void q8_block(const uint8_t* __restrict__ p, float* w) {+ const float d = __half2float(__ushort_as_half(*(const uint16_t*) p));+ const uint16_t* q = (const uint16_t*) (p + 2);+#pragma unroll+ for (int j = 0; j < 16; ++j) {+ const uint16_t v = q[j];+ w[2 * j] = d * (float) (int8_t) (v & 0xff);+ w[2 * j + 1] = d * (float) (int8_t) (v >> 8);+ }+}++// ---- int8 activations (llama.cpp's MMVQ: its q8_1 blocks' rounding)+// rows of k F32 values (row r at x + r ldx; k % 32 == 0) -> q[r][k] int8 and d[r][k / 32] = max|x| / 127 per 32+__global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restrict__ x, size_t ldx, int k,+ int8_t* __restrict__ q, float* __restrict__ d) {+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (b * 32 >= k) return;+ const float v = x[(size_t) r * ldx + (size_t) b * 32 + lane];+ float am = fabsf(v);+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));+ const float sc = am / 127.0f;+ q[(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);+ if (lane == 0) d[(size_t) r * (k / 32) + b] = sc;+}+void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {+ const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);+ q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);+}++// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),+// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4+__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {+ const uint32_t t0 = 0x03020100u, t1 = 0x0C080604u, t2 = 0xFDFEFF00u, t3 = 0xF4F8FAFCu;+ const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);+ uint32_t tmp[2];+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const uint32_t idx = (q4 >> (16 * i)) & 0x7777u;+ const uint32_t lo = __byte_perm(t0, t1, idx), hi = __byte_perm(t2, t3, idx);+ tmp[i] = __byte_perm(lo, hi, sel >> (16 * i));+ }+ return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));+}+// an MXFP4 block's codes (4 words) against 32 int8 activations (xa: values 0..15, xb: 16..31)+__device__ __forceinline__ int mx_dot(const uint4& q, const int4& xa, const int4& xb) {+ int sum = 0;+ int2 v = e2m1_x8(q.x);+ sum = __dp4a(v.x, xa.x, sum);+ sum = __dp4a(v.y, xb.x, sum);+ v = e2m1_x8(q.y);+ sum = __dp4a(v.x, xa.y, sum);+ sum = __dp4a(v.y, xb.y, sum);+ v = e2m1_x8(q.z);+ sum = __dp4a(v.x, xa.z, sum);+ sum = __dp4a(v.y, xb.z, sum);+ v = e2m1_x8(q.w);+ sum = __dp4a(v.x, xa.w, sum);+ sum = __dp4a(v.y, xb.w, sum);+ return sum;+}+// the warp copies the blocks [b0, b0 + nbk) of a row (BB bytes each; the row 4-byte aligned, nbk BB % 4 == 0) into its+// stage in 4-byte words; every lane calls it+template <int BB>+__device__ __forceinline__ void stage_chunk(const uint8_t* row, int b0, int nbk, uint32_t* st) {+ const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);+ __syncwarp(); // (the lanes are done with the chunk before)+ for (int i = threadIdx.x & 31; i < nbk * BB / 4; i += 32) st[i] = src[i];+ __syncwarp();+}+// a row's next chunk of 32 blocks (BB bytes each) in the lanes' registers, then into the warp's stage+template <int BB>+struct Fetch {+ static constexpr int W = (32 * BB / 4 + 31) / 32; // words a lane+ uint32_t r[W];+ int n = 0;+ __device__ __forceinline__ void load(const uint8_t* row, int b0, int nbk) {+ const uint32_t* src = (const uint32_t*) (row + (size_t) b0 * BB);+ n = nbk * BB / 4;+#pragma unroll+ for (int j = 0; j < W; ++j) {+ const int i = (int) (threadIdx.x & 31) + 32 * j;+ r[j] = i < n ? src[i] : 0u;+ }+ }+ __device__ __forceinline__ void store(uint32_t* st) const {+ __syncwarp(); // (the lanes are done with the chunk before)+#pragma unroll+ for (int j = 0; j < W; ++j) {+ const int i = (int) (threadIdx.x & 31) + 32 * j;+ if (i < n) st[i] = r[j];+ }+ __syncwarp();+ }+};++// MXFP4 block i of a stage (17-byte blocks from a 4-byte aligned start): its scale byte, its 16 code bytes as 4 words+__device__ __forceinline__ uint32_t mx_words_s(const uint32_t* st, int i, uint4& q) {+ const int o = 17 * i, w = o >> 2, sh = (o & 3) * 8;+ const uint32_t a0 = st[w], a1 = st[w + 1], a2 = st[w + 2], a3 = st[w + 3], a4 = st[w + 4];+ const uint32_t s0 = __funnelshift_r(a0, a1, sh), s1 = __funnelshift_r(a1, a2, sh), s2 = __funnelshift_r(a2, a3, sh),+ s3 = __funnelshift_r(a3, a4, sh), s4 = a4 >> sh;+ q.x = __funnelshift_r(s0, s1, 8);+ q.y = __funnelshift_r(s1, s2, 8);+ q.z = __funnelshift_r(s2, s3, 8);+ q.w = __funnelshift_r(s3, s4, 8);+ return s0 & 0xffu;+}+// ... the same from global memory, byte by byte (rows not 4-byte aligned)+__device__ __forceinline__ uint32_t mx_words_g(const uint8_t* __restrict__ p, uint4& q) {+ uint32_t w[4];+#pragma unroll+ for (int j = 0; j < 4; ++j)+ w[j] = (uint32_t) p[1 + 4 * j] | (uint32_t) p[2 + 4 * j] << 8 | (uint32_t) p[3 + 4 * j] << 16 |+ (uint32_t) p[4 + 4 * j] << 24;+ q = make_uint4(w[0], w[1], w[2], w[3]);+ return p[0];+}+// Q8_0 block i of a stage (34-byte blocks): its scale, its 32 int8 as 8 words+__device__ __forceinline__ float q8_words_s(const uint32_t* st, int i, int* v) {+ const int o = 34 * i, w = o >> 2, sh = (o & 3) * 8;+ uint32_t a[9];+#pragma unroll+ for (int j = 0; j < 9; ++j) a[j] = st[w + j];+ uint32_t sw[9];+#pragma unroll+ for (int j = 0; j < 8; ++j) sw[j] = __funnelshift_r(a[j], a[j + 1], sh);+ sw[8] = a[8] >> sh;+#pragma unroll+ for (int j = 0; j < 8; ++j) v[j] = (int) __funnelshift_r(sw[j], sw[j + 1], 16);+ return __half2float(__ushort_as_half((uint16_t) (sw[0] & 0xffffu)));+}+__device__ __forceinline__ float q8_words_g(const uint8_t* __restrict__ p, int* v) {+#pragma unroll+ for (int j = 0; j < 8; ++j)+ v[j] = (int) ((uint32_t) p[2 + 4 * j] | (uint32_t) p[3 + 4 * j] << 8 | (uint32_t) p[4 + 4 * j] << 16 |+ (uint32_t) p[5 + 4 * j] << 24);+ return __half2float(__ushort_as_half((uint16_t) (p[0] | p[1] << 8)));+}++// grid (max rows / 8, jobs): a Q8_0 matrix's rows against the jobs' int8 activations (q8_quant's: activation c of job+// j at xq + c k, its scales at xd + c k / 32) - a warp a row, a lane a block a step, the blocks staged (ST) or read+template <int NC, bool ST>+__global__ void __launch_bounds__(kThreads) q8_mv_kernel(const BfJobs J) {+ __shared__ __align__(16) uint32_t stage[kWarps][32 * kQ8Bytes / 4];+ const BfJob& b = J.j[blockIdx.y];+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (row >= b.n) return;+ const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;+ const int nb = b.k / 32;+ float acc[NC];+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;+ Fetch<kQ8Bytes> F;+ if (ST) F.load(w, 0, min(32, nb));+ for (int b0 = 0; b0 < nb; b0 += 32) {+ const int nbk = min(32, nb - b0), kb = b0 + lane;+ if (ST) {+ F.store(stage[threadIdx.x >> 5]);+ if (b0 + 32 < nb) F.load(w, b0 + 32, min(32, nb - b0 - 32));+ }+ if (lane >= nbk) continue;+ int v[8];+ const float dw = ST ? q8_words_s(stage[threadIdx.x >> 5], lane, v) : q8_words_g(w + (size_t) kb * kQ8Bytes, v);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);+ const int4 xa = xp[0], xb = xp[1];+ int sum = __dp4a(v[0], xa.x, 0);+ sum = __dp4a(v[1], xa.y, sum);+ sum = __dp4a(v[2], xa.z, sum);+ sum = __dp4a(v[3], xa.w, sum);+ sum = __dp4a(v[4], xb.x, sum);+ sum = __dp4a(v[5], xb.y, sum);+ sum = __dp4a(v[6], xb.z, sum);+ sum = __dp4a(v[7], xb.w, sum);+ acc[c] = fmaf(dw * b.xd[(size_t) c * nb + kb], (float) sum, acc[c]);+ }+ }+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float v = warp_sum(acc[c]);+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// grid (max rows / 8, jobs): bf16_mv_kernel for MXFP4 (Q8 false) or Q8_0 (Q8 true) matrices - a lane a 32-value block+// a step, dequantised once for the job's activations+template <int NC, bool VEC, bool Q8>+__global__ void __launch_bounds__(kThreads) mx_mv_kernel(const BfJobs J) {+ __shared__ float lut[16];+ lut_init(lut);+ __syncthreads();+ const BfJob& b = J.j[blockIdx.y];+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (row >= b.n) return;+ const uint8_t* w = (const uint8_t*) b.w + (size_t) row * b.ld;+ float acc[NC];+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (int kb = lane; kb < b.k / kMxBlock; kb += 32) {+ float wv[32];+ if (Q8) q8_block(w + (size_t) kb * kQ8Bytes, wv);+ else mx_block(w + (size_t) kb * kMxBytes, lut, wv);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float* x = b.x + c * b.xcs + kb * kMxBlock;+ float a = acc[c];+ if (VEC) {+#pragma unroll+ for (int q = 0; q < 8; ++q) {+ const float4 xv = __ldg((const float4*) x + q);+ a = fmaf(wv[4 * q], xv.x, a);+ a = fmaf(wv[4 * q + 1], xv.y, a);+ a = fmaf(wv[4 * q + 2], xv.z, a);+ a = fmaf(wv[4 * q + 3], xv.w, a);+ }+ } else {+#pragma unroll+ for (int j = 0; j < 32; ++j) a = fmaf(wv[j], x[j], a);+ }+ acc[c] = a;+ }+ }+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float v = warp_sum(acc[c]);+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// a token's experts (grp_ptr null: group i = expert i of the plan, plan_ptr[i] its blob or 0 off the device, its one+// entry i, every entry's input x) or a window's groups (group g: one expert, blob grp_ptr[g], entries e = t * 8 + i at+// grp_ent[grp_start[g] ..), entry e's input x + (e >> 3) x_stride)+struct MxArgs {+ const unsigned long long* plan_ptr;+ const unsigned long long* grp_ptr;+ const int *grp_start, *grp_ent, *n_groups;+ const float* x;+ size_t x_stride;+ float* h; // [entries][n_ff]+ float* part; // [entries][n_embd]+ size_t tr[3];+ int n_embd, n_ff;+ float limit;+ const int8_t* xq; // x rounded to int8 (a token's: row 0; a window's: row t), q8_quant's rows of n_embd+ const float* xd;+ const int8_t* hq; // h rounded to int8, row e = the entry, rows of n_ff+ const float* hd;+};++__device__ __forceinline__ bool mx_group(const MxArgs& a, int grp, const uint8_t*& blob, int& m0, int& m1) {+ if (a.grp_ptr == nullptr) {+ const unsigned long long p = a.plan_ptr[grp];+ if (p == 0ull) return false;+ blob = (const uint8_t*) p;+ m0 = grp;+ m1 = grp + 1;+ return true;+ }+ if (grp >= *a.n_groups) return false;+ blob = (const uint8_t*) a.grp_ptr[grp];+ m0 = a.grp_start[grp];+ m1 = a.grp_start[grp + 1];+ return true;+}++// grid (n_ff / 8, groups): a warp a row of the gate and of the up, for NE of the group's entries a pass (each weight+// block decoded once for them) against the entries' int8 x -> h = the clamped swiglu+template <int NE, bool ST>+__global__ void __launch_bounds__(kThreads) mx_gate_up_kernel(const MxArgs a) {+ __shared__ __align__(16) uint32_t stage[kWarps][2][32 * kMxBytes / 4];+ __shared__ int es[NE], xr[NE];+ const uint8_t* blob;+ int m0, m1;+ if (!mx_group(a, blockIdx.y, blob, m0, m1)) return; // (uniform)+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ const size_t rb = mx_row_bytes(a.n_embd);+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;+ const int nb = a.n_embd / kMxBlock;+ for (int mm = m0; mm < m1; mm += NE) {+ const int ne = min(NE, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) {+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ es[threadIdx.x] = e;+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;+ }+ __syncthreads();+ if (row >= a.n_ff) continue;+ float ag[NE], au[NE];+#pragma unroll+ for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;+ Fetch<kMxBytes> FG, FU;+ if (ST) {+ FG.load(g, 0, min(32, nb));+ FU.load(u, 0, min(32, nb));+ }+ for (int b0 = 0; b0 < nb; b0 += 32) {+ const int nbk = min(32, nb - b0), b = b0 + lane;+ if (ST) {+ FG.store(stage[wp][0]);+ FU.store(stage[wp][1]);+ if (b0 + 32 < nb) {+ FG.load(g, b0 + 32, min(32, nb - b0 - 32));+ FU.load(u, b0 + 32, min(32, nb - b0 - 32));+ }+ }+ if (lane >= nbk) continue;+ uint4 qg, qu;+ const uint32_t eg = ST ? mx_words_s(stage[wp][0], lane, qg) : mx_words_g(g + (size_t) b * kMxBytes, qg);+ const uint32_t eu = ST ? mx_words_s(stage[wp][1], lane, qu) : mx_words_g(u + (size_t) b * kMxBytes, qu);+ const float dg = e8m0_half(eg), du = e8m0_half(eu);+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) b * kMxBlock);+ const int4 xa = xp[0], xb = xp[1];+ const float dx = a.xd[(size_t) xr[i] * nb + b];+ ag[i] = fmaf(dg * dx, (float) mx_dot(qg, xa, xb), ag[i]);+ au[i] = fmaf(du * dx, (float) mx_dot(qu, xa, xb), au[i]);+ }+ }+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const float gv = warp_sum(ag[i]), uv = warp_sum(au[i]);+ if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);+ }+ }+}++// grid (n_embd / 8, groups): a warp a row of the down for NE entries a pass, against their int8 h -> part+template <int NE, bool ST>+__global__ void __launch_bounds__(kThreads) mx_down_kernel(const MxArgs a) {+ __shared__ __align__(16) uint32_t stage[kWarps][32 * kMxBytes / 4];+ __shared__ int es[NE];+ const uint8_t* blob;+ int m0, m1;+ if (!mx_group(a, blockIdx.y, blob, m0, m1)) return;+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ const uint8_t* d = blob + a.tr[2] + (size_t) row * mx_row_bytes(a.n_ff);+ const int nb = a.n_ff / kMxBlock;+ for (int mm = m0; mm < m1; mm += NE) {+ const int ne = min(NE, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ __syncthreads();+ if (row >= a.n_embd) continue;+ float ad[NE];+#pragma unroll+ for (int i = 0; i < NE; ++i) ad[i] = 0.0f;+ Fetch<kMxBytes> FD;+ if (ST) FD.load(d, 0, min(32, nb));+ for (int b0 = 0; b0 < nb; b0 += 32) {+ const int nbk = min(32, nb - b0), b = b0 + lane;+ if (ST) {+ FD.store(stage[wp]);+ if (b0 + 32 < nb) FD.load(d, b0 + 32, min(32, nb - b0 - 32));+ }+ if (lane >= nbk) continue;+ uint4 q;+ const float dd = e8m0_half(ST ? mx_words_s(stage[wp], lane, q) : mx_words_g(d + (size_t) b * kMxBytes, q));+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) b * kMxBlock);+ const int4 ha = hp[0], hb = hp[1];+ ad[i] = fmaf(dd * a.hd[(size_t) es[i] * nb + b], (float) mx_dot(q, ha, hb), ad[i]);+ }+ }+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const float v = warp_sum(ad[i]);+ if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;+ }+ }+}++struct CombArgs {+ const unsigned long long* plan_ptr; // [T][8]+ const float* plan_w; // [T][8]+ const float* part; // [T * 8][n_embd]+ const float* cpu_part;+ size_t cpu_stride;+ const int* cpu_flag;+ const float* sh_out; // [T][n_embd]+ float* out;+ size_t out_stride;+ int k, n_embd;+};+// grid (n_embd / 128, T): each token's outputs - the experts in plan order (an expert off the device adds 0), then+// the CPU lane's part and the shared expert's (x3's combine)+__global__ void __launch_bounds__(128) mx_combine_kernel(const CombArgs a) {+ const int t = blockIdx.y, r = blockIdx.x * 128 + threadIdx.x;+ if (r >= a.n_embd) return;+ float p[8];+ for (int i = 0; i < a.k; ++i) {+ const int e = t * 8 + i;+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;+ }+ a.out[(size_t) t * a.out_stride + r] =+ combine(a.plan_w + t * 8, p, a.k,+ a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);+}++__global__ void mx_wait_kernel(const unsigned* p, unsigned v) {+ if (threadIdx.x != 0) return;+ while (*(volatile const unsigned*) p != v) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700+ __nanosleep(200);+#endif+ }+ __threadfence_system();+}++// ---------------------------------------------------------------- the prompt path's expert sets+// a set's expert i (blob wbase + i stride) owns the sorted rows [bounds[i], bounds[i + 1]); local row = sorted row -+// row0. A block: BM of its rows x BN outputs, the weights dequantised a 32-value block at a time into shared memory,+// F32 FMAs (each thread TM x TN outputs)+constexpr int BM = 64, BN = 64, TM = 4, TN = 4;+static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");++struct SetK {+ const uint8_t* wbase;+ size_t stride;+ size_t tr[3];+ const int* bounds;+ int row0;+ int n_embd, n_ff;+ float limit;+ const float* xf; // gate / up: F32 input rows x[idx[local]] (ldx)+ const int* idx;+ const __half* xh; // down: F16 rows (ldx)+ int ldx;+ __half* h; // gate / up: F16 out+ float* y; // down: F32 out+ int ldo;+ int skip_le; // the experts of at most this many rows left out (the light kernels took them; ds4_iq.cuh)+};++// one 32-value block of BN weight rows (rows c0 .., row bytes rb, block kb) -> ws[32][BN + 1]+__device__ __forceinline__ void mx_tile(const uint8_t* w, size_t rb, int c0, int kb, const float* lut,+ float (*ws)[BN + 1]) {+ const int r = threadIdx.x >> 2, q = threadIdx.x & 3; // row r of the tile, values [8 q, 8 q + 8)+ const uint8_t* p = w + (size_t) (c0 + r) * rb + (size_t) kb * kMxBytes;+ const float d = e8m0_half(p[0]);+ const int b0 = (q & 1) * 8, hi = q >> 1;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const uint32_t c = p[1 + b0 + j];+ ws[q * 8 + j][r] = lut[hi ? (c >> 4) : (c & 15u)] * d;+ }+}++template <bool GATE_UP>+__global__ void __launch_bounds__(kThreads) mx_set_kernel(const SetK a) {+ __shared__ float lut[16];+ __shared__ float wg[32][BN + 1];+ __shared__ float wu[GATE_UP ? 32 : 1][BN + 1];+ __shared__ float xs[BM][33];+ __shared__ int src[BM];+ lut_init(lut);+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform: before any barrier but lut_init's writes, which nobody+ // reads then; skip_le: the light kernels' experts)+ const int nr = min(BM, hi - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GATE_UP ? a.n_embd : a.n_ff;+ const size_t rb = mx_row_bytes(K);+ const int tid = (int) threadIdx.x;+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;+ __syncthreads();+ const int ty = threadIdx.x / (BN / TN), tx = threadIdx.x % (BN / TN);+ float ag[TM][TN], au[TM][TN];+#pragma unroll+ for (int i = 0; i < TM; ++i)+#pragma unroll+ for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;+ for (int kb = 0; kb < K / kMxBlock; ++kb) {+ mx_tile(blob + a.tr[GATE_UP ? 0 : 2], rb, c0, kb, lut, wg);+ if (GATE_UP) mx_tile(blob + a.tr[1], rb, c0, kb, lut, wu);+ for (int i = threadIdx.x; i < BM * kMxBlock; i += kThreads) { // the rows' 32 inputs+ const int r = i / kMxBlock, c = i % kMxBlock, sr = src[r];+ float v = 0.0f;+ if (sr >= 0) {+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + c;+ v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);+ }+ xs[r][c] = v;+ }+ __syncthreads();+#pragma unroll 8+ for (int kk = 0; kk < kMxBlock; ++kk) {+ float xv[TM], gw[TN], uw[TN];+#pragma unroll+ for (int i = 0; i < TM; ++i) xv[i] = xs[ty * TM + i][kk];+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ gw[j] = wg[kk][tx * TN + j];+ if (GATE_UP) uw[j] = wu[kk][tx * TN + j];+ }+#pragma unroll+ for (int i = 0; i < TM; ++i)+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);+ if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);+ }+ }+ __syncthreads();+ }+#pragma unroll+ for (int i = 0; i < TM; ++i) {+ const int r = ty * TM + i;+ if (r >= nr) continue;+ const size_t lr = (size_t) (r0 + r);+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ const int c = c0 + tx * TN + j;+ if (GATE_UP) a.h[lr * a.ldo + c] = __float2half_rn(swiglu(ag[i][j], au[i][j], a.limit));+ else a.y[lr * a.ldo + c] = ag[i][j];+ }+ }+}++// the same on the tensor cores: a block BM rows x BN outputs, 8 warps each 16 rows x 32 outputs (2 fragments, gate and+// up: 4), a 32-value MXFP4 block of K a step (2 WMMA k-steps), the tiles in shared memory as F16+#if !defined(STRATA_USE_HIP)+namespace wm = nvcuda::wmma;+constexpr int kLdT = 40; // halves a tile row (32 + 8: WMMA wants a multiple of 8; the pad spreads the banks)+template <bool GATE_UP>+__global__ void __launch_bounds__(kThreads) mx_set_tc_kernel(const SetK a) {+ struct Tiles {+ __half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];+ };+ struct Out {+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];+ };+ __shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];+ __shared__ float lut[16];+ __shared__ int src[BM];+ Tiles& T = *reinterpret_cast<Tiles*>(smem);+ Out& O = *reinterpret_cast<Out*>(smem);+ lut_init(lut);+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ const int nr = min(BM, hi - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GATE_UP ? a.n_embd : a.n_ff;+ const size_t rb = mx_row_bytes(K);+ const int tid = (int) threadIdx.x, warp = tid >> 5;+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;+ __syncthreads();+ const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the warp's 16 rows, its 2 x 16 outputs+ wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];+ for (int j = 0; j < 2; ++j) {+ wm::fill_fragment(fg[j], 0.0f);+ wm::fill_fragment(fu[j], 0.0f);+ }+ const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)+ for (int kb = 0; kb < K / kMxBlock; ++kb) {+ { // the weights: row tr of the outputs, values 8 tq ..+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {+ const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;+ const float d = e8m0_half(q[0]);+ const int b0 = (tq & 1) * 8, hn = tq >> 1;+ __half* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const uint32_t c = q[1 + b0 + j];+ dst[j] = __float2half_rn(lut[hn ? (c >> 4) : (c & 15u)] * d);+ }+ }+ // the inputs: row tr, values 8 tq ..+ const int sr = src[tr];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ __half v = __float2half_rn(0.0f);+ if (sr >= 0) {+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;+ v = GATE_UP ? __float2half_rn(a.xf[at]) : a.xh[at];+ }+ T.x[tr][tq * 8 + j] = v;+ }+ }+ __syncthreads();+#pragma unroll+ for (int kk = 0; kk < 32; kk += 16) {+ wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;+ wm::load_matrix_sync(fa, &T.x[wm_ * 16][kk], kLdT);+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;+ wm::load_matrix_sync(fb, &T.wg[(wn + j) * 16][kk], kLdT);+ wm::mma_sync(fg[j], fa, fb, fg[j]);+ if (GATE_UP) {+ wm::load_matrix_sync(fb, &T.wu[(wn + j) * 16][kk], kLdT);+ wm::mma_sync(fu[j], fa, fb, fu[j]);+ }+ }+ }+ __syncthreads();+ }+ for (int j = 0; j < 2; ++j) {+ wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);+ if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);+ }+ __syncthreads();+ for (int i = tid; i < BM * BN; i += kThreads) {+ const int r = i / BN, c = i % BN;+ if (r >= nr) continue;+ const size_t lr = (size_t) (r0 + r);+ if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit));+ else a.y[lr * a.ldo + c0 + c] = O.g[r][c];+ }+}+#endif+#if defined(STRATA_USE_HIP)+// the same on an AMD wave32 GPU's WMMA (RDNA3 / RDNA3.5 gfx11, RDNA4 gfx12): 8 waves each 16 rows x 32 outputs, the+// tiles in shared memory as F16, F32 accumulation - each lane gives its row / column's 16 values of a k-step (gfx12:+// its half of them), and holds 8 of the 16 x 16 results (rows 2 i + hi on gfx11, 8 hi + i on gfx12; column lane % 16)+using h16v = _Float16 __attribute__((ext_vector_type(16)));+using f8v = float __attribute__((ext_vector_type(8)));+__device__ __forceinline__ f8v set_wmma(h16v a, h16v b, f8v c) {+#if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) || \+ defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__)+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);+#elif defined(__gfx1200__) || defined(__gfx1201__)+ using h8 = _Float16 __attribute__((ext_vector_type(8)));+ const int k0 = (threadIdx.x & 31) / 16 * 8;+ h8 aa, bb;+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ aa[i] = a[k0 + i];+ bb[i] = b[k0 + i];+ }+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(aa, bb, c);+#else+ __builtin_trap();+ return c;+#endif+}+__device__ __forceinline__ int set_wmma_row(int i, int hi) {+#if defined(__gfx1200__) || defined(__gfx1201__)+ return 8 * hi + i;+#else+ return 2 * i + hi;+#endif+}+constexpr int kLdH = 40; // halves a tile row (32 + 8: the pad spreads the banks)+template <bool GATE_UP>+__global__ void __launch_bounds__(kThreads) mx_set_wmma_kernel(const SetK a) {+ struct Tiles {+ _Float16 x[BM][kLdH], wg[BN][kLdH], wu[GATE_UP ? BN : 1][kLdH];+ };+ struct Out {+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];+ };+ __shared__ __align__(16) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];+ __shared__ float lut[16];+ __shared__ int src[BM];+ Tiles& T = *reinterpret_cast<Tiles*>(smem);+ Out& O = *reinterpret_cast<Out*>(smem);+ lut_init(lut);+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ const int nr = min(BM, hi_r - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GATE_UP ? a.n_embd : a.n_ff;+ const size_t rb = mx_row_bytes(K);+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;+ __syncthreads();+ const int wm_ = warp >> 1, wn = (warp & 1) * 2; // the wave's 16 rows, its 2 x 16 outputs+ f8v fg[2], fu[2];+#pragma unroll+ for (int j = 0; j < 2; ++j)+#pragma unroll+ for (int i = 0; i < 8; ++i) fg[j][i] = fu[j][i] = 0.0f;+ const int tr = tid >> 2, tq = tid & 3; // the thread's tile row, its 8 values [8 tq, 8 tq + 8)+ for (int kb = 0; kb < K / kMxBlock; ++kb) {+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {+ const uint8_t* q = blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb + (size_t) kb * kMxBytes;+ const float d = e8m0_half(q[0]);+ const int b0 = (tq & 1) * 8, hn = tq >> 1;+ _Float16* dst = m == 0 ? &T.wg[tr][tq * 8] : &T.wu[tr][tq * 8];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const uint32_t cc = q[1 + b0 + j];+ dst[j] = (_Float16) (lut[hn ? (cc >> 4) : (cc & 15u)] * d);+ }+ }+ const int sr = src[tr];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ float v = 0.0f;+ if (sr >= 0) {+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * kMxBlock + tq * 8 + j;+ v = GATE_UP ? a.xf[at] : __half2float(a.xh[at]);+ }+ T.x[tr][tq * 8 + j] = (_Float16) v;+ }+ __syncthreads();+#pragma unroll+ for (int kk = 0; kk < 32; kk += 16) {+ h16v av;+#pragma unroll+ for (int i = 0; i < 16; ++i) av[i] = T.x[wm_ * 16 + l16][kk + i];+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ h16v bv;+#pragma unroll+ for (int i = 0; i < 16; ++i) bv[i] = T.wg[(wn + j) * 16 + l16][kk + i];+ fg[j] = set_wmma(av, bv, fg[j]);+ if (GATE_UP) {+#pragma unroll+ for (int i = 0; i < 16; ++i) bv[i] = T.wu[(wn + j) * 16 + l16][kk + i];+ fu[j] = set_wmma(av, bv, fu[j]);+ }+ }+ }+ __syncthreads();+ }+#pragma unroll+ for (int j = 0; j < 2; ++j)+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int r = wm_ * 16 + set_wmma_row(i, hi), cc = (wn + j) * 16 + l16;+ O.g[r][cc] = fg[j][i];+ if (GATE_UP) O.u[r][cc] = fu[j][i];+ }+ __syncthreads();+ for (int i = tid; i < BM * BN; i += kThreads) {+ const int r = i / BN, cc = i % BN;+ if (r >= nr) continue;+ const size_t lr = (size_t) (r0 + r);+ if (GATE_UP) a.h[lr * a.ldo + c0 + cc] = __float2half_rn(swiglu(O.g[r][cc], O.u[r][cc], a.limit));+ else a.y[lr * a.ldo + c0 + cc] = O.g[r][cc];+ }+}+#endif+bool tc_ok() { // sm_70 and later (WMMA F16) / AMD gfx11, gfx12 (wave32 WMMA); STRATA_DS4_NO_TC=1 keeps the F32 kernel+#if defined(STRATA_USE_HIP)+ static const bool ok = [] {+ int dev = 0;+ cudaGetDevice(&dev);+ cudaDeviceProp prop{};+ if (cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;+ const char* v = std::getenv("STRATA_DS4_NO_TC");+ if (v != nullptr && v[0] == '1') return false;+ const char* arch = prop.gcnArchName;+ for (const char* p : {"gfx1100", "gfx1101", "gfx1102", "gfx1103", "gfx1150", "gfx1151", "gfx1152", "gfx1200", "gfx1201"})+ if (std::strncmp(arch, p, 7) == 0) return true;+ return false;+ }();+ return ok;+#else+ static const bool ok = [] {+ int dev = 0, major = 0;+ cudaGetDevice(&dev);+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ const char* v = std::getenv("STRATA_DS4_NO_TC");+ return major >= 7 && !(v != nullptr && v[0] == '1');+ }();+ return ok;+#endif+}++// ---------------------------------------------------------------- scratch+constexpr int kMaxEmbd = 8192;+constexpr int kMaxQK = 32768; // a Q8_0 GEMV's inputs at most+struct Scratch {+ float* part = nullptr; // [8][kMaxEmbd]: a token's experts' down outputs+ int8_t *xq = nullptr, *hq = nullptr, *mq = nullptr; // int8 rows: x [kMaxEnt], h [kMaxEnt * 8], the GEMVs'+ float *xd = nullptr, *hd = nullptr, *md = nullptr; // [kMaxMvJobs * kMaxMvCols] inputs; their scales+};+std::mutex g_mu;+std::unordered_map<cudaStream_t, Scratch> g_scratch;++Scratch* scratch(cudaStream_t s) {+ std::lock_guard<std::mutex> lk(g_mu);+ auto it = g_scratch.find(s);+ if (it != g_scratch.end()) return &it->second;+ Scratch sc;+ const size_t nx = (size_t) kMaxEnt * kMaxEmbd, nh = (size_t) kMaxEnt * 8 * kMaxEmbd,+ nm = (size_t) x3::kMaxMvJobs * x3::kMaxMvCols * kMaxQK;+ if (cudaMalloc(&sc.part, (size_t) 8 * kMaxEmbd * sizeof(float)) != cudaSuccess ||+ cudaMalloc(&sc.xq, nx) != cudaSuccess || cudaMalloc(&sc.xd, nx / 32 * 4) != cudaSuccess ||+ cudaMalloc(&sc.hq, nh) != cudaSuccess || cudaMalloc(&sc.hd, nh / 32 * 4) != cudaSuccess ||+ cudaMalloc(&sc.mq, nm) != cudaSuccess || cudaMalloc(&sc.md, nm / 32 * 4) != cudaSuccess) {+ fail("scratch: cudaMalloc failed");+ return nullptr;+ }+ return &g_scratch.emplace(s, sc).first->second;+}++bool mx_geometry(const char* who, int k, int n_embd, int n_ff) {+ if (k < 1 || k > 8 || n_embd % BN != 0 || n_ff % BN != 0 || n_embd > kMaxEmbd || n_ff % kMxBlock != 0) {+ std::fprintf(stderr, "ds4 gguf %s: geometry not covered (k %d n_embd %d n_ff %d)\n", who, k, n_embd, n_ff);+ g_errors.fetch_add(1, std::memory_order_relaxed);+ return false;+ }+ return true;+}++} // namespace++// ---------------------------------------------------------------- entry points+bool prepare(cudaStream_t s) { return scratch(s) != nullptr; }++int launch_errors() { return g_errors.load(std::memory_order_relaxed); }++size_t type_row_bytes(int type, int64_t k) {+ if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;+ return iq::supported(type) && k % 256 == 0 ? iq::row_bytes(type, k) : 0;+}+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? 256 : 0; }+const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }++namespace {+size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row+ if (m.ld > 0) return (size_t) m.ld;+ return m.fmt == kFmtMXFP4 ? mx_row_bytes(m.k)+ : m.fmt == kFmtQ8_0 ? q8_row_bytes(m.k)+ : m.fmt == kFmtGG ? type_row_bytes(m.K, m.k)+ : (size_t) m.k;+}+// a layer's low-bit experts as ds4_iq's arguments (false: its types not covered - printed)+bool iq_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit, iq::Args& ia, const char* who) {+ ia = iq::Args{};+ for (int r = 0; r < 3; ++r) ia.tr[r] = lay.trellis[r];+ ia.n_embd = n_embd;+ ia.n_ff = n_ff;+ ia.limit = limit;+ ia.tg = expert_type(lay, 0);+ ia.td = expert_type(lay, 2);+ if (ia.tg != expert_type(lay, 1) || !iq::supported(ia.tg) || !iq::supported(ia.td) || n_embd % 256 != 0 ||+ n_ff % 256 != 0) {+ std::fprintf(stderr, "ds4 gguf %s: expert types %s / %s / %s (n_embd %d, n_ff %d) not covered\n", who,+ type_name(expert_type(lay, 0)), type_name(expert_type(lay, 1)), type_name(expert_type(lay, 2)),+ n_embd, n_ff);+ g_errors.fetch_add(1, std::memory_order_relaxed);+ return false;+ }+ return true;+}+template <int NC>+void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {+ const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);+ if (f == 1) {+ if (vec) mx_mv_kernel<NC, true, false><<<grid, kThreads, 0, s>>>(J);+ else mx_mv_kernel<NC, false, false><<<grid, kThreads, 0, s>>>(J);+ } else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)+ if (vec) q8_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);+ else q8_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);+ } else {+ if (vec) bf16_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);+ else bf16_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);+ }+}+void mv_set(int f, const BfJobs& J, int nj, int max_n, int max_c, bool vec, cudaStream_t s) {+ if (nj == 0) return;+ if (max_c <= 1) mv_launch<1>(f, J, nj, max_n, vec, s);+ else if (max_c <= 2) mv_launch<2>(f, J, nj, max_n, vec, s);+ else if (max_c <= 4) mv_launch<4>(f, J, nj, max_n, vec, s);+ else if (max_c <= 8) mv_launch<8>(f, J, nj, max_n, vec, s);+ else mv_launch<16>(f, J, nj, max_n, vec, s);+}+} // namespace++x3::Mat view(const x3::Mat& m, int c0, int nc) {+ x3::Mat v = m;+ const size_t ld = row_stride(m);+ v.ld = (int) ld;+ v.raw = m.fmt != kFmtBF16 ? (const void*) ((const uint8_t*) m.raw + (size_t) c0 * ld)+ : (const void*) ((const uint16_t*) m.raw + (size_t) c0 * ld);+ v.n = nc;+ return v;+}++bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {+ if (n <= 0 || n > x3::kMaxMvJobs) return false;+ BfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch each+ int nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};+ bool vec[3] = {true, true, true};+ iq::MvJob IJ[x3::kMaxMvJobs]; // the low-bit ones (ds4_iq: a launch a type)+ int nij = 0;+ for (int i = 0; i < n; ++i) {+ const x3::MvJob& q = jobs[i];+ const x3::Mat& m = *q.m;+ if (m.fmt == kFmtGG) { // the activations rounded to int8 (job i's rows of the scratch), then ds4_iq's GEMV+ Scratch* sc = scratch(s);+ if (sc == nullptr || m.raw == nullptr || q.xf == nullptr || q.y == nullptr || q.ncol < 1 ||+ q.ncol > x3::kMaxMvCols || m.k > kMaxQK || type_row_bytes(m.K, m.k) == 0) {+ std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (type %s k %d n %d)\n", i, type_name(m.K), m.k, m.n);+ g_errors.fetch_add(1, std::memory_order_relaxed);+ return false;+ }+ const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;+ q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ iq::MvJob& b = IJ[nij++];+ b.w = (const uint8_t*) m.raw;+ b.ld = row_stride(m);+ b.type = m.K;+ b.k = m.k;+ b.n = m.n;+ b.ncol = q.ncol;+ b.xq = sc->mq + at;+ b.xd = sc->md + at / 32;+ b.y = q.y;+ b.ycs = q.y_cs;+ b.bias = q.bias;+ b.alpha = q.alpha;+ continue;+ }+ const int f = m.fmt == kFmtMXFP4 ? 1 : m.fmt == kFmtQ8_0 ? 2 : 0;+ if ((m.fmt != kFmtBF16 && m.fmt != kFmtMXFP4 && m.fmt != kFmtQ8_0) || m.raw == nullptr || q.xf == nullptr ||+ q.y == nullptr || q.ncol < 1 || q.ncol > x3::kMaxMvCols || m.k <= 0 || m.n <= 0 ||+ (f != 0 && m.k % kMxBlock != 0)) {+ std::fprintf(stderr, "ds4 gguf mv: job %d unsupported (fmt %d k %d n %d, %d activations)\n", i, m.fmt, m.k,+ m.n, q.ncol);+ g_errors.fetch_add(1, std::memory_order_relaxed);+ return false;+ }+ BfJob& b = J[f].j[nj[f]++];+ b.w = m.raw;+ b.ld = row_stride(m);+ b.x = q.xf;+ b.xcs = q.xf_cs;+ b.y = q.y;+ b.ycs = q.y_cs;+ b.bias = q.bias;+ b.alpha = q.alpha;+ b.k = m.k;+ b.n = m.n;+ b.ncol = q.ncol;+ max_n[f] = std::max(max_n[f], m.n);+ max_c[f] = std::max(max_c[f], q.ncol);+ const bool xa = ((uintptr_t) b.x & 15u) == 0 && (q.ncol == 1 || b.xcs % 4 == 0);+ if (f == 2) { // the activations rounded to int8 first (each job its own rows of the scratch)+ Scratch* sc = scratch(s);+ if (sc == nullptr || m.k > kMaxQK) {+ fail("mv: Q8_0 inputs past the scratch");+ return false;+ }+ const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;+ b.xq = sc->mq + at;+ b.xd = sc->md + at / 32;+ q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ vec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;+ } else {+ vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));+ }+ }+ for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);+ if (nij > 0 && !iq::mv(IJ, nij, s)) {+ fail("mv: a low-bit job not covered");+ return false;+ }+ check("mv");+ return true;+}++void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {+ if ((m.fmt != kFmtBF16 && m.fmt != kFmtQ8_0) || m.raw == nullptr) {+ fail("rows: not a BF16 or Q8_0 matrix");+ return;+ }+ const size_t ld = row_stride(m);+ const size_t total = (size_t) m.n * (size_t) m.k;+ const unsigned blocks = (unsigned) std::min<size_t>((total + 255) / 256, 65535);+ if (m.fmt == kFmtQ8_0) q8_rows_kernel<<<blocks, 256, 0, s>>>((const uint8_t*) m.raw, ld, m.k, m.n, dst, bf16);+ else bf16_rows_kernel<<<blocks, 256, 0, s>>>((const uint16_t*) m.raw, ld, m.k, m.n, dst, bf16);+ check("rows");+}++namespace {+// (staged when the matrix's rows are 4-byte aligned: 17 nb % 4 == 0; the blobs' offsets are)+template <int NE>+struct GateUpK {+ static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {+ if (mx_row_bytes(m.n_embd) % 4 == 0) mx_gate_up_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);+ else mx_gate_up_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);+ }+};+template <int NE>+struct DownK {+ static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {+ if (mx_row_bytes(m.n_ff) % 4 == 0) mx_down_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);+ else mx_down_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);+ }+};+template <template <int> class Kern>+void launch_groups(const MxArgs& m, int rows, int groups, int ne_max, cudaStream_t s) {+ const dim3 grid((unsigned) ((rows + kWarps - 1) / kWarps), (unsigned) groups);+ if (ne_max <= 1) Kern<1>::run(m, grid, s);+ else if (ne_max <= 4) Kern<4>::run(m, grid, s);+ else if (ne_max <= 8) Kern<8>::run(m, grid, s);+ else Kern<16>::run(m, grid, s);+}+MxArgs mx_args(const x3::ExpertLayout& lay, int n_embd, int n_ff, float limit) {+ MxArgs m{};+ for (int r = 0; r < 3; ++r) m.tr[r] = lay.trellis[r];+ m.n_embd = n_embd;+ m.n_ff = n_ff;+ m.limit = limit;+ return m;+}+} // namespace++void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s) {+ if (!mx_geometry("moe_gate_up", a.k, a.n_embd, a.n_ff) || h == nullptr) return;+ Scratch* sc = scratch(s);+ if (sc == nullptr) return;+ if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's kernels+ iq::Args ia;+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_gate_up")) return;+ ia.plan_ptr = a.plan_ptr;+ ia.h = h;+ ia.xq = sc->xq;+ ia.xd = sc->xd;+ q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);+ iq::gate_up(ia, a.k, 1, s);+ check("moe_gate_up");+ return;+ }+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);+ m.plan_ptr = a.plan_ptr;+ m.x = x;+ m.h = h;+ m.xq = sc->xq;+ m.xd = sc->xd;+ q8_quant(x, (size_t) a.n_embd, 1, a.n_embd, sc->xq, sc->xd, s);+ launch_groups<GateUpK>(m, a.n_ff, a.k, 1, s);+ check("moe_gate_up");+}++void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {+ if (!mx_geometry("moe_down", a.k, a.n_embd, a.n_ff) || h == nullptr) return;+ Scratch* sc = scratch(s);+ if (sc == nullptr) return;+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);+ m.plan_ptr = a.plan_ptr;+ m.h = const_cast<float*>(h);+ m.part = sc->part;+ m.hq = sc->hq;+ m.hd = sc->hd;+ q8_quant(h, (size_t) a.n_ff, a.k, a.n_ff, sc->hq, sc->hd, s);+ if (!mx_experts(a.lay)) { // low-bit experts: ds4_iq's down, then the same combine+ iq::Args ia;+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "moe_down")) return;+ ia.plan_ptr = a.plan_ptr;+ ia.part = sc->part;+ ia.hq = sc->hq;+ ia.hd = sc->hd;+ iq::down(ia, a.k, 1, s);+ } else {+ launch_groups<DownK>(m, a.n_embd, a.k, 1, s);+ }+ CombArgs c{};+ c.plan_ptr = a.plan_ptr;+ c.plan_w = a.plan_w;+ c.part = sc->part;+ c.cpu_part = a.cpu_part;+ c.cpu_stride = (size_t) a.n_embd;+ c.cpu_flag = a.cpu_flag;+ c.sh_out = sh_out;+ c.out = out;+ c.out_stride = (size_t) a.n_embd;+ c.k = a.k;+ c.n_embd = a.n_embd;+ mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s>>>(c);+ check("moe_down");+}++bool win_experts(const x3::WinArgs& a, cudaStream_t s) {+ if (a.T < 1 || a.T > kMaxEnt || !mx_geometry("win_experts", a.k, a.n_embd, a.n_ff) || a.h == nullptr ||+ a.part == nullptr || a.out == nullptr) {+ std::fprintf(stderr, "ds4 gguf win_experts: not covered (T %d)\n", a.T);+ return false;+ }+ MxArgs m = mx_args(a.lay, a.n_embd, a.n_ff, a.limit);+ m.grp_ptr = a.grp_ptr;+ m.grp_start = a.grp_start;+ m.grp_ent = a.grp_ent;+ m.n_groups = a.n_groups;+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ m.x = a.x;+ m.x_stride = a.x_stride;+ m.h = a.h;+ m.part = a.part;+ m.xq = sc->xq;+ m.xd = sc->xd;+ m.hq = sc->hq;+ m.hd = sc->hd;+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)+ const bool mx = mx_experts(a.lay);+ iq::Args ia;+ if (!mx) { // low-bit experts: ds4_iq's kernels on the same groups and scratch+ if (!iq_args(a.lay, a.n_embd, a.n_ff, a.limit, ia, "win_experts")) return false;+ ia.grp_ptr = a.grp_ptr;+ ia.grp_start = a.grp_start;+ ia.grp_ent = a.grp_ent;+ ia.n_groups = a.n_groups;+ ia.h = a.h;+ ia.part = a.part;+ ia.xq = sc->xq;+ ia.xd = sc->xd;+ ia.hq = sc->hq;+ ia.hd = sc->hd;+ }+ q8_quant(a.x, a.x_stride, a.T, a.n_embd, sc->xq, sc->xd, s);+ if (mx) launch_groups<GateUpK>(m, a.n_ff, groups, a.T, s);+ else iq::gate_up(ia, groups, a.T, s);+ q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)+ if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);+ else iq::down(ia, groups, a.T, s);+ if (a.wait_ptr != nullptr) mx_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);+ CombArgs c{};+ c.plan_ptr = a.plan_ptr;+ c.plan_w = a.plan_w;+ c.part = a.part;+ c.cpu_part = a.cpu_part;+ c.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;+ c.cpu_flag = a.cpu_flag;+ c.sh_out = a.sh_out;+ c.out = a.out;+ c.out_stride = a.out_stride;+ c.k = a.k;+ c.n_embd = a.n_embd;+ mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s>>>(c);+ check("win_experts");+ return true;+}++namespace {+iq::SetArgs iq_set(const x3::SetArgs& sa) { // a set as ds4_iq's arguments (the inputs / outputs and types to fill in)+ iq::SetArgs q;+ q.wbase = sa.wbase;+ q.stride = sa.stride;+ for (int r = 0; r < 3; ++r) q.tr[r] = sa.lay.trellis[r];+ q.bounds = sa.bounds;+ q.row0 = sa.row0;+ q.max_rows = sa.max_rows;+ q.n_exp = sa.n_exp;+ q.n_embd = sa.n_embd;+ q.n_ff = sa.n_ff;+ q.limit = sa.limit;+ return q;+}+} // namespace++void set_gate_up(const x3::SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,+ cudaStream_t s) {+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;+ if (!mx_geometry("set_gate_up", 1, sa.n_embd, sa.n_ff)) return;+ if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets+ iq::Args ia;+ if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_gate_up")) return;+ iq::SetArgs q = iq_set(sa);+ q.xf = x;+ q.idx = row_idx;+ q.ldx = ldx;+ q.h = h16;+ q.ldo = ldh;+ q.tg = ia.tg;+ q.td = ia.td;+ if (!iq::set_gate_up(q, s)) fail("set_gate_up: not covered");+ check("set_gate_up");+ return;+ }+ int skip = 0;+ { // the experts of at most 16 rows: the light kernels (each weight read once; ds4_iq.cuh), the others the tiles+ iq::SetArgs q = iq_set(sa);+ q.xf = x;+ q.idx = row_idx;+ q.ldx = ldx;+ q.h = h16;+ q.ldo = ldh;+ q.tg = q.td = kTypeMXFP4;+ if (iq::light_gate_up(q, s)) {+ if (sa.max_rows <= 16) {+ check("set_gate_up");+ return;+ }+ skip = 16;+ }+ }+ SetK k{};+ k.skip_le = skip;+ k.wbase = sa.wbase;+ k.stride = sa.stride;+ for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];+ k.bounds = sa.bounds;+ k.row0 = sa.row0;+ k.n_embd = sa.n_embd;+ k.n_ff = sa.n_ff;+ k.limit = sa.limit;+ k.xf = x;+ k.idx = row_idx;+ k.ldx = ldx;+ k.h = (__half*) h16;+ k.ldo = ldh;+ const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_ff / BN), (unsigned) sa.n_exp);+#if !defined(STRATA_USE_HIP)+ if (tc_ok()) mx_set_tc_kernel<true><<<grid, kThreads, 0, s>>>(k);+#else+ if (tc_ok()) mx_set_wmma_kernel<true><<<grid, kThreads, 0, s>>>(k);+#endif+ else+ mx_set_kernel<true><<<grid, kThreads, 0, s>>>(k);+ check("set_gate_up");+}++void set_down(const x3::SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;+ if (!mx_geometry("set_down", 1, sa.n_embd, sa.n_ff)) return;+ if (!mx_experts(sa.lay)) { // low-bit experts: ds4_iq's sets+ iq::Args ia;+ if (!iq_args(sa.lay, sa.n_embd, sa.n_ff, sa.limit, ia, "set_down")) return;+ iq::SetArgs q = iq_set(sa);+ q.xh = h16;+ q.ldx = ldh;+ q.y = y;+ q.ldo = ldy;+ q.tg = ia.tg;+ q.td = ia.td;+ if (!iq::set_down(q, s)) fail("set_down: not covered");+ check("set_down");+ return;+ }+ int skip = 0;+ { // (as set_gate_up's)+ iq::SetArgs q = iq_set(sa);+ q.xh = h16;+ q.ldx = ldh;+ q.y = y;+ q.ldo = ldy;+ q.tg = q.td = kTypeMXFP4;+ if (iq::light_down(q, s)) {+ if (sa.max_rows <= 16) {+ check("set_down");+ return;+ }+ skip = 16;+ }+ }+ SetK k{};+ k.skip_le = skip;+ k.wbase = sa.wbase;+ k.stride = sa.stride;+ for (int r = 0; r < 3; ++r) k.tr[r] = sa.lay.trellis[r];+ k.bounds = sa.bounds;+ k.row0 = sa.row0;+ k.n_embd = sa.n_embd;+ k.n_ff = sa.n_ff;+ k.xh = (const __half*) h16;+ k.ldx = ldh;+ k.y = y;+ k.ldo = ldy;+ const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_embd / BN), (unsigned) sa.n_exp);+#if !defined(STRATA_USE_HIP)+ if (tc_ok()) mx_set_tc_kernel<false><<<grid, kThreads, 0, s>>>(k);+#else+ if (tc_ok()) mx_set_wmma_kernel<false><<<grid, kThreads, 0, s>>>(k);+#endif+ else+ mx_set_kernel<false><<<grid, kThreads, 0, s>>>(k);+ check("set_down");+}++} // namespace strata::ds4::ggufdiff --git a/src/dsv41/ds4_gguf.cuh b/src/dsv41/ds4_gguf.cuhnew file mode 100644index 000000000..6257c8a7f--- /dev/null+++ b/src/dsv41/ds4_gguf.cuh@@ -0,0 +1,76 @@+// src/ds4/ds4_gguf.cuh - a GGUF's weights on the device for maya-ds4 (unsloth's DeepSeek-V4.1-Flash-1017 GGUFs,+// tools/ds4/pack_gguf.py): the dense matrices in BF16 and the routed experts in MXFP4. They stand in for+// strata::kernels::exl3's matrices and experts (an x3::Mat with fmt kFmtBF16, an x3::ExpertLayout with fmt kFmtMXFP4)+// with the same arguments and the same results' layout, so the engine's paths (decode, verify windows, chunked prompts,+// tensor parallel) take either format.+//+// MXFP4 is ggml's block_mxfp4: 32 values in 17 bytes, an E8M0 scale e then 16 bytes of E2M1 codes - value j of the+// block in byte j's low nibble (j < 16) or byte j - 16's high nibble, worth kvalues[code] * 2^(e - 128) with kvalues+// {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12} (the E2M1 values doubled). A matrix is [out][in / 32] blocks+// (row r = output r), as the GGUF stores it, so an expert's blob is its gate, up and down matrices one after another.+// Every product is an F32 sum of exactly dequantised weights times F32 activations, and a window's entry gets the bits+// the token path gives it (the same kernel, the same order).+#pragma once++#include "strata/kernels/exl3.hpp"++#include <cstddef>+#include <cstdint>+#include <cuda_runtime.h>++namespace strata::ds4::gguf {++namespace x3 = strata::kernels::exl3;++constexpr int kFmtBF16 = 1; // x3::Mat::fmt+constexpr int kFmtMXFP4 = 2; // x3::ExpertLayout::fmt (and an expert's matrix as an x3::Mat)+constexpr int kFmtQ8_0 = 3; // x3::Mat::fmt: ggml's Q8_0, an FP16 scale and 32 int8 in 34 bytes (UD-Q4_K_XL's dense)+constexpr int kFmtGG = 4; // x3::Mat::fmt: ggml's low-bit block type m.K (ds4_iq.cuh's: IQ2_XXS .. Q4_K), [n][k / 256]+constexpr int kMxBlock = 32, kMxBytes = 17, kQ8Bytes = 34;+constexpr int kTypeMXFP4 = 39; // ggml's type id++/// Bytes of an MXFP4 row of k values (k % 32 == 0), and of a matrix of n such rows.+__host__ __device__ inline size_t mx_row_bytes(int64_t k) { return (size_t) (k / kMxBlock) * kMxBytes; }+__host__ __device__ inline size_t mx_bytes(int64_t n, int64_t k) { return (size_t) n * mx_row_bytes(k); }+__host__ __device__ inline size_t q8_row_bytes(int64_t k) { return (size_t) (k / 32) * kQ8Bytes; }++/// A GGUF expert layer (x3::ExpertLayout::fmt kFmtMXFP4: block-quantized experts read in place) keeps each role's ggml+/// type in lay.K[r] (r: gate, up, down; 0 = MXFP4): MXFP4 takes these kernels, the low-bit types ds4_iq.cuh's (gate+/// and up of one type). Roles of one layer may differ (down of another type); MXFP4 and a low-bit type do not mix.+inline int expert_type(const x3::ExpertLayout& l, int r) { return l.K[r] == 0 ? kTypeMXFP4 : l.K[r]; }+inline bool mx_experts(const x3::ExpertLayout& l) {+ return expert_type(l, 0) == kTypeMXFP4 && expert_type(l, 1) == kTypeMXFP4 && expert_type(l, 2) == kTypeMXFP4;+}+/// Bytes of a row of k values of an expert type (0: not one the device kernels take, or k not whole blocks).+size_t type_row_bytes(int type, int64_t k);+/// Values a block of an expert type (32: MXFP4, 256: the low-bit ones; 0: neither).+int type_block(int type);+/// The type's name (ggml's), "?" when unknown.+const char* type_name(int type);++/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).+bool prepare(cudaStream_t s);++/// x3::mv's contract for GGUF matrices (F32 activations): each job's m->fmt kFmtBF16 (raw = BF16 [n][ld], ld values a+/// row), kFmtMXFP4 (raw = MXFP4 [n][k / 32] blocks, ld bytes a row; 0: mx_row_bytes(k)) - an expert's matrix in VRAM -+/// or kFmtQ8_0 (raw = Q8_0 [n][k / 32] blocks, ld bytes a row; 0: q8_row_bytes(k)).+bool mv(const x3::MvJob* jobs, int n, cudaStream_t s);+/// The columns [c0, c0 + nc) of such a matrix: its rows c0 .. c0 + nc - 1, a contiguous range.+x3::Mat view(const x3::Mat& m, int c0, int nc);+/// x3::reconstruct_rows' layout from a BF16 or Q8_0 matrix: dst[r][c] = W[c][r] (n rows of k values), F16 or BF16+/// bits (a Q8_0 weight rounded to them).+void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s);++/// x3::moe_gate_up / moe_down / win_experts / set_gate_up / set_down for a GGUF's experts (lay.fmt == kFmtMXFP4; the+/// roles' types in lay.K, see expert_type).+void moe_gate_up(const x3::MoeArgs& a, const float* x, float* h, cudaStream_t s);+void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);+bool win_experts(const x3::WinArgs& a, cudaStream_t s);+void set_gate_up(const x3::SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,+ cudaStream_t s);+void set_down(const x3::SetArgs& a, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s);++/// Launch failures of these kernels since the process started (each is also printed).+int launch_errors();++} // namespace strata::ds4::ggufdiff --git a/src/dsv41/ds4_gguf_cpu.cpp b/src/dsv41/ds4_gguf_cpu.cppnew file mode 100644index 000000000..beae19dcf--- /dev/null+++ b/src/dsv41/ds4_gguf_cpu.cpp@@ -0,0 +1,416 @@+// src/ds4/ds4_gguf_cpu.cpp - a GGUF's MXFP4 experts on the CPU lane (see ds4_gguf_cpu.hpp).+#include "ds4_gguf_cpu.hpp"++#include "strata/kernels/cpu/exl3_cpu.hpp"++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <string>+#include <vector>++#if defined(__x86_64__) || defined(_M_X64)+#define MX_X86 1+#include <immintrin.h>+#if defined(_MSC_VER)+#include <intrin.h>+#else+#include <cpuid.h>+#endif+#endif++// a function's instruction set (GCC / Clang, clang-cl too; MSVC compiles any intrinsic as it is): the file is built for+// the baseline x86-64, and each tier's code runs only where detect() says the CPU and the OS take it+#if defined(MX_X86) && (defined(__GNUC__) || defined(__clang__))+#define MX_TARGET(s) __attribute__((target(s)))+#else+#define MX_TARGET(s)+#endif+#define MX_AVX2 MX_TARGET("avx2,fma")+// AVX-VNNI's intrinsics: GCC 11, Clang 12, MSVC 2022+#if defined(MX_X86) && (defined(_MSC_VER) || (defined(__clang__) && __clang_major__ >= 12) || \+ (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 11))+#define MX_AVXVNNI 1+#endif++namespace strata::ds4::gguf {+namespace {++constexpr int kBlock = 32, kBytes = 17;+constexpr float kValues[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f,+ 0.0f, -1.0f, -2.0f, -3.0f, -4.0f, -6.0f, -8.0f, -12.0f};+constexpr int kCodes[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};++float e8m0_half(uint32_t e) { // 2^(e - 128), as the device's+ const uint32_t bits = e < 2 ? 0x00200000u << e : (e - 1) << 23;+ float f;+ std::memcpy(&f, &bits, 4);+ return f;+}+struct E8m0Table {+ float v[256];+ E8m0Table() {+ for (int e = 0; e < 256; ++e) v[e] = e8m0_half((uint32_t) e);+ }+};+const E8m0Table& e8() {+ static const E8m0Table t;+ return t;+}++void rows_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {+ for (int r = r0; r < r1; ++r) {+ const uint8_t* p = w + (size_t) r * rb;+ float acc[kMxCpuMaxN] = {};+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {+ const float d = e8m0_half(p[0]);+ float wv[kBlock];+ for (int j = 0; j < 16; ++j) {+ wv[j] = kValues[p[1 + j] & 15] * d;+ wv[j + 16] = kValues[p[1 + j] >> 4] * d;+ }+ for (int i = 0; i < n; ++i) {+ const float* xb = x[i] + b * kBlock;+ float s = 0.0f;+ for (int j = 0; j < kBlock; ++j) s += wv[j] * xb[j];+ acc[i] += s;+ }+ }+ for (int i = 0; i < n; ++i) y[i][r - r0] = acc[i];+ }+}++#if defined(MX_X86)+MX_AVX2 float hsum(__m256 v) {+ const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));+ const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));+ return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));+}++// a block: the 16 code bytes -> 32 int8 values through a shuffle table (low nibbles: values 0..15, high: 16..31),+// widened to F32 in 4 x 8, times the block's scale (exact), FMAs against each activation+MX_AVX2 void rows_avx2(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n,+ float* const* y) {+ const __m128i lut = _mm_setr_epi8(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12);+ const __m128i m4 = _mm_set1_epi8(0x0f);+ for (int r = r0; r < r1; ++r) {+ const uint8_t* p = w + (size_t) r * rb;+ __m256 acc[kMxCpuMaxN];+ for (int i = 0; i < n; ++i) acc[i] = _mm256_setzero_ps();+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {+ const __m256 d = _mm256_set1_ps(e8m0_half(p[0]));+ const __m128i q = _mm_loadu_si128((const __m128i*) (p + 1));+ const __m128i lo = _mm_shuffle_epi8(lut, _mm_and_si128(q, m4));+ const __m128i hi = _mm_shuffle_epi8(lut, _mm_and_si128(_mm_srli_epi16(q, 4), m4));+ const __m256 w0 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo)));+ const __m256 w1 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo, 8))));+ const __m256 w2 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi)));+ const __m256 w3 = _mm256_mul_ps(d, _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi, 8))));+ for (int i = 0; i < n; ++i) {+ const float* xb = x[i] + b * kBlock;+ __m256 a = acc[i];+ a = _mm256_fmadd_ps(w0, _mm256_loadu_ps(xb), a);+ a = _mm256_fmadd_ps(w1, _mm256_loadu_ps(xb + 8), a);+ a = _mm256_fmadd_ps(w2, _mm256_loadu_ps(xb + 16), a);+ a = _mm256_fmadd_ps(w3, _mm256_loadu_ps(xb + 24), a);+ acc[i] = a;+ }+ }+ for (int i = 0; i < n; ++i) y[i][r - r0] = hsum(acc[i]);+ }+}++// ---- the int8 path's vector tiers (ds4_mx_kern.inc)+#define MXK_NS mx_avx2+#define MXK_ATTR MX_TARGET("avx2,fma")+#define MXK_256 1+#include "ds4_mx_kern.inc"+#if defined(MX_AVXVNNI)+#define MXK_NS mx_avxvnni+#define MXK_ATTR MX_TARGET("avx2,fma,avxvnni")+#define MXK_256 1+#define MXK_VNNI 1+#include "ds4_mx_kern.inc"+#endif+#define MXK_NS mx_avx512+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,fma")+#define MXK_512 1+#include "ds4_mx_kern.inc"+#define MXK_NS mx_avx512vnni+#define MXK_ATTR MX_TARGET("avx512f,avx512bw,avx512vnni,fma")+#define MXK_512 1+#define MXK_VNNI 1+#include "ds4_mx_kern.inc"+#endif++// ---- the int8 path's scalar tier: ds4_mx_kern.inc's order (8 accumulators, block j of a group of 8 into L(j)), so it+// gives the vector tiers' bits - also the tests' check on them. The activation's layout (ds4_mx_kern.inc), for any k+// % 32 == 0: the int8 values in quads (a partial last quad padded), then per group of 8 blocks the scales and the+// offset corrections in lane order (a partial last group padded)+inline int act_lane(int b) { return 8 * (b >> 3) + ((b & 7) & 1) * 4 + ((b & 7) >> 1); }+inline size_t act_qbytes(int k) { return (size_t) 128 * (size_t) ((k / kBlock + 3) / 4); }+inline size_t act_groups(int k) { return (size_t) ((k / kBlock + 7) / 8); }+inline int act_q(const uint8_t* act, int b, int j) { // block b's value j (0..31)+ return ((const int8_t*) act)[(b >> 2) * 128 + (j >= 16 ? 64 : 0) + 16 * (b & 3) + (j & 15)];+}+inline float act_d(const uint8_t* act, int k, int b) { return ((const float*) (act + act_qbytes(k)))[act_lane(b)]; }++void quant_scalar(const float* x, int b0, int b1, uint8_t* act, int k) {+ int8_t* q = (int8_t*) act;+ float* dx = (float*) (act + act_qbytes(k));+ int32_t* corr = (int32_t*) (act + act_qbytes(k) + 32 * act_groups(k));+ for (int b = b0; b < b1; ++b, x += kBlock) {+ float am = 0.0f;+ for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(x[j]));+ const float sc = am / 127.0f;+ int8_t v[kBlock];+ int s = 0;+ for (int j = 0; j < kBlock; ++j) {+ v[j] = am > 0.0f ? (int8_t) (int) std::nearbyint(x[j] / sc) : (int8_t) 0;+ s += v[j];+ }+ int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);+ for (int j = 0; j < 16; ++j) {+ lo[j] = v[j];+ lo[64 + j] = v[16 + j];+ }+ dx[act_lane(b)] = sc;+ corr[act_lane(b)] = -12 * s;+ }+}++void rows_q_scalar(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {+ const float* t = e8().v;+ for (int r = r0; r < r1; ++r) {+ const uint8_t* p = w + (size_t) r * rb;+ float acc[kMxCpuMaxN][8] = {};+ for (int b = 0; b < k / kBlock; ++b, p += kBytes) {+ int wv[kBlock];+ for (int j = 0; j < 16; ++j) {+ wv[j] = kCodes[p[1 + j] & 15];+ wv[j + 16] = kCodes[p[1 + j] >> 4];+ }+ const int l = act_lane(b) & 7;+ for (int i = 0; i < n; ++i) {+ const uint8_t* a = (const uint8_t*) act[i];+ int s = 0;+ for (int j = 0; j < kBlock; ++j) s += wv[j] * act_q(a, b, j);+ acc[i][l] = std::fma((float) s, t[p[0]] * act_d(a, k, b), acc[i][l]);+ }+ }+ for (int i = 0; i < n; ++i) {+ const float* A = acc[i];+ y[i][r - r0] = ((A[0] + A[4]) + (A[2] + A[6])) + ((A[1] + A[5]) + (A[3] + A[7]));+ }+ }+}++// ---- which tiers this CPU runs+struct Cpu {+ bool avx2 = false, vnni256 = false, avx512 = false, vnni512 = false, vbmi = false, bmi2 = false;+};+Cpu detect() {+ Cpu c;+#if defined(MX_X86)+ unsigned r[4] = {0, 0, 0, 0};+ auto cpuid = [&](unsigned leaf, unsigned sub) {+#if defined(_MSC_VER)+ int x[4];+ __cpuidex(x, (int) leaf, (int) sub);+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];+#else+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);+#endif+ };+ cpuid(0, 0);+ const unsigned max_leaf = r[0];+ if (max_leaf < 7) return c;+ cpuid(1, 0);+ const bool fma = (r[2] >> 12) & 1u, osxsave = (r[2] >> 27) & 1u, avx = (r[2] >> 28) & 1u;+ if (!fma || !osxsave || !avx) return c;+#if defined(_MSC_VER)+ const unsigned long long xcr0 = _xgetbv(0);+#else+ unsigned lo = 0, hi = 0;+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;+#endif+ if ((xcr0 & 0x6) != 0x6) return c; // the OS saves the YMM state+ cpuid(7, 0);+ const unsigned b7 = r[1], c7 = r[2];+ c.avx2 = (b7 >> 5) & 1u;+ const bool zmm_os = (xcr0 & 0xE6) == 0xE6;+ c.avx512 = c.avx2 && zmm_os && ((b7 >> 16) & 1u) && ((b7 >> 30) & 1u); // F, BW+ c.vnni512 = c.avx512 && ((c7 >> 11) & 1u);+ c.vbmi = c.avx512 && ((c7 >> 1) & 1u); // AVX-512 VBMI+ c.bmi2 = (b7 >> 8) & 1u;+ cpuid(7, 1);+ c.vnni256 = c.avx2 && ((r[0] >> 4) & 1u); // AVX-VNNI+#endif+ return c;+}+const Cpu& cpu() {+ static const Cpu c = detect();+ return c;+}++// The tier for this CPU, measured: every vector tier it runs times the same small warm job (64 rows of a 5120-wide+// matrix, in L2, against 1, 2 and 4 activations - the compute a row takes, which is where the tiers differ; streaming+// from RAM they all reach the memory's speed) in interleaved rounds, the best round of each counting. The choice never+// changes an answer (the tiers give the same bits), only the speed. DS4_MX_TIER=<name> picks one by hand.+std::string g_tier_report;+int measure_tier(const std::vector<int>& cand);+int pick_tier() {+ if (const char* f = std::getenv("DS4_MX_TIER"))+ for (int t = 0; t < kMxTiers; ++t)+ if (std::string(f) == mx_tier_name(t) && mx_tier_ok(t)) {+ g_tier_report = std::string("DS4_MX_TIER=") + f;+ return t;+ }+ std::vector<int> cand;+ for (int t = 1; t < kMxTiers; ++t)+ if (mx_tier_ok(t)) cand.push_back(t);+ if (cand.empty()) return kMxScalar;+ if (cand.size() == 1) return cand[0];+ return measure_tier(cand);+}+int measure_tier(const std::vector<int>& cand) {+ constexpr int K = 5120, R = 64, NT = 4;+ const size_t rb = (size_t) K / kBlock * kBytes;+ std::vector<uint8_t> w(rb * R), act(mx_act_bytes(K) * NT);+ uint32_t s = 0x9E3779B9u;+ for (size_t i = 0; i < w.size(); ++i) {+ s = s * 1664525u + 1013904223u;+ w[i] = (uint8_t) (s >> 24);+ }+ for (size_t b = 0; b < w.size() / kBytes; ++b) w[b * kBytes] = (uint8_t) (118 + b % 6);+ std::vector<float> x(K);+ for (int i = 0; i < NT; ++i) {+ for (int j = 0; j < K; ++j) {+ s = s * 1664525u + 1013904223u;+ x[(size_t) j] = (float) ((int) (s >> 16) - 32768) / 32768.0f;+ }+ quant_scalar(x.data(), 0, K / kBlock, act.data() + (size_t) i * mx_act_bytes(K), K);+ }+ const void* ap[NT];+ for (int i = 0; i < NT; ++i) ap[i] = act.data() + (size_t) i * mx_act_bytes(K);+ float y[NT][R];+ float* yp[NT] = {y[0], y[1], y[2], y[3]};+ std::vector<double> best(cand.size(), 1e30);+ for (int round = 0; round < 6; ++round)+ for (size_t c = 0; c < cand.size(); ++c) {+ const auto t0 = std::chrono::steady_clock::now();+ for (int rep = 0; rep < 8; ++rep)+ for (int n : {1, 2, 4}) mx_rows_tier(cand[c], w.data(), rb, K, 0, R, ap, n, yp, 0);+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();+ if (round > 0) best[c] = std::min(best[c], us); // (round 0: the clock and the caches warm)+ }+ size_t bi = 0;+ for (size_t c = 1; c < cand.size(); ++c)+ if (best[c] < best[bi]) bi = c;+ char buf[64];+ for (size_t c = 0; c < cand.size(); ++c) {+ std::snprintf(buf, sizeof buf, "%s%s %.0f us", c ? ", " : "", mx_tier_name(cand[c]), best[c]);+ g_tier_report += buf;+ }+ return cand[bi];+}+int prefetch_bytes() {+ const char* f = std::getenv("DS4_MX_PF");+ return f != nullptr ? std::max(0, std::atoi(f)) : 0;+}++} // namespace++void q8_round(const float* x, int k, float* out) {+ for (int b = 0; b < k / kBlock; ++b) {+ const float* xb = x + b * kBlock;+ float am = 0.0f;+ for (int j = 0; j < kBlock; ++j) am = std::max(am, std::fabs(xb[j]));+ const float sc = am / 127.0f;+ for (int j = 0; j < kBlock; ++j)+ out[b * kBlock + j] = am > 0.0f ? std::nearbyint(xb[j] / sc) * sc : 0.0f;+ }+}++void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y) {+#if defined(MX_X86)+ static const bool avx2 = strata::kernels::cpu::exl3_cpu_tier() >= 1;+ if (avx2) return rows_avx2(w, rb, k, r0, r1, x, n, y);+#endif+ rows_scalar(w, rb, k, r0, r1, x, n, y);+}++// ---- the int8 path+bool mx_tier_ok(int t) {+ switch (t) {+ case kMxScalar: return true;+ case kMxAvx2: return cpu().avx2;+#if defined(MX_AVXVNNI)+ case kMxAvxVnni: return cpu().vnni256;+#endif+ case kMxAvx512: return cpu().avx512;+ case kMxAvx512Vnni: return cpu().vnni512;+ default: return false;+ }+}+const char* mx_tier_name(int t) {+ switch (t) {+ case kMxScalar: return "scalar";+ case kMxAvx2: return "avx2";+ case kMxAvxVnni: return "avxvnni";+ case kMxAvx512: return "avx512";+ case kMxAvx512Vnni: return "avx512vnni";+ default: return "?";+ }+}+int mx_tier() {+ static const int t = pick_tier();+ return t;+}+const char* mx_tier_report() {+ (void) mx_tier();+ return g_tier_report.c_str();+}++void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k) {+#if defined(MX_X86)+ if (cpu().avx2 && k % 256 == 0) return mx_avx2::quant(x, b0, b1, (uint8_t*) act, k);+#endif+ quant_scalar(x, b0, b1, (uint8_t*) act, k);+}+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k) {+ quant_scalar(x, b0, b1, (uint8_t*) act, k);+}+int mx_act_value(const void* act, int b, int j) { return act_q((const uint8_t*) act, b, j); }+float mx_act_scale(const void* act, int k, int b) { return act_d((const uint8_t*) act, k, b); }++void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,+ float* const* y, int pf) {+ if (k % 256 != 0 || !mx_tier_ok(t)) t = kMxScalar; // (the vector tiers take a group of 8 blocks a step)+ const float* tb = e8().v;+#if defined(MX_X86)+ switch (t) {+ case kMxAvx2: return mx_avx2::rows(w, rb, k, r0, r1, act, n, y, tb, pf);+#if defined(MX_AVXVNNI)+ case kMxAvxVnni: return mx_avxvnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);+#endif+ case kMxAvx512: return mx_avx512::rows(w, rb, k, r0, r1, act, n, y, tb, pf);+ case kMxAvx512Vnni: return mx_avx512vnni::rows(w, rb, k, r0, r1, act, n, y, tb, pf);+ default: break;+ }+#endif+ (void) tb, (void) pf;+ for (int i0 = 0; i0 < n; i0 += kMxCpuMaxN)+ rows_q_scalar(w, rb, k, r0, r1, act + i0, std::min(kMxCpuMaxN, n - i0), y + i0);+}++void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y) {+ static const int pf = prefetch_bytes();+ mx_rows_tier(mx_tier(), w, rb, k, r0, r1, act, n, y, pf);+}++} // namespace strata::ds4::ggufdiff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hppnew file mode 100644index 000000000..170c0f499--- /dev/null+++ b/src/dsv41/ds4_gguf_cpu.hpp@@ -0,0 +1,75 @@+// src/ds4/ds4_gguf_cpu.hpp - a GGUF's MXFP4 experts on the CPU lane of maya-ds4 (the format: ds4_gguf.cuh). The+// weights are read where they lie (the GGUF file's mapping), each block decoded once for all the activations of a+// call (mx_rows: int8 dots, the CPU's best vector tier at run time; a scalar loop otherwise).+#pragma once++#include <cstddef>+#include <cstdint>+#include <string>++namespace strata::ds4::gguf {++constexpr int kMxCpuMaxN = 8; // activations a call++/// y[i][r - r0] = (row r of W) . x[i] for r0 <= r < r1 and i < n (n <= kMxCpuMaxN): W's rows hold k values+/// (k % 32 == 0) as MXFP4 blocks, rb bytes apart. (F32 activations: the tests' and the benchmark's old path.)+void mx_cpu_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const float* const* x, int n, float* const* y);++/// A GGUF's low-bit experts (ds4_iq.cuh's types) on the CPU lane - ggml-cpu's arithmetic through the GLM CPU lane's+/// native experts (strata::kernels::cpu::native_*: its AVX-512 / AVX2 multi-token rows where they are ggml's bit for bit,+/// ggml's dots otherwise; ds4_iq_cpu.cpp): an activation quantized once to the weight type's vec_dot_type (Q8_K: an F32+/// scale per 256 values - the device takes one per 32, so the two sides agree to that rounding), each row one dot.+struct IqCpu {+ int tg = 0, td = 0, n_embd = 0, n_ff = 0;+ int gu_act = 0, d_act = 0; // the activations' ggml types+ size_t gu_row = 0, d_row = 0; // bytes a weight row+ size_t act_bytes = 0, h_bytes = 0; // bytes of x (n_embd) and of h (n_ff) as those+};+/// The layer's geometry; false (err) when this build or ggml-cpu cannot take it.+bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);+/// x (k values) as the activation type `act` (f.gu_act / f.d_act) into dst.+void iq_cpu_quant(int act, const float* x, int k, void* dst);+/// ff[t][r] = clamped swiglu(gate_r . a[t], up_r . a[t]) for rows [r0, r1) (limit 0: no clamp), nt tokens.+void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,+ float* const* ff, int r0, int r1, float limit);+/// out[t][r] = down_r . hq[t] for rows [r0, r1).+void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);+/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32+/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then+/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of the+/// blocks' sums. The vector tiers (ds4_mx_kern.inc): AVX2 vpmaddubsw, AVX-VNNI and AVX-512 (VNNI) vpdpbusd; the CPU's+/// fastest is taken at run time (DS4_MX_TIER=<name> picks another one it runs). Every tier, the scalar one included,+/// sums in the same order: they give the same bits, so the choice never changes an answer.+enum MxTier { kMxScalar = 0, kMxAvx2, kMxAvxVnni, kMxAvx512, kMxAvx512Vnni, kMxTiers };+/// Bytes of an activation of k values (k % 32 == 0) for mx_rows: the int8 values in quads of blocks, then a scale and+/// an offset correction a block in groups of 8 (ds4_mx_kern.inc) - 1.25 k for k % 256 == 0.+inline size_t mx_act_bytes(int k) {+ const size_t nb = (size_t) k / 32;+ return 128 * ((nb + 3) / 4) + 64 * ((nb + 7) / 8);+}+/// (tests) value j (0..31) of block b of an activation, and the block's scale+int mx_act_value(const void* act, int b, int j);+float mx_act_scale(const void* act, int k, int b);+/// Blocks [b0, b1) of a k-value activation (x at block b0's first value: x[32 (b - b0) + j]) into act.+void mx_quant_blocks(const float* x, int b0, int b1, void* act, int k);+void mx_quant_blocks_scalar(const float* x, int b0, int b1, void* act, int k); // (the same bytes, no vectors)+inline void mx_quant(const float* x, int k, void* act) { mx_quant_blocks(x, 0, k / 32, act, k); }+/// y[i][r - r0] = (row r of W) . act[i] for r0 <= r < r1, i < n (any n; act[i] mx_quant's of k values).+void mx_rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n, float* const* y);+/// The same with tier t (the scalar one when the CPU does not run t, or k % 256 != 0); pf: bytes prefetched ahead.+void mx_rows_tier(int t, const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,+ float* const* y, int pf = 0);+int mx_tier(); // the tier mx_rows takes (measured on first use: the fastest a core, ds4_gguf_cpu.cpp)+const char* mx_tier_report(); // the measurement behind it (each candidate's time), or how it was picked+bool mx_tier_ok(int t); // this CPU runs tier t+const char* mx_tier_name(int t);+/// Whether the rows of `type` are ds4's own here (IQ3_XXS / IQ3_S on AVX2 / AVX-512 VBMI: ggml-cpu's numbers bit for+/// bit, checked against its dot on first use; DS4_IQ_OWN=0 turns them off) rather than ggml's / the GLM lane's.+bool iq_cpu_own_rows(int type);++/// x rounded as the device's MXFP4 kernels round their activations (q8_quant: per 32 values sc = max|x| / 127, q =+/// round-half-even(x / sc), the same F32 operations, so the same q), into out as q sc - an expert gives the same sums+/// on the CPU lane as in VRAM (up to their order), whichever side the expert cache puts it on.+void q8_round(const float* x, int k, float* out);++} // namespace strata::ds4::ggufdiff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cunew file mode 100644index 000000000..22ff70b05--- /dev/null+++ b/src/dsv41/ds4_iq.cu@@ -0,0 +1,1018 @@+// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are+// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.+#include "ds4_iq.cuh"++#include <cuda_fp16.h>+#if !defined(STRATA_USE_HIP)+#include <mma.h>+#endif++#define GGML_COMMON_DECL_CUDA+#define GGML_COMMON_IMPL_CUDA+#include "ggml-common.h"++#include <algorithm>+#include <cstdio>+#include <cstdlib>+#include <mutex>+#include <unordered_map>++namespace strata::ds4::gguf::iq {+namespace {++constexpr int kWarps = 8, kThreads = kWarps * 32;++// ---------------------------------------------------------------- the types+template <int T>+struct Ty;+template <> struct Ty<kIQ2_XXS> { static constexpr int B = 66, TAB = 512, MIN = 0; };+template <> struct Ty<kIQ2_S> { static constexpr int B = 82, TAB = 2048, MIN = 0; };+template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN = 0; };+template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };+template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };+template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };+template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };+template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };+// MXFP4 (ds4_gguf.cu's own kernels everywhere else): a block of 32 values, an E8M0 scale then 16 code bytes; the light+// sets take it too+constexpr int kMXFP4 = 39;+template <> struct Ty<kMXFP4> { static constexpr int B = 17, TAB = 0, MIN = 0; };+template <int T> constexpr int kQK = T == kMXFP4 ? 32 : 256; // values a block+static_assert(sizeof(block_iq2_xxs) == 66 && sizeof(block_iq2_s) == 82 && sizeof(block_iq3_xxs) == 98 &&+ sizeof(block_iq3_s) == 110 && sizeof(block_iq4_xs) == 136 && sizeof(block_q4_K) == 144 &&+ sizeof(block_q2_K) == 84 && sizeof(block_q3_K) == 110,+ "ggml's block sizes");++template <int T>+__host__ __device__ __forceinline__ size_t row_bytes_t(int k) { return (size_t) (k / kQK<T>) * Ty<T>::B; }++// the type's codebook into shared memory (u32 words; IQ4_XS: its 16 values, from bytes - an int8 table's alignment)+template <int T>+__device__ __forceinline__ void load_tab(uint32_t* tab) {+ if constexpr (T == kIQ4_XS) {+ if (threadIdx.x < 4) {+ const uint8_t* kv = (const uint8_t*) kvalues_iq4nl + 4 * threadIdx.x;+ tab[threadIdx.x] = (uint32_t) kv[0] | (uint32_t) kv[1] << 8 | (uint32_t) kv[2] << 16 | (uint32_t) kv[3] << 24;+ }+ } else if constexpr (Ty<T>::TAB > 0) {+ const uint32_t* src = T == kIQ2_XXS ? (const uint32_t*) iq2xxs_grid+ : T == kIQ2_S ? (const uint32_t*) iq2s_grid+ : T == kIQ3_XXS ? (const uint32_t*) iq3xxs_grid+ : (const uint32_t*) iq3s_grid;+ for (int i = threadIdx.x; i < Ty<T>::TAB; i += blockDim.x) tab[i] = src[i];+ }+}++__device__ __forceinline__ uint32_t ld2(const uint8_t* p) { // 4 bytes from a 2-byte aligned address+ const uint16_t* q = (const uint16_t*) p;+ return (uint32_t) q[0] | ((uint32_t) q[1] << 16);+}+__device__ __forceinline__ uint32_t ld4(const uint8_t* p) { return *(const uint32_t*) p; }+__device__ __forceinline__ float h2f(const uint8_t* p) { return __half2float(__ushort_as_half(*(const uint16_t*) p)); }+// 7 sign bits (bits 0..6 of v) -> the 8 signs, bit 7 their parity (ggml's ksigns_iq2xs)+__device__ __forceinline__ uint32_t ksigns8(uint32_t v) {+ v &= 127u;+ return v | ((__popc(v) & 1u) << 7);+}+// 4 grid bytes with the signs of bits 0..3 of b applied (bit i: byte i negative) - llama.cpp's HIP form, cheap where the+// SIMD byte instructions (__vcmpne4 / __vsub4) are emulated (Volta, RDNA): every grid byte is nonzero and <= 62, so+// (g ^ 0xff) + 1 = -g in each byte with no carry into the next+__device__ __forceinline__ int sgn4(uint32_t g, uint32_t b) {+ const uint32_t lo = (((b & 15u) * 0x10204080u) & 0x80808080u) >> 7;+ return (int) ((g ^ (lo * 255u)) + lo);+}+__device__ __forceinline__ int2 tab16(uint32_t q4, const uint32_t* t) { // vecdotq's get_int_from_table_16+ uint32_t tmp[2];+ const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const uint32_t sh = 16 * i;+ const uint32_t lo = __byte_perm(t[0], t[1], q4 >> sh), hi = __byte_perm(t[2], t[3], q4 >> sh);+ tmp[i] = __byte_perm(lo, hi, sel >> sh);+ }+ return make_int2((int) __byte_perm(tmp[0], tmp[1], 0x6420), (int) __byte_perm(tmp[0], tmp[1], 0x7531));+}+__device__ __forceinline__ int q3k_scale(const uint8_t* sc, int is) {+ const int lo = is < 8 ? sc[is] & 15 : sc[is - 8] >> 4;+ const int hi = (sc[8 + (is & 3)] >> (2 * (is >> 2))) & 3;+ return (lo | (hi << 4)) - 32;+}+__device__ __forceinline__ void q4k_scale(const uint8_t* q, int j, int& d, int& m) { // ggml's get_scale_min_k4+ if (j < 4) {+ d = q[j] & 63;+ m = q[j + 4] & 63;+ } else {+ d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);+ m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4);+ }+}++// a 32-value sub-block (kb: its index in the row) as 32 int8 codes and its scales: weight i = d s q_i - m, s and m of+// its half (i < 16: s0, m0; else s1, m1)+struct Sub {+ int q[8];+ float d, m0, m1;+ int s0, s1;+};+template <int T>+__device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb, const uint32_t* tab, Sub& s) {+ const int ib = kb & 7;+ const uint8_t* b = row + (size_t) (kQK<T> == 32 ? kb : kb >> 3) * Ty<T>::B;+ if constexpr (T == kMXFP4) { // codes: value j in byte j's low nibble, j + 16 in its high one (any alignment)+ const uint32_t t[4] = {0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu}; // ggml's kvalues_mxfp4+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const uint8_t* q = b + 1 + 4 * i;+ const int2 v = tab16((uint32_t) q[0] | (uint32_t) q[1] << 8 | (uint32_t) q[2] << 16 | (uint32_t) q[3] << 24, t);+ s.q[i] = v.x;+ s.q[4 + i] = v.y;+ }+ const uint32_t e = b[0]; // 2^(e - 128): ggml's GGML_E8M0_TO_FP32_HALF (the codes are the E2M1 values doubled)+ s.d = __uint_as_float(e < 2 ? 0x00200000u << e : (e - 1) << 23);+ s.s0 = s.s1 = 1;+ (void) ib;+ (void) tab;+ } else if constexpr (T == kIQ2_XXS) {+ const uint8_t* q = b + 2 + 8 * ib;+ const uint32_t a0 = ld2(q), a1 = ld2(q + 4);+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint2 g = ((const uint2*) tab)[(a0 >> (8 * l)) & 255u];+ const uint32_t sg = ksigns8(a1 >> (7 * l));+ s.q[2 * l] = sgn4(g.x, sg);+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);+ }+ s.s0 = s.s1 = 2 * (int) (a1 >> 28) + 1;+ s.d = h2f(b) * 0.125f;+ } else if constexpr (T == kIQ2_S) {+ const uint8_t* qs = b + 2;+ const uint32_t i4 = ld2(qs + 4 * ib), sg4 = ld2(qs + 32 + 4 * ib), h = b[66 + ib], sc = b[74 + ib];+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint2 g = ((const uint2*) tab)[((i4 >> (8 * l)) & 255u) | ((h << (8 - 2 * l)) & 0x300u)];+ const uint32_t sg = sg4 >> (8 * l);+ s.q[2 * l] = sgn4(g.x, sg);+ s.q[2 * l + 1] = sgn4(g.y, sg >> 4);+ }+ s.s0 = 2 * (int) (sc & 15) + 1;+ s.s1 = 2 * (int) (sc >> 4) + 1;+ s.d = h2f(b) * 0.125f;+ } else if constexpr (T == kIQ3_XXS) {+ const uint8_t* qs = b + 2;+ const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), aux = ld2(qs + 64 + 4 * ib);+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);+ const uint32_t sg = ksigns8(aux >> (7 * l));+ s.q[2 * l] = sgn4(tab[(w >> sh) & 255u], sg);+ s.q[2 * l + 1] = sgn4(tab[(w >> (sh + 8)) & 255u], sg >> 4);+ }+ s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;+ s.d = h2f(b) * 0.25f;+ } else if constexpr (T == kIQ3_S) {+ const uint8_t* qs = b + 2;+ const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);+#pragma unroll+ for (int l = 0; l < 4; ++l) {+ const uint32_t w = l < 2 ? w0 : w1, sh = 16 * (l & 1);+ const uint32_t i1 = ((w >> sh) & 255u) | ((h << (8 - 2 * l)) & 256u);+ const uint32_t i2 = ((w >> (sh + 8)) & 255u) | ((h << (7 - 2 * l)) & 256u);+ const uint32_t sg = s4 >> (8 * l);+ s.q[2 * l] = sgn4(tab[i1], sg);+ s.q[2 * l + 1] = sgn4(tab[i2], sg >> 4);+ }+ s.s0 = s.s1 = 1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15);+ s.d = h2f(b);+ } else if constexpr (T == kIQ4_XS) {+ const uint32_t sh = *(const uint16_t*) (b + 2);+ const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);+ const uint8_t* qs = b + 8 + 16 * ib;+#pragma unroll+ for (int t = 0; t < 4; ++t) {+ const int2 v = tab16(ld4(qs + 4 * t), tab);+ s.q[t] = v.x;+ s.q[4 + t] = v.y;+ }+ s.s0 = s.s1 = ls - 32;+ s.d = h2f(b);+ } else if constexpr (T == kQ4_K) {+ int sc, m;+ q4k_scale(b + 4, ib, sc, m);+ const uint8_t* qs = b + 16 + 32 * (ib >> 1);+ const int sh = 4 * (ib & 1);+#pragma unroll+ for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu);+ s.s0 = s.s1 = sc;+ s.d = h2f(b);+ s.m0 = s.m1 = h2f(b + 2) * (float) m;+ } else if constexpr (T == kQ2_K) {+ const int n = ib >> 2, j = ib & 3;+ const uint8_t* qs = b + 16 + 32 * n;+#pragma unroll+ for (int t = 0; t < 8; ++t) s.q[t] = (int) ((ld4(qs + 4 * t) >> (2 * j)) & 0x03030303u);+ const int c0 = b[8 * n + 2 * j], c1 = b[8 * n + 2 * j + 1];+ const float dmin = h2f(b + 82);+ s.s0 = c0 & 15;+ s.s1 = c1 & 15;+ s.d = h2f(b + 80);+ s.m0 = dmin * (float) (c0 >> 4);+ s.m1 = dmin * (float) (c1 >> 4);+ } else { // Q3_K+ const int n = ib >> 2, j = ib & 3;+ const uint8_t* qs = b + 32 + 32 * n;+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ const uint32_t ql = (ld2(qs + 4 * t) >> (2 * j)) & 0x03030303u;+ const uint32_t neg = ((~ld2(b + 4 * t)) >> (4 * n + j)) & 0x01010101u;+ s.q[t] = (int) (ql | (neg * 0xFCu)); // (q - 4 in a byte: q + 0xFC, no carry for q <= 3)+ }+ s.s0 = q3k_scale(b + 96, 8 * n + 2 * j);+ s.s1 = q3k_scale(b + 96, 8 * n + 2 * j + 1);+ s.d = h2f(b + 108);+ }+}+// the sub-block against 32 int8 activations (xa: values 0..15, xb: 16..31), before their scale: an exact integer dot,+// its scales in F32 (explicit roundings: every instantiation the same bits)+template <int T>+__device__ __forceinline__ float sub_dot(const Sub& s, const int4& xa, const int4& xb) {+ int lo = __dp4a(s.q[0], xa.x, 0);+ lo = __dp4a(s.q[1], xa.y, lo);+ lo = __dp4a(s.q[2], xa.z, lo);+ lo = __dp4a(s.q[3], xa.w, lo);+ int hi = __dp4a(s.q[4], xb.x, 0);+ hi = __dp4a(s.q[5], xb.y, hi);+ hi = __dp4a(s.q[6], xb.z, hi);+ hi = __dp4a(s.q[7], xb.w, hi);+ float v = __fmul_rn(s.d, (float) (s.s0 * lo + s.s1 * hi));+ if constexpr (Ty<T>::MIN) {+ int sl = __dp4a(0x01010101, xa.x, 0);+ sl = __dp4a(0x01010101, xa.y, sl);+ sl = __dp4a(0x01010101, xa.z, sl);+ sl = __dp4a(0x01010101, xa.w, sl);+ int sh = __dp4a(0x01010101, xb.x, 0);+ sh = __dp4a(0x01010101, xb.y, sh);+ sh = __dp4a(0x01010101, xb.z, sh);+ sh = __dp4a(0x01010101, xb.w, sh);+ v = __fsub_rn(v, __fadd_rn(__fmul_rn(s.m0, (float) sl), __fmul_rn(s.m1, (float) sh)));+ }+ return v;+}++// values [8 p, 8 p + 8) of sub-block kb, as F32 (the prompt sets' tiles): ggml's dequantize_row_* values+template <int T>+__device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, int p, const uint32_t* tab, float* w) {+ const int ib = kb & 7;+ const uint8_t* b = row + (size_t) (kb >> 3) * Ty<T>::B;+ if constexpr (T == kIQ2_XXS) {+ const uint8_t* q = b + 2 + 8 * ib;+ const uint32_t a1 = ld2(q + 4);+ const uint2 g = ((const uint2*) tab)[q[p]];+ const uint32_t v = (a1 >> (7 * p)) & 127u, sg = v | ((__popc(v) & 1u) << 7);+ const float dl = h2f(b) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;+ }+ } else if constexpr (T == kIQ2_S) {+ const uint8_t* qs = b + 2;+ const uint32_t idx = qs[4 * ib + p] | ((((uint32_t) b[66 + ib]) << (8 - 2 * p)) & 0x300u);+ const uint32_t sg = qs[32 + 4 * ib + p], sc = b[74 + ib];+ const uint2 g = ((const uint2*) tab)[idx];+ const float dl = h2f(b) * (float) (2 * (int) (p < 2 ? sc & 15 : sc >> 4) + 1) * 0.125f;+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;+ }+ } else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {+ const uint8_t* qs = b + 2;+ uint32_t g1, g2, sg;+ float dl;+ if constexpr (T == kIQ3_XXS) {+ const uint32_t aux = ld2(qs + 64 + 4 * ib), v = (aux >> (7 * p)) & 127u;+ g1 = tab[qs[8 * ib + 2 * p]];+ g2 = tab[qs[8 * ib + 2 * p + 1]];+ sg = v | ((__popc(v) & 1u) << 7);+ dl = h2f(b) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;+ } else {+ const uint32_t h = b[66 + ib];+ g1 = tab[qs[8 * ib + 2 * p] | ((h << (8 - 2 * p)) & 256u)];+ g2 = tab[qs[8 * ib + 2 * p + 1] | ((h << (7 - 2 * p)) & 256u)];+ sg = b[74 + 4 * ib + p];+ dl = h2f(b) * (float) (1 + 2 * (int) ((b[106 + (ib >> 1)] >> (4 * (ib & 1))) & 15));+ }+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const float gv = (float) ((j < 4 ? g1 >> (8 * j) : g2 >> (8 * (j - 4))) & 255u);+ w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;+ }+ } else if constexpr (T == kIQ4_XS) {+ const uint32_t sh = *(const uint16_t*) (b + 2);+ const int ls = ((b[4 + (ib >> 1)] >> (4 * (ib & 1))) & 15) | (((sh >> (2 * ib)) & 3) << 4);+ const float dl = h2f(b) * (float) (ls - 32);+ const uint8_t* qs = b + 8 + 16 * ib + 8 * (p & 1);+ const int8_t* kv = (const int8_t*) tab;+#pragma unroll+ for (int j = 0; j < 8; ++j) w[j] = dl * (float) kv[p < 2 ? qs[j] & 15 : qs[j] >> 4];+ } else if constexpr (T == kQ4_K) {+ int sc, m;+ q4k_scale(b + 4, ib, sc, m);+ const float d1 = h2f(b) * (float) sc, m1 = h2f(b + 2) * (float) m;+ const uint8_t* qs = b + 16 + 32 * (ib >> 1) + 8 * p;+#pragma unroll+ for (int j = 0; j < 8; ++j) w[j] = d1 * (float) ((ib & 1) ? qs[j] >> 4 : qs[j] & 15) - m1;+ } else if constexpr (T == kQ2_K) {+ const int n = ib >> 2, jj = ib & 3;+ const int c = b[8 * n + 2 * jj + (p >> 1)];+ const float dl = h2f(b + 80) * (float) (c & 15), ml = h2f(b + 82) * (float) (c >> 4);+ const uint8_t* qs = b + 16 + 32 * n + 8 * p;+#pragma unroll+ for (int j = 0; j < 8; ++j) w[j] = dl * (float) ((qs[j] >> (2 * jj)) & 3) - ml;+ } else { // Q3_K+ const int n = ib >> 2, jj = ib & 3;+ const float dl = h2f(b + 108) * (float) q3k_scale(b + 96, 8 * n + 2 * jj + (p >> 1));+ const uint8_t* qs = b + 32 + 32 * n + 8 * p;+ const uint8_t* hm = b + 8 * p;+#pragma unroll+ for (int j = 0; j < 8; ++j)+ w[j] = dl * (float) ((int) ((qs[j] >> (2 * jj)) & 3) - (((hm[j] >> (4 * n + jj)) & 1) ? 0 : 4));+ }+}++__device__ __forceinline__ float warp_sum(float v) {+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);+ return v;+}+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) { // (ds4_gguf.cu's)+ if (limit > 0.0f) {+ gv = fminf(gv, limit);+ uv = fminf(fmaxf(uv, -limit), limit);+ }+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);+}+__device__ __forceinline__ bool group(const Args& a, int grp, const uint8_t*& blob, int& m0, int& m1) {+ if (a.grp_ptr == nullptr) {+ const unsigned long long p = a.plan_ptr[grp];+ if (p == 0ull) return false;+ blob = (const uint8_t*) p;+ m0 = grp;+ m1 = grp + 1;+ return true;+ }+ if (grp >= *a.n_groups) return false;+ blob = (const uint8_t*) a.grp_ptr[grp];+ m0 = a.grp_start[grp];+ m1 = a.grp_start[grp + 1];+ return true;+}++// ---------------------------------------------------------------- token / window experts+// RW rows a warp, 32 / RW lanes a row (a lane a sub-block a step), their sums over those lanes+template <int RW>+__device__ __forceinline__ float lanes_sum(float v) {+#pragma unroll+ for (int o = 16 / RW; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);+ return v;+}+constexpr int kGuRW = 1, kDnRW = 2; // rows a warp: gate / up (n_embd 5120: 10 sub-blocks a lane), down (2304: 9)++// grid (n_ff / (8 RW), groups): RW rows of the gate and of the up a warp, NE of the group's entries a pass (each+// sub-block decoded once for them) -> h = the clamped swiglu+template <int T, int NE, int RW = kGuRW>+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const Args a) {+ constexpr int LPR = 32 / RW;+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ int es[NE], xr[NE];+ const uint8_t* blob;+ int m0, m1;+ if (!group(a, blockIdx.y, blob, m0, m1)) return; // (uniform)+ load_tab<T>(tab);+ const int lane32 = threadIdx.x & 31;+ const int row = (blockIdx.x * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;+ const size_t rb = row_bytes_t<T>(a.n_embd);+ const uint8_t* g = blob + a.tr[0] + (size_t) row * rb;+ const uint8_t* u = blob + a.tr[1] + (size_t) row * rb;+ const int nsb = a.n_embd / 32;+ for (int mm = m0; mm < m1; mm += NE) {+ const int ne = min(NE, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) {+ const int e = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ es[threadIdx.x] = e;+ xr[threadIdx.x] = a.grp_ptr != nullptr ? e >> 3 : 0;+ }+ __syncthreads();+ if (row >= a.n_ff) continue; // (a whole warp: n_ff % (8 RW) == 0)+ float ag[NE], au[NE];+#pragma unroll+ for (int i = 0; i < NE; ++i) ag[i] = au[i] = 0.0f;+ for (int kb = lane; kb < nsb; kb += LPR) {+ Sub sg, su;+ decode<T>(g, kb, tab, sg);+ decode<T>(u, kb, tab, su);+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const int4* xp = (const int4*) (a.xq + (size_t) xr[i] * a.n_embd + (size_t) kb * 32);+ const int4 xa = xp[0], xb = xp[1];+ const float dx = a.xd[(size_t) xr[i] * nsb + kb];+ ag[i] = fmaf(dx, sub_dot<T>(sg, xa, xb), ag[i]);+ au[i] = fmaf(dx, sub_dot<T>(su, xa, xb), au[i]);+ }+ }+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const float gv = lanes_sum<RW>(ag[i]), uv = lanes_sum<RW>(au[i]);+ if (lane == 0) a.h[(size_t) es[i] * a.n_ff + row] = swiglu(gv, uv, a.limit);+ }+ }+}++// grid (n_embd / (8 RW), groups): RW rows of the down a warp for NE entries a pass, against their int8 h -> part+template <int T, int NE, int RW = kDnRW>+__global__ void __launch_bounds__(kThreads) down_kernel(const Args a) {+ constexpr int LPR = 32 / RW;+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ int es[NE];+ const uint8_t* blob;+ int m0, m1;+ if (!group(a, blockIdx.y, blob, m0, m1)) return;+ load_tab<T>(tab);+ const int lane32 = threadIdx.x & 31;+ const int row = (blockIdx.x * kWarps + (threadIdx.x >> 5)) * RW + lane32 / LPR, lane = lane32 % LPR;+ const uint8_t* d = blob + a.tr[2] + (size_t) row * row_bytes_t<T>(a.n_ff);+ const int nsb = a.n_ff / 32;+ for (int mm = m0; mm < m1; mm += NE) {+ const int ne = min(NE, m1 - mm);+ __syncthreads();+ if ((int) threadIdx.x < ne) es[threadIdx.x] = a.grp_ptr != nullptr ? a.grp_ent[mm + threadIdx.x] : mm + threadIdx.x;+ __syncthreads();+ if (row >= a.n_embd) continue;+ float ad[NE];+#pragma unroll+ for (int i = 0; i < NE; ++i) ad[i] = 0.0f;+ for (int kb = lane; kb < nsb; kb += LPR) {+ Sub sd;+ decode<T>(d, kb, tab, sd);+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const int4* hp = (const int4*) (a.hq + (size_t) es[i] * a.n_ff + (size_t) kb * 32);+ const int4 ha = hp[0], hb = hp[1];+ ad[i] = fmaf(a.hd[(size_t) es[i] * nsb + kb], sub_dot<T>(sd, ha, hb), ad[i]);+ }+ }+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const float v = lanes_sum<RW>(ad[i]);+ if (lane == 0) a.part[(size_t) es[i] * a.n_embd + row] = v;+ }+ }+}++// ---------------------------------------------------------------- the prompt path's sets+constexpr int BM = 64, BN = 64, TM = 4, TN = 4;+static_assert((BM / TM) * (BN / TN) == kThreads, "one output tile a thread");++// F32 FMAs (any device): a block RM rows (64, or 16 for experts of a few rows: a short prompt's) x BN outputs, a 32-value+// sub-block of K a step, the weights dequantised into shared memory (each thread 8 values of a row), each thread RT x TN+// outputs. (deq8 is the tile loader a tensor-core form takes too.)+template <int T, bool GATE_UP, int RM = BM>+__global__ void __launch_bounds__(kThreads) set_kernel(const SetArgs a) {+ constexpr int RT = RM / (kThreads / (BN / TN));+ static_assert(RT >= 1 && RT * (kThreads / (BN / TN)) == RM, "the block's rows");+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ float wg[32][BN + 1];+ __shared__ float wu[GATE_UP ? 32 : 1][BN + 1];+ __shared__ float xs[RM][33];+ __shared__ int src[RM];+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * RM;+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ load_tab<T>(tab);+ const int nr = min(RM, hi - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GATE_UP ? a.n_embd : a.n_ff;+ const size_t rb = row_bytes_t<T>(K);+ const int tid = (int) threadIdx.x;+ if (tid < RM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;+ __syncthreads();+ const int ty = tid / (BN / TN), tx = tid % (BN / TN);+ const int tr = tid >> 2, tq = tid & 3;+ float ag[RT][TN], au[RT][TN];+#pragma unroll+ for (int i = 0; i < RT; ++i)+#pragma unroll+ for (int j = 0; j < TN; ++j) ag[i][j] = au[i][j] = 0.0f;+ for (int kb = 0; kb < K / 32; ++kb) {+ float w8[8];+ deq8<T>(blob + a.tr[GATE_UP ? 0 : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);+#pragma unroll+ for (int j = 0; j < 8; ++j) wg[tq * 8 + j][tr] = w8[j];+ if (GATE_UP) {+ deq8<T>(blob + a.tr[1] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);+#pragma unroll+ for (int j = 0; j < 8; ++j) wu[tq * 8 + j][tr] = w8[j];+ }+ for (int i = tid; i < RM * 32; i += kThreads) {+ const int r = i / 32, c = i % 32, sr = src[r];+ float v = 0.0f;+ if (sr >= 0) {+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + c;+ v = GATE_UP ? a.xf[at] : __half2float(__ushort_as_half(a.xh[at]));+ }+ xs[r][c] = v;+ }+ __syncthreads();+#pragma unroll 8+ for (int kk = 0; kk < 32; ++kk) {+ float xv[RT], gw[TN], uw[TN];+#pragma unroll+ for (int i = 0; i < RT; ++i) xv[i] = xs[ty * RT + i][kk];+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ gw[j] = wg[kk][tx * TN + j];+ if (GATE_UP) uw[j] = wu[kk][tx * TN + j];+ }+#pragma unroll+ for (int i = 0; i < RT; ++i)+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ ag[i][j] = fmaf(xv[i], gw[j], ag[i][j]);+ if (GATE_UP) au[i][j] = fmaf(xv[i], uw[j], au[i][j]);+ }+ }+ __syncthreads();+ }+#pragma unroll+ for (int i = 0; i < RT; ++i) {+ const int r = ty * RT + i;+ if (r >= nr) continue;+ const size_t lr = (size_t) (r0 + r);+#pragma unroll+ for (int j = 0; j < TN; ++j) {+ const int c = c0 + tx * TN + j;+ if (GATE_UP) a.h[lr * a.ldo + c] = __half_as_ushort(__float2half_rn(swiglu(ag[i][j], au[i][j], a.limit)));+ else a.y[lr * a.ldo + c] = ag[i][j];+ }+ }+}++#if !defined(STRATA_USE_HIP)+// the same on the tensor cores (sm_70+): 8 warps each 16 rows x 32 outputs, the tiles F16 in shared memory+namespace wm = nvcuda::wmma;+constexpr int kLdT = 40;+template <int T, bool GATE_UP>+__global__ void __launch_bounds__(kThreads) set_tc_kernel(const SetArgs a) {+ struct Tiles {+ __half x[BM][kLdT], wg[BN][kLdT], wu[GATE_UP ? BN : 1][kLdT];+ };+ struct Out {+ float g[BM][BN + 4], u[GATE_UP ? BM : 1][BN + 4];+ };+ __shared__ __align__(32) unsigned char smem[sizeof(Tiles) > sizeof(Out) ? sizeof(Tiles) : sizeof(Out)];+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ int src[BM];+ Tiles& Tl = *reinterpret_cast<Tiles*>(smem);+ Out& O = *reinterpret_cast<Out*>(smem);+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi || hi - lo <= a.skip_le) return; // (uniform; skip_le: the light kernels' experts)+ load_tab<T>(tab);+ const int nr = min(BM, hi - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GATE_UP ? a.n_embd : a.n_ff;+ const size_t rb = row_bytes_t<T>(K);+ const int tid = (int) threadIdx.x, warp = tid >> 5;+ if (tid < BM) src[tid] = tid < nr ? (GATE_UP ? a.idx[r0 + tid] : r0 + tid) : -1;+ __syncthreads();+ const int wm_ = warp >> 1, wn = (warp & 1) * 2;+ wm::fragment<wm::accumulator, 16, 16, 16, float> fg[2], fu[2];+ for (int j = 0; j < 2; ++j) {+ wm::fill_fragment(fg[j], 0.0f);+ wm::fill_fragment(fu[j], 0.0f);+ }+ const int tr = tid >> 2, tq = tid & 3;+ for (int kb = 0; kb < K / 32; ++kb) {+ for (int m = 0; m < (GATE_UP ? 2 : 1); ++m) {+ float w8[8];+ deq8<T>(blob + a.tr[GATE_UP ? m : 2] + (size_t) (c0 + tr) * rb, kb, tq, tab, w8);+ __half* dst = m == 0 ? &Tl.wg[tr][tq * 8] : &Tl.wu[tr][tq * 8];+#pragma unroll+ for (int j = 0; j < 8; ++j) dst[j] = __float2half_rn(w8[j]);+ }+ const int sr = src[tr];+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ __half v = __float2half_rn(0.0f);+ if (sr >= 0) {+ const size_t at = (size_t) sr * a.ldx + (size_t) kb * 32 + tq * 8 + j;+ v = GATE_UP ? __float2half_rn(a.xf[at]) : __ushort_as_half(a.xh[at]);+ }+ Tl.x[tr][tq * 8 + j] = v;+ }+ __syncthreads();+#pragma unroll+ for (int kk = 0; kk < 32; kk += 16) {+ wm::fragment<wm::matrix_a, 16, 16, 16, __half, wm::row_major> fa;+ wm::load_matrix_sync(fa, &Tl.x[wm_ * 16][kk], kLdT);+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ wm::fragment<wm::matrix_b, 16, 16, 16, __half, wm::col_major> fb;+ wm::load_matrix_sync(fb, &Tl.wg[(wn + j) * 16][kk], kLdT);+ wm::mma_sync(fg[j], fa, fb, fg[j]);+ if (GATE_UP) {+ wm::load_matrix_sync(fb, &Tl.wu[(wn + j) * 16][kk], kLdT);+ wm::mma_sync(fu[j], fa, fb, fu[j]);+ }+ }+ }+ __syncthreads();+ }+ for (int j = 0; j < 2; ++j) {+ wm::store_matrix_sync(&O.g[wm_ * 16][(wn + j) * 16], fg[j], BN + 4, wm::mem_row_major);+ if (GATE_UP) wm::store_matrix_sync(&O.u[wm_ * 16][(wn + j) * 16], fu[j], BN + 4, wm::mem_row_major);+ }+ __syncthreads();+ for (int i = tid; i < BM * BN; i += kThreads) {+ const int r = i / BN, c = i % BN;+ if (r >= nr) continue;+ const size_t lr = (size_t) (r0 + r);+ if (GATE_UP) a.h[lr * a.ldo + c0 + c] = __half_as_ushort(__float2half_rn(swiglu(O.g[r][c], O.u[r][c], a.limit)));+ else a.y[lr * a.ldo + c0 + c] = O.g[r][c];+ }+}+#endif++// ---------------------------------------------------------------- GEMVs+constexpr int kMaxJobs = 8;+struct Jobs {+ MvJob j[kMaxJobs];+};+template <int T, int NC>+__global__ void __launch_bounds__(kThreads) mv_kernel(const Jobs J) {+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ load_tab<T>(tab);+ __syncthreads();+ const MvJob& b = J.j[blockIdx.y];+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (row >= b.n) return;+ const uint8_t* w = b.w + (size_t) row * b.ld;+ const int nsb = b.k / 32;+ float acc[NC];+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (int kb = lane; kb < nsb; kb += 32) {+ Sub s;+ decode<T>(w, kb, tab, s);+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const int4* xp = (const int4*) (b.xq + (size_t) c * b.k + (size_t) kb * 32);+ acc[c] = fmaf(b.xd[(size_t) c * nsb + kb], sub_dot<T>(s, xp[0], xp[1]), acc[c]);+ }+ }+#pragma unroll+ for (int c = 0; c < NC; ++c)+ if (c < b.ncol) {+ const float v = warp_sum(acc[c]);+ if (lane == 0) b.y[c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+}++// ---------------------------------------------------------------- the prompt sets' light experts+// A set's experts of a few rows (<= kLightRows: a prompt of a few hundred tokens spreads ~10 over most of 384 experts)+// in the token kernels' form: each weight sub-block decoded once and dotted with every row's int8 input (rounded per 32+// values as the token path rounds them) - the weights read once at the memory's speed, where a 64-row GEMM tile did 64+// rows' work for them. Each expert by its own rows: the light kernels take the experts of <= kLightRows rows, the tiles+// the others (SetArgs::skip_le: they skip the light ones) - every expert's sums in one fixed order whatever the set.+// Expert e's row i is input row e kLightRows + i of the scratch, output (local) row bounds[e] - row0 + i.+constexpr int kLightRows = 16;+constexpr int kLightMaxExp = 512; // a set's experts at most (the scratch: kLightMaxExp kLightRows rows)++// grid (k / 32 / 8, n_exp kLightRows): the light experts' rows of k values (gate / up: F32 x[idx[lr]]; down: F16 h[lr],+// ldx apart) -> int8 q[e kLightRows + i][k], scales d[..][k / 32]+__global__ void __launch_bounds__(kThreads) light_q8_kernel(const float* __restrict__ xf, const uint16_t* __restrict__ xh,+ const int* __restrict__ idx, int ldx, int k,+ const int* __restrict__ bounds, int row0,+ int8_t* __restrict__ q, float* __restrict__ d) {+ const int e = blockIdx.y / kLightRows, i = blockIdx.y % kLightRows;+ const int rows = bounds[e + 1] - bounds[e];+ const int b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (i >= rows || rows > kLightRows || b * 32 >= k) return;+ const int lr = bounds[e] - row0 + i;+ const size_t at = (size_t) b * 32 + lane, qr = (size_t) blockIdx.y;+ const float v = xf != nullptr ? xf[(size_t) idx[lr] * ldx + at] : __half2float(__ushort_as_half(xh[(size_t) lr * ldx + at]));+ float am = fabsf(v);+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));+ const float sc = am / 127.0f; // (ds4_gguf.cu's q8_quant: the same operations)+ q[qr * k + at] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);+ if (lane == 0) d[qr * (k / 32) + b] = sc;+}++// grid (n_ff / 8 or n_embd / 8, n_exp): a light expert's rows, a warp a row of the weights (gate and up, or down), every+// row of the expert a pass -> h16 (the F16 swiglu) or y+template <int T, int NE, bool GATE_UP>+__global__ void __launch_bounds__(kThreads) light_kernel(const SetArgs a, const int8_t* __restrict__ xq,+ const float* __restrict__ xd) {+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ const int e = blockIdx.y;+ const int lo = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];+ if (ne <= 0 || ne > NE) return; // (uniform: the tiles take an expert of more rows)+ load_tab<T>(tab);+ __syncthreads();+ const int K = GATE_UP ? a.n_embd : a.n_ff, N = GATE_UP ? a.n_ff : a.n_embd, nsb = K / 32;+ const int row = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ if (row >= N) return;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const size_t rb = row_bytes_t<T>(K);+ const uint8_t* w0 = blob + a.tr[GATE_UP ? 0 : 2] + (size_t) row * rb;+ const uint8_t* w1 = blob + a.tr[1] + (size_t) row * rb;+ const size_t q0 = (size_t) e * kLightRows;+ float a0[NE], a1[NE];+#pragma unroll+ for (int i = 0; i < NE; ++i) a0[i] = a1[i] = 0.0f;+ for (int kb = lane; kb < nsb; kb += 32) {+ Sub s0, s1;+ decode<T>(w0, kb, tab, s0);+ if (GATE_UP) decode<T>(w1, kb, tab, s1);+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const int4* xp = (const int4*) (xq + (q0 + i) * K + (size_t) kb * 32);+ const int4 xa = xp[0], xb = xp[1];+ const float dx = xd[(q0 + i) * nsb + kb];+ a0[i] = fmaf(dx, sub_dot<T>(s0, xa, xb), a0[i]);+ if (GATE_UP) a1[i] = fmaf(dx, sub_dot<T>(s1, xa, xb), a1[i]);+ }+ }+#pragma unroll+ for (int i = 0; i < NE; ++i)+ if (i < ne) {+ const float v0 = warp_sum(a0[i]);+ const float v1 = GATE_UP ? warp_sum(a1[i]) : 0.0f;+ if (lane == 0) {+ const size_t lr = (size_t) (lo + i);+ if (GATE_UP) a.h[lr * a.ldo + row] = __half_as_ushort(__float2half_rn(swiglu(v0, v1, a.limit)));+ else a.y[lr * a.ldo + row] = v0;+ }+ }+}++bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps the F32 kernel+#if defined(STRATA_USE_HIP)+ return false;+#else+ static const bool ok = [] {+ int dev = 0, major = 0;+ cudaGetDevice(&dev);+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ const char* v = std::getenv("STRATA_DS4_NO_TC");+ return major >= 7 && !(v != nullptr && v[0] == '1');+ }();+ return ok;+#endif+}++// ---------------------------------------------------------------- dispatch+// F(T) for the type t at run time; false when it is not one of these+#define DS4_IQ_SWITCH(t, F) \+ switch (t) { \+ case kIQ2_XXS: F(kIQ2_XXS); break; \+ case kIQ2_S: F(kIQ2_S); break; \+ case kIQ3_XXS: F(kIQ3_XXS); break; \+ case kIQ3_S: F(kIQ3_S); break; \+ case kIQ4_XS: F(kIQ4_XS); break; \+ case kQ4_K: F(kQ4_K); break; \+ case kQ2_K: F(kQ2_K); break; \+ case kQ3_K: F(kQ3_K); break; \+ default: return false; \+ }++template <int T, int NE>+void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { gate_up_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }+template <int T, int NE>+void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { down_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }+template <int T>+void gu_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {+ if (ne <= 1) launch_gu<T, 1>(a, grid, s);+ else if (ne <= 4) launch_gu<T, 4>(a, grid, s);+ else if (ne <= 8) launch_gu<T, 8>(a, grid, s);+ else launch_gu<T, 16>(a, grid, s);+}+template <int T>+void dn_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {+ if (ne <= 1) launch_dn<T, 1>(a, grid, s);+ else if (ne <= 4) launch_dn<T, 4>(a, grid, s);+ else if (ne <= 8) launch_dn<T, 8>(a, grid, s);+ else launch_dn<T, 16>(a, grid, s);+}+template <int T, bool GU>+void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {+#if !defined(STRATA_USE_HIP)+ if (tc_ok()) {+ set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);+ return;+ }+#endif+ if (a.max_rows <= 16) { // (a block of 64 rows would do 64 rows' FMAs for the few there are)+ grid.x = (unsigned) ((a.max_rows + 15) / 16);+ set_kernel<T, GU, 16><<<grid, kThreads, 0, s>>>(a);+ return;+ }+ set_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);+}+// the light sets' int8 rows, per stream (kLightMaxExp kLightRows rows of k)+struct LightBuf {+ int8_t* q = nullptr;+ float* d = nullptr;+ size_t cap = 0;+};+std::mutex g_lmu;+std::unordered_map<cudaStream_t, LightBuf> g_light;+LightBuf* light_buf(cudaStream_t s, int k) {+ std::lock_guard<std::mutex> lk(g_lmu);+ LightBuf& b = g_light[s];+ const size_t need = (size_t) kLightMaxExp * kLightRows * (size_t) std::max(k, 8192);+ if (b.cap < need) {+ if (b.q != nullptr) cudaFree(b.q);+ if (b.d != nullptr) cudaFree(b.d);+ b = LightBuf{};+ if (cudaMalloc(&b.q, need) != cudaSuccess || cudaMalloc(&b.d, need / 32 * sizeof(float)) != cudaSuccess) {+ std::fprintf(stderr, "ds4 iq: the light sets' scratch (%zu MB) failed\n", need >> 20);+ return nullptr;+ }+ b.cap = need;+ }+ return &b;+}+bool light_on() { // STRATA_DS4_NO_LIGHT=1: every set through the GEMM tiles+ static const bool on = [] {+ const char* v = std::getenv("STRATA_DS4_NO_LIGHT");+ return !(v != nullptr && v[0] == '1');+ }();+ return on;+}+template <int T, bool GU>+void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {+ const dim3 grid((unsigned) (((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps), (unsigned) a.n_exp);+ if (a.max_rows <= 4) light_kernel<T, 4, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);+ else if (a.max_rows <= 8) light_kernel<T, 8, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);+ else light_kernel<T, kLightRows, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);+}+bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {+ const int K = gu ? a.n_embd : a.n_ff;+ LightBuf* b = light_buf(s, K);+ if (b == nullptr) return false;+ const dim3 gq((unsigned) ((K / 32 + kWarps - 1) / kWarps), (unsigned) (a.n_exp * kLightRows));+ light_q8_kernel<<<gq, kThreads, 0, s>>>(gu ? a.xf : nullptr, gu ? nullptr : a.xh, a.idx, a.ldx, K, a.bounds,+ a.row0, b->q, b->d);+ if (gu) {+ if (a.tg == kMXFP4) {+ launch_light<kMXFP4, true>(a, *b, s);+ return true;+ }+#define LG_(T) launch_light<T, true>(a, *b, s)+ DS4_IQ_SWITCH(a.tg, LG_)+#undef LG_+ } else {+ if (a.td == kMXFP4) {+ launch_light<kMXFP4, false>(a, *b, s);+ return true;+ }+#define LD_(T) launch_light<T, false>(a, *b, s)+ DS4_IQ_SWITCH(a.td, LD_)+#undef LD_+ }+ return true;+}++template <int T>+void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {+ const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);+ if (max_c <= 1) mv_kernel<T, 1><<<grid, kThreads, 0, s>>>(J);+ else if (max_c <= 4) mv_kernel<T, 4><<<grid, kThreads, 0, s>>>(J);+ else mv_kernel<T, 16><<<grid, kThreads, 0, s>>>(J);+}++} // namespace++bool supported(int t) { return block_bytes(t) > 0; }++int block_bytes(int t) {+ switch (t) {+ case kIQ2_XXS: return Ty<kIQ2_XXS>::B;+ case kIQ2_S: return Ty<kIQ2_S>::B;+ case kIQ3_XXS: return Ty<kIQ3_XXS>::B;+ case kIQ3_S: return Ty<kIQ3_S>::B;+ case kIQ4_XS: return Ty<kIQ4_XS>::B;+ case kQ4_K: return Ty<kQ4_K>::B;+ case kQ2_K: return Ty<kQ2_K>::B;+ case kQ3_K: return Ty<kQ3_K>::B;+ default: return 0;+ }+}++const char* name(int t) {+ switch (t) {+ case kIQ2_XXS: return "IQ2_XXS";+ case kIQ2_S: return "IQ2_S";+ case kIQ3_XXS: return "IQ3_XXS";+ case kIQ3_S: return "IQ3_S";+ case kIQ4_XS: return "IQ4_XS";+ case kQ4_K: return "Q4_K";+ case kQ2_K: return "Q2_K";+ case kQ3_K: return "Q3_K";+ default: return "?";+ }+}++bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+ if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;+ const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);+#define GU_(T) gu_ne<T>(a, grid, ne_max, s)+ DS4_IQ_SWITCH(a.tg, GU_)+#undef GU_+ return true;+}++bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+ if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;+ const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);+#define DN_(T) dn_ne<T>(a, grid, ne_max, s)+ DS4_IQ_SWITCH(a.td, DN_)+#undef DN_+ return true;+}++bool light_gate_up(const SetArgs& a, cudaStream_t s) {+ if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;+ return light_set(a, true, s);+}+bool light_down(const SetArgs& a, cudaStream_t s) {+ if (a.n_exp > kLightMaxExp || !light_on() || a.n_embd % 256 != 0 || a.n_ff % 256 != 0) return false;+ return light_set(a, false, s);+}++bool set_gate_up(const SetArgs& a, cudaStream_t s) {+ if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;+ SetArgs t = a;+ if (light_gate_up(a, s)) { // the experts of a few rows; the tiles the others+ if (a.max_rows <= kLightRows) return true;+ t.skip_le = kLightRows;+ }+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_ff / BN), (unsigned) a.n_exp);+#define SG_(T) launch_set<T, true>(t, grid, s)+ DS4_IQ_SWITCH(a.tg, SG_)+#undef SG_+ return true;+}++bool set_down(const SetArgs& a, cudaStream_t s) {+ if (a.n_ff % 256 != 0 || a.n_embd % BN != 0) return false;+ SetArgs t = a;+ if (light_down(a, s)) {+ if (a.max_rows <= kLightRows) return true;+ t.skip_le = kLightRows;+ }+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (a.n_embd / BN), (unsigned) a.n_exp);+#define SD_(T) launch_set<T, false>(t, grid, s)+ DS4_IQ_SWITCH(a.td, SD_)+#undef SD_+ return true;+}++bool mv(const MvJob* jobs, int n, cudaStream_t s) {+ // the jobs by type, up to kMaxJobs a launch+ bool done[64] = {};+ if (n > 64) return false;+ for (int i = 0; i < n; ++i)+ while (!done[i]) {+ const int t = jobs[i].type;+ Jobs J{};+ int nj = 0, max_n = 0, max_c = 1;+ for (int k = i; k < n && nj < kMaxJobs; ++k) {+ if (done[k] || jobs[k].type != t) continue;+ if (jobs[k].k % 256 != 0 || jobs[k].ncol < 1 || jobs[k].ncol > 16) return false;+ J.j[nj++] = jobs[k];+ max_n = std::max(max_n, jobs[k].n);+ max_c = std::max(max_c, jobs[k].ncol);+ done[k] = true;+ }+#define MV_(T) mv_nc<T>(J, nj, max_n, max_c, s)+ DS4_IQ_SWITCH(t, MV_)+#undef MV_+ }+ return true;+}++} // namespace strata::ds4::gguf::iqdiff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhnew file mode 100644index 000000000..1b4486d69--- /dev/null+++ b/src/dsv41/ds4_iq.cuh@@ -0,0 +1,102 @@+// src/ds4/ds4_iq.cuh - a GGUF's routed experts in ggml's low-bit block types on the device for maya-ds4: the i-quants+// IQ2_XXS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS and the K-quants Q2_K, Q3_K, Q4_K (256 values a block, ggml's layouts:+// ggml-common.h, llama.cpp MIT). ds4_gguf.cu sends an expert layer here when its experts are not MXFP4+// (x3::ExpertLayout::fmt kFmtMXFP4 with K[r] = role r's ggml type; K[r] == kTypeMXFP4: ds4_gguf.cu's own kernels).+//+// Every weight is decoded exactly (ggml's dequantize_row_*: a block scale times an integer scale times a grid or code+// value) and every product takes the activations rounded to int8 per 32 values with an F32 scale (ds4_gguf.cu's+// q8_quant): a 32-value sub-block's sum is an exact integer dot (dp4a) times its scales in F32 - no integer division as+// llama.cpp's vecdotq takes (its sums truncate), so these are the double reference's sums to F32 rounding. A window's+// entry gets the token path's bits (the same kernel, the same order).+#pragma once++#include <cstddef>+#include <cstdint>+#include <cuda_runtime.h>++namespace strata::ds4::gguf::iq {++// ggml type ids+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,+ kIQ4_XS = 23;++/// Whether the device kernels take this ggml type.+bool supported(int type);+/// Bytes of a block (256 values) of the type (0: not one of these).+int block_bytes(int type);+/// Bytes of a row of k values (k % 256 == 0).+inline size_t row_bytes(int type, int64_t k) { return (size_t) (k / 256) * (size_t) block_bytes(type); }+/// The type's name ("?" when unknown).+const char* name(int type);++/// The token / window experts (ds4_gguf.cu's MxArgs, with the roles' types): group g = an expert (grp_ptr null: expert g+/// of the plan, plan_ptr[g] its blob or 0, its entry g, input row 0; else blob grp_ptr[g], entries grp_ent[grp_start[g]+/// ..], entry e's input row e >> 3). x / h as int8 rows with an F32 scale per 32 (xq, xd: rows of n_embd; hq, hd: row e+/// = entry e, rows of n_ff).+struct Args {+ const unsigned long long* plan_ptr = nullptr;+ const unsigned long long* grp_ptr = nullptr;+ const int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;+ size_t tr[3] = {0, 0, 0}; // the roles' offsets in a blob+ int n_embd = 0, n_ff = 0;+ float limit = 0.0f;+ const int8_t* xq = nullptr;+ const float* xd = nullptr;+ const int8_t* hq = nullptr;+ const float* hd = nullptr;+ float* h = nullptr; // [entries][n_ff]: the clamped swiglu+ float* part = nullptr; // [entries][n_embd]: each entry's down output+ int tg = 0, td = 0; // gate / up's type, down's+};+/// grid (n_ff / 8, groups): h of every entry (ne_max: a group's most entries). false: a type not covered.+bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s);+/// grid (n_embd / 8, groups): part of every entry.+bool down(const Args& a, int groups, int ne_max, cudaStream_t s);++/// The prompt path's expert sets (ds4_gguf.cu's SetK): expert i's blob at wbase + i stride owns the sorted rows+/// [bounds[i], bounds[i + 1]) (local row = sorted row - row0); gate / up: h16[local] = F16 swiglu over x[idx[local]]+/// (F32 rows, ldx); down: y[local] = down(h16[local]) (F16 rows in, ldx; F32 out, ldo). A set's experts of at most 16+/// rows take the token kernels' form (each weight read once, the rows' inputs rounded to int8 per 32 as the token path+/// rounds them; STRATA_DS4_NO_LIGHT=1: never), the others GEMM tiles (tensor cores on CUDA, F16 operands).+struct SetArgs {+ const uint8_t* wbase = nullptr;+ size_t stride = 0;+ size_t tr[3] = {0, 0, 0};+ const int* bounds = nullptr;+ int row0 = 0, max_rows = 0, n_exp = 0;+ int n_embd = 0, n_ff = 0;+ float limit = 0.0f;+ const float* xf = nullptr;+ const int* idx = nullptr;+ const uint16_t* xh = nullptr;+ int ldx = 0;+ uint16_t* h = nullptr;+ float* y = nullptr;+ int ldo = 0;+ int tg = 0, td = 0;+ int skip_le = 0; // (the tiles) the experts of at most this many rows are left out: the light kernels took them+};+bool set_gate_up(const SetArgs& a, cudaStream_t s);+bool set_down(const SetArgs& a, cudaStream_t s);+/// The light form alone (the set's experts of at most 16 rows; the others untouched), for any of these types and MXFP4+/// (tg / td 39: ds4_gguf.cu's experts): false (nothing launched) when it is switched off - the caller's tiles take every+/// expert then; true: they take those of more rows (skip_le 16).+bool light_gate_up(const SetArgs& a, cudaStream_t s);+bool light_down(const SetArgs& a, cudaStream_t s);++/// GEMV jobs on such matrices (an expert's views on the host-routed path): y[c][r] = alpha (row r . x_c) + bias[r] for+/// ncol activations, x as int8 rows (xq + c k, scales xd + c k / 32).+struct MvJob {+ const uint8_t* w = nullptr;+ size_t ld = 0; // bytes a row+ int type = 0, k = 0, n = 0, ncol = 1;+ const int8_t* xq = nullptr;+ const float* xd = nullptr;+ float* y = nullptr;+ size_t ycs = 0;+ const float* bias = nullptr;+ float alpha = 1.0f;+};+bool mv(const MvJob* jobs, int n, cudaStream_t s);++} // namespace strata::ds4::gguf::iqdiff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cppnew file mode 100644index 000000000..b2d8f2409--- /dev/null+++ b/src/dsv41/ds4_iq_cpu.cpp@@ -0,0 +1,766 @@+// src/ds4/ds4_iq_cpu.cpp - a GGUF's low-bit experts on the CPU lane (see ds4_gguf_cpu.hpp): the GLM CPU lane's native+// experts (src/kernels/cpu/native_expert.cpp), whose arithmetic is ggml-cpu's - and for IQ3_XXS / IQ3_S (Maya-S's+// down, Maya-M's gate / up / down) rows of our own, the same numbers bit for bit, faster.+#include "ds4_gguf_cpu.hpp"++#if defined(STRATA_NATIVE_EXPERTS)+#include "strata/kernels/cpu/native_expert.hpp"++#define GGML_COMMON_DECL_CPP+#define GGML_COMMON_IMPL_CPP+#include "ggml-common.h"+#include "ggml-cpu.h"+#include "ggml.h"+#endif++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <limits>+#include <mutex>+#include <string>+#include <vector>++#if defined(__x86_64__) || defined(_M_X64)+#define IQ_X86 1+#include <immintrin.h>+#if defined(_MSC_VER)+#include <intrin.h>+#else+#include <cpuid.h>+#endif+#endif+#if defined(IQ_X86) && (defined(__GNUC__) || defined(__clang__))+#define IQ_AVX2 __attribute__((target("avx2,fma")))+#else+#define IQ_AVX2+#endif++namespace strata::ds4::gguf {++#if defined(STRATA_NATIVE_EXPERTS)+namespace kc = strata::kernels::cpu;+namespace {+kc::NativeFmt native(const IqCpu& f) { // (the fields the rows read; no lookups)+ kc::NativeFmt n;+ n.gu_type = f.tg;+ n.d_type = f.td;+ n.gu_act = f.gu_act;+ n.d_act = f.d_act;+ n.n_embd = f.n_embd;+ n.n_ff = f.n_ff;+ n.gu_row = f.gu_row;+ n.d_row = f.d_row;+ n.up_off = f.gu_row * (size_t) f.n_ff;+ n.down_off = 2 * n.up_off;+ n.bytes = n.down_off + f.d_row * (size_t) f.n_embd;+ n.act_bytes = f.act_bytes;+ n.h_bytes = f.h_bytes;+ return n;+}++// ---- what this CPU runs: AVX2 + FMA (the OS saving the YMM state), and AVX-512 F / BW + VBMI with BMI2 (the ZMM state)+struct IqCpuIsa {+ bool avx2 = false, w512 = false;+};+IqCpuIsa iq_isa() {+ IqCpuIsa c;+#if defined(IQ_X86)+ unsigned r[4] = {0, 0, 0, 0};+ auto cpuid = [&](unsigned leaf, unsigned sub) {+#if defined(_MSC_VER)+ int x[4];+ __cpuidex(x, (int) leaf, (int) sub);+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];+#else+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);+#endif+ };+ cpuid(0, 0);+ if (r[0] < 7) return c;+ cpuid(1, 0);+ if (!((r[2] >> 12) & 1u) || !((r[2] >> 27) & 1u) || !((r[2] >> 28) & 1u)) return c; // FMA, OSXSAVE, AVX+#if defined(_MSC_VER)+ const unsigned long long xcr0 = _xgetbv(0);+#else+ unsigned lo = 0, hi = 0;+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;+#endif+ if ((xcr0 & 0x6) != 0x6) return c;+ cpuid(7, 0);+ c.avx2 = (r[1] >> 5) & 1u;+ c.w512 = c.avx2 && (xcr0 & 0xE6) == 0xE6 && ((r[1] >> 16) & 1u) && ((r[1] >> 30) & 1u) && ((r[2] >> 1) & 1u) &&+ ((r[1] >> 8) & 1u); // AVX-512 F, BW, VBMI; BMI2+#endif+ return c;+}+const IqCpuIsa& isa() {+ static const IqCpuIsa c = iq_isa();+ return c;+}++// ---- IQ3_XXS / IQ3_S rows of our own. ggml-cpu's AVX2 dots build each 32-value chunk's grid magnitudes from eight+// scalar table loads and inserts, and redo it for every token; here the grid comes in with one vpgatherdd from the+// same table, the signs are built in registers (the GLM rows' sgn7_vec / sgn_vec), and every token of a call applies+// the decoded chunk. The arithmetic is ggml's to the bit: the same exact int32 lanes per block (lane l: the chunks'+// grid entries l, each times its 2 s + 1), then for each block fma(d_x d_y, lanes, acc) and hsum_float_8 (IQ3_XXS:+// times 0.25) - checked against ggml-cpu's own dot once per type (iq3_check), else ggml's rows are used.+inline float h2f(uint16_t h) { // F16 -> F32, exactly+ const uint32_t s = (uint32_t) (h & 0x8000u) << 16, e = (h >> 10) & 0x1fu, m = h & 0x3ffu;+ uint32_t b;+ if (e == 0) {+ const float f = (float) m * 5.9604644775390625e-8f; // m 2^-24 (exact)+ std::memcpy(&b, &f, 4);+ b |= s;+ } else if (e == 31) {+ b = s | 0x7f800000u | (m << 13);+ } else {+ b = s | ((e + 112u) << 23) | (m << 13);+ }+ float f;+ std::memcpy(&f, &b, 4);+ return f;+}+inline uint32_t u32(const uint8_t* p) {+ uint32_t v;+ std::memcpy(&v, p, 4);+ return v;+}++#if defined(IQ_X86)+IQ_AVX2 inline float hsum_ggml(__m256 x) { // ggml's hsum_float_8, the same additions in the same order+ __m128 r = _mm256_extractf128_ps(x, 1);+ r = _mm_add_ps(r, _mm256_castps256_ps128(x));+ r = _mm_add_ps(r, _mm_movehl_ps(r, r));+ r = _mm_add_ss(r, _mm_movehdup_ps(r));+ return _mm_cvtss_f32(r);+}+// IQ2_S / IQ3_S: 32 explicit sign bits -> 32 bytes of -1 / +1 (ggml's bit selector)+IQ_AVX2 inline __m256i sgn32(uint32_t m) {+ const __m256i sh = _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, //+ 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3);+ const __m256i sel = _mm256_set1_epi64x((long long) 0x8040201008040201ull);+ const __m256i b = _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) m), sh), sel);+ return _mm256_or_si256(_mm256_cmpeq_epi8(b, sel), _mm256_set1_epi8(1));+}+// table entries into a vector without the shuffle port (ggml's _mm256_set_epi64x / _epi32 of table loads ends up in+// inserts, one shuffle each; here each entry is a broadcast load and the lanes are put together by dword blends)+IQ_AVX2 inline __m256i bq(const void* p) { return _mm256_castpd_si256(_mm256_broadcast_sd((const double*) p)); }+IQ_AVX2 inline __m256i bd(const void* p) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*) p)); }+IQ_AVX2 inline __m256i tab4(const uint64_t* t, unsigned a, unsigned b, unsigned c, unsigned d) { // qwords t[a] t[b] t[c] t[d]+ return _mm256_blend_epi32(_mm256_blend_epi32(bq(t + a), bq(t + b), 0x0C), _mm256_blend_epi32(bq(t + c), bq(t + d), 0xC0),+ 0xF0);+}+IQ_AVX2 inline __m256i tab8(const uint32_t* t, const unsigned* i) { // dwords t[i[0]] .. t[i[7]]+ const __m256i a = _mm256_blend_epi32(bd(t + i[0]), bd(t + i[1]), 0x02), b = _mm256_blend_epi32(bd(t + i[2]), bd(t + i[3]), 0x08);+ const __m256i c = _mm256_blend_epi32(bd(t + i[4]), bd(t + i[5]), 0x20), e = _mm256_blend_epi32(bd(t + i[6]), bd(t + i[7]), 0x80);+ return _mm256_blend_epi32(_mm256_blend_epi32(a, b, 0x0C), _mm256_blend_epi32(c, e, 0xC0), 0xF0);+}+// the 8 sign bytes (-1 / +1) of each 7-bit sign index with its parity bit: ggml's keven_signs_q2xs, as +-1 bytes+alignas(64) uint64_t g_ks64[128];+void build_ks64() {+ for (int i = 0; i < 128; ++i) {+ int par = 0;+ for (int b = 0; b < 7; ++b) par ^= (i >> b) & 1;+ const int s = i | (par << 7);+ uint64_t v = 0;+ for (int j = 0; j < 8; ++j) v |= (uint64_t) (((s >> j) & 1) ? 0xFFu : 0x01u) << (8 * j);+ g_ks64[i] = v;+ }+}+IQ_AVX2 inline __m256i sgn7t(uint32_t w) { // a chunk's four 7-bit sign indices (bits 0..27 of w)+ return tab4(g_ks64, w & 127u, (w >> 7) & 127u, (w >> 14) & 127u, (w >> 21) & 127u);+}+IQ_AVX2 inline __m256i sc16(int a, int b) { // int16 lanes 0-7 a (values 0-15), 8-15 b (values 16-31)+ return _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_set1_epi16((short) a)), _mm_set1_epi16((short) b), 1);+}+bool g_gather18 = false, g_gather21 = false; // the IQ3_XXS / IQ3_S grids by vpgatherdd (own(): if faster here)+// IQ2_XXS's grid entry with its signs applied, for every grid index g and sign index i, at g << 7 | i (256 KB, built on+// first use): one lookup instead of two; the products are then (q8 + 128) x signed grid less 128 x the grid's pair sums+// (vpmaddubsw takes u8 x s8), the same integers as ggml's |grid| x signed q8+alignas(64) uint64_t g_ct2x[256 * 128];+bool g_ct16 = false; // (own(): if faster here)+void build_ct2x() {+ for (int g = 0; g < 256; ++g)+ for (int i = 0; i < 128; ++i) {+ const uint64_t sg = g_ks64[i]; // (+-1 bytes)+ uint64_t v = 0;+ for (int j = 0; j < 8; ++j) {+ const int gb = (int) ((iq2xxs_grid[g] >> (8 * j)) & 255u);+ const int sb = ((sg >> (8 * j)) & 255u) == 0xFFu ? -gb : gb;+ v |= (uint64_t) (uint8_t) (int8_t) sb << (8 * j);+ }+ g_ct2x[(size_t) g << 7 | (size_t) i] = v;+ }+}++// rows [r0, r1) (out[t][r - r0]) of NT tokens; TY 16 = IQ2_XXS, 22 = IQ2_S, 18 = IQ3_XXS, 21 = IQ3_S. A pair of+// 32-value chunks a step: grid magnitudes g1 / g2, sign vectors s1 / s2 (-1 / +1), scales c1 / c2 - decoded once - then+// every token: q8 times the signs, vpmaddubsw against the grid, vpmaddwd by the scales, into ggml's 8 int32 lanes+template <int TY, int NT>+IQ_AVX2 void iq_rows_avx2(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,+ int r1) {+ const __m256i sub = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);+ const bool gat = TY == 18 ? g_gather18 : TY == 21 ? g_gather21 : false;+ const bool ct = TY == 16 && g_ct16;+ const __m256i k80 = _mm256_set1_epi8((char) 0x80);+ constexpr float K = TY == 18 ? 0.25f : TY == 21 ? 1.0f : 0.125f;+ constexpr size_t BB = TY == 16 ? sizeof(block_iq2_xxs) : TY == 22 ? sizeof(block_iq2_s) :+ TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);+ for (int r = r0; r < r1; ++r) {+ const uint8_t* row = w + (size_t) r * rb;+ __m256 accf[NT];+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = row + (size_t) i * BB;+ __m256i sumi[NT];+ for (int t = 0; t < NT; ++t) sumi[t] = _mm256_setzero_si256();+ uint16_t dh;+ std::memcpy(&dh, blk, 2);+ for (int ib = 0; ib < QK_K / 32; ib += 2) {+ __m256i g1, g2, s1, s2, c1, c2;+ if (TY == 16) { // IQ2_XXS: per chunk a word of 4 grid indices, a word of 4 sign indices + the scale+ const uint8_t* q = blk + 2 + 8 * ib;+ const uint32_t a0 = u32(q), a1 = u32(q + 4), b0 = u32(q + 8), b1 = u32(q + 12);+ if (ct) { // (g1 / g2 the signed grids; s1 / s2 their 128 x pair sums)+ g1 = tab4(g_ct2x, (a0 & 255u) << 7 | (a1 & 127u), ((a0 >> 8) & 255u) << 7 | ((a1 >> 7) & 127u),+ ((a0 >> 16) & 255u) << 7 | ((a1 >> 14) & 127u), (a0 >> 24) << 7 | ((a1 >> 21) & 127u));+ g2 = tab4(g_ct2x, (b0 & 255u) << 7 | (b1 & 127u), ((b0 >> 8) & 255u) << 7 | ((b1 >> 7) & 127u),+ ((b0 >> 16) & 255u) << 7 | ((b1 >> 14) & 127u), (b0 >> 24) << 7 | ((b1 >> 21) & 127u));+ s1 = _mm256_maddubs_epi16(k80, g1);+ s2 = _mm256_maddubs_epi16(k80, g2);+ } else {+ g1 = tab4(iq2xxs_grid, a0 & 255u, (a0 >> 8) & 255u, (a0 >> 16) & 255u, a0 >> 24);+ g2 = tab4(iq2xxs_grid, b0 & 255u, (b0 >> 8) & 255u, (b0 >> 16) & 255u, b0 >> 24);+ s1 = sgn7t(a1);+ s2 = sgn7t(b1);+ }+ c1 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));+ c2 = _mm256_set1_epi16((short) (2 * (b1 >> 28) + 1));+ } else if (TY == 22) { // IQ2_S: 10-bit grid indices (qs | 2 bits of qh), explicit signs, 2 scales a chunk+ const uint8_t* qs = blk + 2 + 4 * ib;+ const uint8_t* qh = blk + 2 + QK_K / 4;+ const uint8_t* sg = blk + 2 + QK_K / 8 + 4 * ib;+ const uint8_t* scl = blk + 2 + QK_K / 4 + QK_K / 32;+ const unsigned h0 = qh[ib], h1 = qh[ib + 1];+ g1 = tab4(iq2s_grid, qs[0] | ((h0 << 8) & 0x300u), qs[1] | ((h0 << 6) & 0x300u), qs[2] | ((h0 << 4) & 0x300u),+ qs[3] | ((h0 << 2) & 0x300u));+ g2 = tab4(iq2s_grid, qs[4] | ((h1 << 8) & 0x300u), qs[5] | ((h1 << 6) & 0x300u), qs[6] | ((h1 << 4) & 0x300u),+ qs[7] | ((h1 << 2) & 0x300u));+ s1 = sgn32(u32(sg));+ s2 = sgn32(u32(sg + 4));+ c1 = sc16(2 * (scl[ib] & 0xf) + 1, 2 * (scl[ib] >> 4) + 1);+ c2 = sc16(2 * (scl[ib + 1] & 0xf) + 1, 2 * (scl[ib + 1] >> 4) + 1);+ } else if (TY == 18) { // IQ3_XXS: 8 grid indices a chunk, then the chunks' sign / scale words+ const uint8_t* q3 = blk + 2 + 8 * ib;+ const uint8_t* gas = blk + 2 + QK_K / 4 + 4 * ib;+ if (gat) {+ g1 = _mm256_i32gather_epi32((const int*) iq3xxs_grid, _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) q3)), 4);+ g2 = _mm256_i32gather_epi32((const int*) iq3xxs_grid,+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (q3 + 8))), 4);+ } else {+ unsigned i1[8], i2[8];+ for (int k = 0; k < 8; ++k) {+ i1[k] = q3[k];+ i2[k] = q3[8 + k];+ }+ g1 = tab8(iq3xxs_grid, i1);+ g2 = tab8(iq3xxs_grid, i2);+ }+ const uint32_t a0 = u32(gas), a1 = u32(gas + 4);+ s1 = sgn7t(a0);+ s2 = sgn7t(a1);+ c1 = _mm256_set1_epi16((short) (2 * (a0 >> 28) + 1));+ c2 = _mm256_set1_epi16((short) (2 * (a1 >> 28) + 1));+ } else { // IQ3_S: 9-bit grid indices (qs | a bit of qh), explicit signs+ const uint8_t* qs = blk + 2 + 8 * ib;+ const uint8_t* qh = blk + 2 + QK_K / 4;+ const uint8_t* sg = blk + 2 + QK_K / 4 + QK_K / 32 + 4 * ib;+ const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + ib / 2];+ const unsigned h0 = qh[ib], h1 = qh[ib + 1];+ if (gat) {+ const __m256i hb1 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h0), sub), _mm256_set1_epi32(1)), 8);+ const __m256i hb2 = _mm256_slli_epi32(_mm256_and_si256(_mm256_srlv_epi32(_mm256_set1_epi32((int) h1), sub), _mm256_set1_epi32(1)), 8);+ g1 = _mm256_i32gather_epi32((const int*) iq3s_grid,+ _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) qs)), hb1), 4);+ g2 = _mm256_i32gather_epi32((const int*) iq3s_grid,+ _mm256_or_si256(_mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*) (qs + 8))), hb2), 4);+ } else {+ unsigned i1[8], i2[8];+ for (int k = 0; k < 8; ++k) {+ i1[k] = qs[k] | (((h0 >> k) & 1u) << 8);+ i2[k] = qs[8 + k] | (((h1 >> k) & 1u) << 8);+ }+ g1 = tab8(iq3s_grid, i1);+ g2 = tab8(iq3s_grid, i2);+ }+ s1 = sgn32(u32(sg));+ s2 = sgn32(u32(sg + 4));+ c1 = _mm256_set1_epi16((short) (2 * (sc & 0xf) + 1));+ c2 = _mm256_set1_epi16((short) (2 * (sc >> 4) + 1));+ }+ if (ct) {+ for (int t = 0; t < NT; ++t) {+ const int8_t* q8 = y[t][i].qs + 32 * ib;+ const __m256i u1 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) q8), k80); // (q8 + 128)+ const __m256i u2 = _mm256_xor_si256(_mm256_loadu_si256((const __m256i*) (q8 + 32)), k80);+ const __m256i d1 = _mm256_sub_epi16(_mm256_maddubs_epi16(u1, g1), s1);+ const __m256i d2 = _mm256_sub_epi16(_mm256_maddubs_epi16(u2, g2), s2);+ sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));+ }+ continue;+ }+ for (int t = 0; t < NT; ++t) {+ const int8_t* q8 = y[t][i].qs + 32 * ib;+ const __m256i d1 = _mm256_maddubs_epi16(g1, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) q8), s1));+ const __m256i d2 = _mm256_maddubs_epi16(g2, _mm256_sign_epi8(_mm256_loadu_si256((const __m256i*) (q8 + 32)), s2));+ sumi[t] = _mm256_add_epi32(sumi[t], _mm256_add_epi32(_mm256_madd_epi16(d1, c1), _mm256_madd_epi16(d2, c2)));+ }+ }+ const float dx = h2f(dh);+ for (int t = 0; t < NT; ++t)+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(sumi[t]), accf[t]);+ }+ for (int t = 0; t < NT; ++t) out[t][r - r0] = K * hsum_ggml(accf[t]);+ }+}++// ---- the same rows in 512-bit lanes (AVX-512 BW + VBMI, BMI2: Ice Lake and later Intel, Zen 4 / Zen 5), the GLM+// lane's one-token IQ2_XXS scheme (src/kernels/cpu/iq_avx512_one.cpp) for 3-bit grids: every grid byte is one of 8+// levels, so an entry is four 3-bit codes (a u16) and the grid's codes fit in registers - IQ3_XXS's 256 in 8 zmm,+// IQ3_S's 512 in 16; a block's 64 indices are looked up 32 at a time with two-register word permutes and blends, each+// 64 values' 16 codes spread to their bytes by a word permute, a multishift, an and and a byte shuffle; IQ3_XXS's signs+// come from pdep and a parity fold, IQ3_S's are a mask register's bits. Two chunks a register, folded to ggml's 8+// lanes, then ggml's float steps: the same numbers to the bit (own_check).+#if defined(__GNUC__) || defined(__clang__)+#define IQ_512 __attribute__((target("avx512f,avx512bw,avx512vbmi,bmi2,avx2,fma")))+#else+#define IQ_512+#endif+alignas(64) uint16_t g_code3x[256], g_code3s[512];+bool g_codes_ok = false;++IQ_512 inline uint32_t signs4(uint32_t w) { // ksigns_iq2xs for the four 7-bit fields of w: i | parity(i) << 7+ const uint64_t s = _pdep_u64(w & 0x0FFFFFFFu, 0x7F7F7F7Full);+ uint64_t p = s ^ ((s >> 4) & 0x0F0F0F0Full);+ p ^= (p >> 2) & 0x03030303ull;+ p ^= (p >> 1) & 0x01010101ull;+ return (uint32_t) (s | ((p & 0x01010101ull) << 7));+}+// each grid entry as four 3-bit level codes; false if a byte is not one of the 8 levels (a grid this file does not know)+bool build_codes3() {+ static const int lx[8] = {4, 12, 20, 28, 36, 44, 52, 62}, ls[8] = {1, 3, 5, 7, 9, 11, 13, 15};+ auto code = [](uint32_t e, const int* lv) -> int {+ int c = 0;+ for (int k = 0; k < 4; ++k) {+ const int b = (int) ((e >> (8 * k)) & 255u);+ int l = -1;+ for (int j = 0; j < 8; ++j)+ if (lv[j] == b) l = j;+ if (l < 0) return -1;+ c |= l << (3 * k);+ }+ return c;+ };+ for (int i = 0; i < 256; ++i) {+ const int c = code(iq3xxs_grid[i], lx);+ if (c < 0) return false;+ g_code3x[i] = (uint16_t) c;+ }+ for (int i = 0; i < 512; ++i) {+ const int c = code(iq3s_grid[i], ls);+ if (c < 0) return false;+ g_code3s[i] = (uint16_t) c;+ }+ return true;+}+IQ_512 inline __m256i fold(__m512i a) { // a block's 16 int32 lanes -> ggml's 8 (lane k + lane k + 8), exact+ return _mm256_add_epi32(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64(a, 1));+}+IQ_512 inline __m512i scales4(int s0, int s1, int s2, int s3) { // int16 lanes 0-7 s0, 8-15 s1, 16-23 s2, 24-31 s3+ const __m512i idx = _mm512_set_epi16(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, //+ 0, 0, 0, 0, 0, 0, 0, 0);+ const uint64_t packed = (uint64_t) (uint16_t) s0 | ((uint64_t) (uint16_t) s1 << 16) |+ ((uint64_t) (uint16_t) s2 << 32) | ((uint64_t) (uint16_t) s3 << 48);+ return _mm512_permutexvar_epi16(idx, _mm512_set1_epi64((long long) packed));+}+// the codes of 32 word indices: 64 entries a register pair, NP pairs, blended by the index's bits 6, 7 (and 8)+template <int NP>+IQ_512 inline __m512i lookup(const __m512i* T, __m512i idx) {+ __m512i r[NP];+ for (int j = 0; j < NP; ++j) r[j] = _mm512_permutex2var_epi16(T[2 * j], idx, T[2 * j + 1]);+ const __mmask32 m6 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(64));+ const __mmask32 m7 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(128));+ __m512i a = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[0], r[1]), _mm512_mask_blend_epi16(m6, r[2], r[3]));+ if (NP == 8) {+ const __mmask32 m8 = _mm512_test_epi16_mask(idx, _mm512_set1_epi16(256));+ const __m512i b = _mm512_mask_blend_epi16(m7, _mm512_mask_blend_epi16(m6, r[NP > 4 ? 4 : 0], r[NP > 5 ? 5 : 0]),+ _mm512_mask_blend_epi16(m6, r[NP > 6 ? 6 : 0], r[NP > 7 ? 7 : 0]));+ a = _mm512_mask_blend_epi16(m8, a, b);+ }+ return a;+}++template <int TY, int NT>+IQ_512 void iq3_rows_512(const uint8_t* w, size_t rb, int nb, const block_q8_K* const* y, float* const* out, int r0,+ int r1) {+ constexpr int NTAB = TY == 18 ? 8 : 16;+ constexpr size_t BB = TY == 18 ? sizeof(block_iq3_xxs) : sizeof(block_iq3_s);+ __m512i T[NTAB];+ for (int k = 0; k < NTAB; ++k)+ T[k] = _mm512_load_si512((const void*) ((TY == 18 ? g_code3x : g_code3s) + 32 * k));+ // code -> grid byte (IQ3_XXS 4 .. 62, IQ3_S 1 .. 15) in every 128-bit lane; byte b of an output qword <- 3 bits at+ // [0, 3, 6, 9] (its first entry's code, word 0) or [32, 35, 38, 41] (the second's, word 2)+ const __m512i lut = TY == 18 ? _mm512_set4_epi32(0, 0, 0x3E342C24, 0x1C140C04) : _mm512_set4_epi32(0, 0, 0x0F0D0B09, 0x07050301);+ const __m512i ctl = _mm512_set1_epi64(0x2926232009060300LL);+ const __m512i seven = _mm512_set1_epi8(7), zero = _mm512_setzero_si512();+ __m512i pidx[2]; // a 64-value group h of 32 codes: output qword m <- entries 16 h + 2 m (word 0), + 1 (word 2)+ for (int h = 0; h < 2; ++h) {+ alignas(64) uint16_t p[32];+ for (int m = 0; m < 8; ++m) {+ p[4 * m] = p[4 * m + 1] = (uint16_t) (16 * h + 2 * m);+ p[4 * m + 2] = p[4 * m + 3] = (uint16_t) (16 * h + 2 * m + 1);+ }+ pidx[h] = _mm512_load_si512((const void*) p);+ }+ for (int r = r0; r < r1; ++r) {+ const uint8_t* row = w + (size_t) r * rb;+ __m256 accf[NT];+ for (int t = 0; t < NT; ++t) accf[t] = _mm256_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = row + (size_t) i * BB;+ __m512i codes[2];+ if (TY == 18) {+ const __m512i q = _mm512_loadu_si512((const void*) (blk + 2));+ codes[0] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(q)));+ codes[1] = lookup<4>(T, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(q, 1)));+ } else {+ const uint8_t* qh = blk + 2 + QK_K / 4;+ const __m512i hi = _mm512_set1_epi16(256);+ const __m512i ia = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 2)));+ const __m512i ib = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*) (blk + 34)));+ // (index k of chunk c: qs[8 c + k] | bit k of qh[c] << 8 - the 32 bits of qh[0..3] are chunks 0..3's)+ codes[0] = lookup<8>(T, _mm512_or_si512(ia, _mm512_maskz_mov_epi16((__mmask32) u32(qh), hi)));+ codes[1] = lookup<8>(T, _mm512_or_si512(ib, _mm512_maskz_mov_epi16((__mmask32) u32(qh + 4), hi)));+ }+ __m512i acc[NT];+ for (int t = 0; t < NT; ++t) acc[t] = _mm512_setzero_si512();+ for (int p = 0; p < 4; ++p) {+ const __m512i rep = _mm512_permutexvar_epi16(pidx[p & 1], codes[p >> 1]);+ const __m512i g = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_multishift_epi64_epi8(ctl, rep), seven));+ uint64_t sm;+ int sa, sb;+ if (TY == 18) {+ const uint8_t* sw = blk + 2 + QK_K / 4 + 8 * p;+ const uint32_t wa = u32(sw), wb = u32(sw + 4);+ sm = (uint64_t) signs4(wa) | ((uint64_t) signs4(wb) << 32);+ sa = 2 * (int) (wa >> 28) + 1;+ sb = 2 * (int) (wb >> 28) + 1;+ } else {+ std::memcpy(&sm, blk + 2 + QK_K / 4 + QK_K / 32 + 8 * p, 8);+ const int sc = blk[2 + QK_K / 4 + QK_K / 32 + QK_K / 8 + p];+ sa = 2 * (sc & 0xf) + 1;+ sb = 2 * (sc >> 4) + 1;+ }+ const __mmask64 m = _cvtu64_mask64(sm);+ const __m512i S = scales4(sa, sa, sb, sb);+ for (int t = 0; t < NT; ++t) {+ const __m512i yv = _mm512_loadu_si512((const void*) (y[t][i].qs + 64 * p));+ const __m512i ys = _mm512_mask_sub_epi8(yv, m, zero, yv);+ acc[t] = _mm512_add_epi32(acc[t], _mm512_madd_epi16(_mm512_maddubs_epi16(g, ys), S));+ }+ }+ uint16_t dh;+ std::memcpy(&dh, blk, 2);+ const float dx = h2f(dh);+ for (int t = 0; t < NT; ++t)+ accf[t] = _mm256_fmadd_ps(_mm256_set1_ps(dx * y[t][i].d), _mm256_cvtepi32_ps(fold(acc[t])), accf[t]);+ }+ for (int t = 0; t < NT; ++t) out[t][r - r0] = (TY == 18 ? 0.25f : 1.0f) * hsum_ggml(accf[t]);+ }+}++template <int TY>+void iq_rows_n(const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {+ static const bool w512 = [] {+ const char* v = std::getenv("DS4_IQ_AVX2"); // (=1: the AVX2 rows on an AVX-512 CPU, to compare)+ return (TY == 18 || TY == 21) && g_codes_ok && isa().w512 && !(v != nullptr && std::atoi(v) != 0);+ }();+ const int nb = n / QK_K;+ for (int t0 = 0; t0 < nt; t0 += 4) {+ const block_q8_K* y[4];+ float* o[4];+ const int c = nt - t0 < 4 ? nt - t0 : 4;+ for (int t = 0; t < c; ++t) {+ y[t] = (const block_q8_K*) act[t0 + t];+ o[t] = out[t0 + t];+ }+ if constexpr (TY == 18 || TY == 21) {+ if (w512) {+ switch (c) {+ case 1: iq3_rows_512<TY, 1>(w, rb, nb, y, o, r0, r1); break;+ case 2: iq3_rows_512<TY, 2>(w, rb, nb, y, o, r0, r1); break;+ case 3: iq3_rows_512<TY, 3>(w, rb, nb, y, o, r0, r1); break;+ default: iq3_rows_512<TY, 4>(w, rb, nb, y, o, r0, r1); break;+ }+ continue;+ }+ }+ switch (c) {+ case 1: iq_rows_avx2<TY, 1>(w, rb, nb, y, o, r0, r1); break;+ case 2: iq_rows_avx2<TY, 2>(w, rb, nb, y, o, r0, r1); break;+ case 3: iq_rows_avx2<TY, 3>(w, rb, nb, y, o, r0, r1); break;+ default: iq_rows_avx2<TY, 4>(w, rb, nb, y, o, r0, r1); break;+ }+ }+}+#endif // IQ_X86++// out[t][r - r0] for the rows of `type` (an own type only)+void own_rows(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0,+ int r1) {+#if defined(IQ_X86)+ if (type == GGML_TYPE_IQ2_XXS) return iq_rows_n<16>(w, rb, n, act, nt, out, r0, r1);+ if (type == GGML_TYPE_IQ2_S) return iq_rows_n<22>(w, rb, n, act, nt, out, r0, r1);+ if (type == GGML_TYPE_IQ3_XXS) return iq_rows_n<18>(w, rb, n, act, nt, out, r0, r1);+ if (type == GGML_TYPE_IQ3_S) return iq_rows_n<21>(w, rb, n, act, nt, out, r0, r1);+#endif+ (void) type, (void) w, (void) rb, (void) n, (void) act, (void) nt, (void) out, (void) r0, (void) r1;+}++// our rows against ggml-cpu's dot, bit for bit: random blocks of the type (its F16 scale a small normal value), Q8_K+// activations from ggml's quantizer, 1, 2, 3, 5 and 9 tokens, 13 rows of 8 blocks+bool own_check(int type) {+ ggml_cpu_init();+ const ggml_type t = (ggml_type) type;+ const int n = 2048, rows = 13;+ const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);+ uint64_t st = 0x9E3779B97F4A7C15ull;+ auto next = [&st] {+ st ^= st << 13;+ st ^= st >> 7;+ st ^= st << 17;+ return st;+ };+ std::vector<uint8_t> w(rb * rows);+ for (auto& b : w) b = (uint8_t) next();+ for (size_t o = 0; o < w.size(); o += bb) {+ const ggml_fp16_t d = ggml_fp32_to_fp16(0.002f + 0.001f * (float) (next() % 16));+ std::memcpy(&w[o], &d, sizeof d);+ }+ const auto* q8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_K);+ const auto dot = ggml_get_type_traits_cpu(t)->vec_dot;+ for (int nt : {1, 2, 3, 5, 9}) {+ std::vector<uint8_t> a(ab * (size_t) nt);+ std::vector<float> x((size_t) n), got((size_t) nt * rows), ref((size_t) nt * rows);+ std::vector<const void*> ap((size_t) nt);+ std::vector<float*> op((size_t) nt);+ for (int k = 0; k < nt; ++k) {+ for (auto& v : x) v = ((float) (next() % 2001) - 1000.0f) * 1e-3f;+ q8->from_float(x.data(), a.data() + ab * (size_t) k, n);+ ap[(size_t) k] = a.data() + ab * (size_t) k;+ op[(size_t) k] = got.data() + (size_t) k * rows;+ }+ own_rows(type, w.data(), rb, n, ap.data(), nt, op.data(), 0, rows);+ for (int k = 0; k < nt; ++k)+ for (int r = 0; r < rows; ++r) dot(n, &ref[(size_t) k * rows + r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) k], 0, 1);+ if (std::memcmp(got.data(), ref.data(), got.size() * sizeof(float)) != 0) return false;+ }+ return true;+}++// the time (us, best of 5 rounds) of 32 rows of 16 blocks of the type against one token: ours with the IQ3 grids as+// broadcasts + blends (variant 1) or by vpgatherdd (2), or the native lane's rows (0: the GLM lane's / ggml-cpu's)+double time_variant(int type, int variant) {+ const ggml_type t = (ggml_type) type;+ const int n = 4096, rows = 32;+ const size_t rb = ggml_row_size(t, n), bb = ggml_type_size(t), ab = ggml_row_size(GGML_TYPE_Q8_K, n);+ std::vector<uint8_t> w(rb * rows), a(ab);+ uint32_t s = 12345u;+ for (auto& b : w) b = (uint8_t) ((s = s * 1664525u + 1013904223u) >> 24);+ for (size_t o = 0; o < w.size(); o += bb) {+ const ggml_fp16_t d = ggml_fp32_to_fp16(0.003f);+ std::memcpy(&w[o], &d, sizeof d);+ }+ std::vector<float> x((size_t) n), y((size_t) rows);+ for (auto& v : x) v = (float) ((int) ((s = s * 1664525u + 1013904223u) >> 16) - 32768) / 32768.0f;+ ggml_get_type_traits_cpu(GGML_TYPE_Q8_K)->from_float(x.data(), a.data(), n);+ const void* ap[1] = {a.data()};+ float* yp[1] = {y.data()};+ kc::NativeFmt nf; // (the native rows timed as a gate / up pair of the same rows: 2 x rows dots, as ours below)+ std::string err;+ if (variant == 0 && !kc::native_fmt(type, type, n, QK_K, nf, err)) return 1e30;+ bool* gflag = type == GGML_TYPE_IQ3_XXS ? &g_gather18 : type == GGML_TYPE_IQ3_S ? &g_gather21 :+ type == GGML_TYPE_IQ2_XXS ? &g_ct16 : nullptr;+ const bool keep = gflag != nullptr && *gflag;+ if (gflag != nullptr) *gflag = variant >= 2;+ double best = 1e30;+ for (int round = 0; round < 6; ++round) {+ const auto t0 = std::chrono::steady_clock::now();+ for (int rep = 0; rep < 4; ++rep) {+ if (variant == 0) {+ kc::native_gu_rows_split(nf, w.data(), w.data(), ap, 1, yp, 0, rows, std::numeric_limits<float>::infinity());+ } else {+ own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);+ own_rows(type, w.data(), rb, n, ap, 1, yp, 0, rows);+ }+ }+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();+ if (round > 0) best = std::min(best, us);+ }+ if (gflag != nullptr) *gflag = keep;+ return best;+}++// our rows for this type, here: IQ3_XXS / IQ3_S on AVX2 or AVX-512 VBMI, IQ2_XXS / IQ2_S on AVX2 - checked once+// against ggml-cpu's dot (each variant), then the fastest of ours and the native lane's measured on this CPU (they+// give the same bits, so the choice changes no answer). DS4_IQ_OWN=0: the native lane's rows; =1: ours without+// measuring (DS4_IQ_GATHER=1: the IQ3 grids by vpgatherdd); DS4_IQ_AVX2=1: our AVX2 rows on an AVX-512 CPU too.+bool own(int type) {+#if defined(IQ_X86)+ int k;+ switch (type) {+ case GGML_TYPE_IQ3_XXS: k = 0; break;+ case GGML_TYPE_IQ3_S: k = 1; break;+ case GGML_TYPE_IQ2_XXS: k = 2; break;+ case GGML_TYPE_IQ2_S: k = 3; break;+ default: return false;+ }+ static int mode = -1; // DS4_IQ_OWN: -1 measured, 0 never, 1 always+ static bool force_avx2 = false, force_gather = false;+ static const bool allowed = [] {+ const char* v = std::getenv("DS4_IQ_OWN");+ mode = v == nullptr ? -1 : std::atoi(v) != 0 ? 1 : 0;+ const char* a2 = std::getenv("DS4_IQ_AVX2");+ force_avx2 = a2 != nullptr && std::atoi(a2) != 0;+ const char* gt = std::getenv("DS4_IQ_GATHER");+ force_gather = gt != nullptr && std::atoi(gt) != 0;+ g_codes_ok = build_codes3(); // (the 512-bit rows' tables; without them those rows are not used)+ build_ks64();+ build_ct2x();+ return mode != 0 && isa().avx2;+ }();+ // (IQ2 on an AVX-512 CPU: the GLM lane's AVX-512 rows serve them, unless DS4_IQ_AVX2)+ if (!allowed || (k >= 2 && isa().w512 && !force_avx2)) return false;+ static std::once_flag once[4];+ static bool ok[4];+ std::call_once(once[k], [&] {+ bool* gflag = k == 0 ? &g_gather18 : k == 1 ? &g_gather21 : k == 2 ? &g_ct16 : nullptr; // (the second form)+ bool good = own_check(type); // (the blend form; the 512-bit rows where they serve)+ bool good_g = false;+ if (gflag != nullptr) {+ *gflag = true;+ good_g = own_check(type);+ *gflag = false;+ }+ if (mode == 1) {+ if (gflag != nullptr) *gflag = force_gather && good_g;+ ok[k] = good || (force_gather && good_g);+ return;+ }+ // the variants timed in turns, the best of each kept (other work on the machine - the load's read-ahead, the+ // prompt - then hits them alike), and ours taken only when clearly faster (10 %): the choice changes no answer,+ // a wrong one costs time (Uranus' Golden Cove: one-shot timings 30-80 % apart run to run, the choice flipping)+ double tn = 1e30, tb = 1e30, tg = 1e30;+ for (int it = 0; it < 5; ++it) {+ tn = std::min(tn, time_variant(type, 0));+ if (good) tb = std::min(tb, time_variant(type, 1));+ if (good_g) tg = std::min(tg, time_variant(type, 2));+ }+ if (gflag != nullptr) *gflag = tg < tb;+ ok[k] = std::min(tb, tg) < 0.9 * tn;+ if (std::getenv("DS4_IQ_VERBOSE") != nullptr)+ std::fprintf(stderr, "ds4: CPU lane %s rows: native %.0f us, ours %.0f us%s -> %s\n", ggml_type_name((ggml_type) type),+ tn, tb, gflag != nullptr ? (std::string(" (second form: gather / one table ") + std::to_string((int) tg) + " us)").c_str() : "",+ ok[k] ? (gflag != nullptr && *gflag ? "ours (second form)" : "ours") : "native");+ });+ return ok[k];+#else+ (void) type;+ return false;+#endif+}+} // namespace++bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {+ kc::NativeFmt n;+ if (!kc::native_fmt(tg, td, n_embd, n_ff, n, err)) return false;+ f.tg = tg;+ f.td = td;+ f.n_embd = n_embd;+ f.n_ff = n_ff;+ f.gu_act = n.gu_act;+ f.d_act = n.d_act;+ f.gu_row = n.gu_row;+ f.d_row = n.d_row;+ f.act_bytes = n.act_bytes;+ f.h_bytes = n.h_bytes;+ return true;+}++void iq_cpu_quant(int act, const float* x, int k, void* dst) {+ kc::NativeFmt n;+ n.gu_act = act;+ n.n_embd = k;+ kc::native_quant_act(n, x, dst);+}++void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt,+ float* const* ff, int r0, int r1, float limit) {+ if (r1 > r0 && own(f.tg)) { // (native_gu_rows_split's epilogue, on the same values)+ const int nr = r1 - r0;+ thread_local std::vector<float> buf;+ thread_local std::vector<float*> go, uo;+ buf.resize((size_t) 2 * nt * nr);+ go.resize((size_t) nt);+ uo.resize((size_t) nt);+ for (int t = 0; t < nt; ++t) {+ go[(size_t) t] = buf.data() + (size_t) t * nr;+ uo[(size_t) t] = buf.data() + (size_t) (nt + t) * nr;+ }+ own_rows(f.tg, gate, f.gu_row, f.n_embd, act, nt, go.data(), r0, r1);+ own_rows(f.tg, up, f.gu_row, f.n_embd, act, nt, uo.data(), r0, r1);+ for (int t = 0; t < nt; ++t)+ for (int r = 0; r < nr; ++r) {+ float g = go[(size_t) t][r], u = uo[(size_t) t][r];+ if (limit > 0.0f) {+ g = std::fmin(g, limit);+ u = std::fmin(std::fmax(u, -limit), limit);+ }+ ff[t][r0 + r] = (g / (1.f + std::exp(-g))) * u;+ }+ return;+ }+ kc::native_gu_rows_split(native(f), gate, up, act, nt, ff, r0, r1,+ limit > 0.0f ? limit : std::numeric_limits<float>::infinity());+}++void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {+ if (r1 > r0 && own(f.td)) {+ thread_local std::vector<float*> o;+ o.resize((size_t) nt);+ for (int t = 0; t < nt; ++t) o[(size_t) t] = out[t] + r0;+ own_rows(f.td, down, f.d_row, f.n_ff, hq, nt, o.data(), r0, r1);+ return;+ }+ kc::native_down_rows_split(native(f), down, hq, nt, out, r0, r1);+}++bool iq_cpu_own_rows(int type) { return own(type); }+#else+bool iq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {+ err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts' CPU lane is not in it";+ return false;+}+void iq_cpu_quant(int, const float*, int, void*) {}+void iq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int,+ float) {}+void iq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}+bool iq_cpu_own_rows(int) { return false; }+#endif++} // namespace strata::ds4::ggufdiff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cunew file mode 100644index 000000000..688fe1701--- /dev/null+++ b/src/dsv41/ds4_kernels.cu@@ -0,0 +1,2060 @@+// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary+// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.+#include "ds4_kernels.cuh"+#include "ds4_numerics.cuh"+#include "ds4_q6k.cuh"++#include <cuda_fp16.h>++#include <algorithm>+#include <atomic>+#include <cfloat>+#include <cstdio>++namespace strata::ds4 {+namespace {++constexpr int kThreads = 256;++__device__ __forceinline__ float bf16f(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }++__device__ __forceinline__ uint16_t f2bf16(float f) { // round to nearest even (the values here are exact anyway)+ uint32_t u = __float_as_uint(f);+ if ((u & 0x7f800000u) == 0x7f800000u) return (uint16_t) (u >> 16);+ u += 0x7fffu + ((u >> 16) & 1u);+ return (uint16_t) (u >> 16);+}++__device__ __forceinline__ float warp_sum(float v) {+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);+ return v;+}+__device__ __forceinline__ float warp_max(float v) {+ for (int o = 16; o > 0; o >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, o));+ return v;+}++// block-wide sum for blockDim.x <= 1024 (a multiple of 32); every thread gets the result+__device__ float block_sum(float v) {+ __shared__ float red[32];+ __shared__ float total;+ const int lane = threadIdx.x & 31, w = threadIdx.x >> 5;+ v = warp_sum(v);+ if (lane == 0) red[w] = v;+ __syncthreads();+ if (w == 0) {+ float t = lane < (int) (blockDim.x >> 5) ? red[lane] : 0.0f;+ t = warp_sum(t);+ if (lane == 0) total = t;+ }+ __syncthreads();+ const float r = total;+ __syncthreads();+ return r;+}++// 2^ceil(log2(x)) for a positive normal x (the reference's fast_round_scale)+__device__ __forceinline__ float pow2_ceil(float x) {+ const uint32_t b = __float_as_uint(x);+ const int e = (int) ((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);+ return __uint_as_float((uint32_t) (e + 127) << 23);+}++// a load of memory another device or the host writes, never from a stale cache line (CUDA's __ldcv; HIP: volatile+// 32-bit loads)+template <class T>+__device__ __forceinline__ T ld_fresh(const T* p) {+#if defined(STRATA_USE_HIP)+ T v;+ const volatile uint32_t* s = (const volatile uint32_t*) p;+ uint32_t* d = (uint32_t*) &v;+#pragma unroll+ for (int i = 0; i < (int) (sizeof(T) / 4); ++i) d[i] = s[i];+ return v;+#else+ return __ldcv(p);+#endif+}++__device__ __forceinline__ float fp8_rt(float v) { return e4m3_round(v); } // E4M3, nearest even (input clamped)++__device__ __forceinline__ float e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even+ const float a = fabsf(v);+ float r;+ if (a <= 0.25f) r = 0.0f;+ else if (a < 0.75f) r = 0.5f;+ else if (a <= 1.25f) r = 1.0f;+ else if (a < 1.75f) r = 1.5f;+ else if (a <= 2.5f) r = 2.0f;+ else if (a < 3.5f) r = 3.0f;+ else if (a <= 5.0f) r = 4.0f;+ else r = 6.0f;+ return copysignf(r, v);+}++// ---------------------------------------------------------------- the caches' packed rows (kv_row_bytes)+// the E2M1 code of a value e2m1_rt returned (bit 3 the sign, -0 kept; magnitudes 0 .5 1 1.5 2 3 4 6 as 0 .. 7)+__device__ __forceinline__ uint32_t e2m1_code(float q) {+ const float a = fabsf(q);+ const uint32_t m = a <= 1.5f ? (uint32_t) (a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;+ return m | ((__float_as_uint(q) >> 31) << 3);+}+// the E4M3 code of v / 2^k for a v whose quotient is an E4M3 value (|v / 2^k| <= 448, at most 4 significant bits, a+// multiple of 2^-9; v 0 or a normal F32): integer arithmetic, exact, -0 kept+__device__ __forceinline__ uint32_t e4m3_code(float v, int k = 0) {+ const uint32_t b = __float_as_uint(v), sg = (b >> 24) & 0x80u, a = b & 0x7fffffffu;+ if (a == 0u) return sg;+ const int e = (int) (a >> 23) - 127 - k; // the quotient's binade+ if (e >= -6) return sg | ((uint32_t) (e + 7) << 3) | ((a >> 20) & 7u);+ return sg | (((a & 0x7fffffu) | 0x800000u) >> (14 - e)); // a subnormal: m 2^-9+}+// an E2M1 code's value times 2^-14, by the conversion of an FP16 (the code's sign, exponent and mantissa bit are an+// FP16's at bits 15 and 11..9: exact, subnormals too) - the readers' decode: (value 2^-14) (scale 2^14) is value * scale+// exactly, both products exact+__device__ __forceinline__ float e2m1_val14(uint32_t c) {+ return __half2float(__ushort_as_half((unsigned short) (((c & 8u) << 12) | ((c & 7u) << 9))));+}+// a UE8M0 scale byte's value (the byte the F32 exponent: 2^(b - 127); 0 -> 0, an unwritten row's)+__device__ __forceinline__ float ue8m0_val(uint32_t b) { return __uint_as_float(b << 23); }++// ---------------------------------------------------------------- embed+__global__ void embed_kernel(const uint16_t* __restrict__ rows, float* __restrict__ h, int dim, int hc) {+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (e >= dim) return;+ const float v = bf16f(rows[(size_t) t * dim + e]);+ for (int s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;+}++// ---------------------------------------------------------------- finish+constexpr int kFinMax = 32; // values a thread holds: len <= 32 * 256++// kPack: the rounded rows packed to f.pdst (f.quant's cache format) instead of written to x or f.dst+template <bool kPack>+__global__ void __launch_bounds__(kThreads) finish_kernel(float* x, int len, long long stride, Finish f) {+ float* v = x + (long long) blockIdx.x * stride;+ float r[kFinMax];+ const int nj = (len + kThreads - 1) / kThreads;+ float ss = 0.0f;+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ r[j] = e < len ? v[e] : 0.0f;+ ss += r[j] * r[j];+ }+ if (f.w != nullptr) {+ const float tot = block_sum(ss);+ const float inv = rsqrtf(tot / (float) len + f.eps);+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ if (e < len) r[j] = r[j] * inv * f.w[e];+ }+ }+ if (f.rd > 0) {+ const int r0 = len - f.rd;+ const int pos = f.pos + (int) (blockIdx.x / f.vpp) * f.pos_step;+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ const float other = __shfl_xor_sync(0xffffffffu, r[j], 1);+ if (e >= r0 && e < len) {+ const int i = (e - r0) >> 1;+ const float ang = (float) pos * f.freqs[i];+ const float c = cosf(ang);+ const float sn = f.inverse ? -sinf(ang) : sinf(ang);+ if (((e - r0) & 1) == 0) r[j] = r[j] * c - other * sn; // re = a c - b s+ else r[j] = other * sn + r[j] * c; // im = a s + b c+ }+ }+ }+ if (f.quant != 0 && !kPack) {+ const int g = f.quant == 2 ? 16 : 32;+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ float a = e < len ? fabsf(r[j]) : 0.0f;+ for (int o = g / 2; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o));+ if (e >= len) continue;+ if (f.quant == 1) {+ const float s = pow2_ceil(fmaxf(a, 1e-4f) * (1.0f / 448.0f));+ r[j] = fp8_rt(fminf(fmaxf(r[j] / s, -448.0f), 448.0f)) * s;+ } else if (f.quant == 2) {+ const float s = fp8_rt(fmaxf(a, 6.0f * 0.001953125f) / 6.0f);+ r[j] = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f)) * s;+ } else {+ const float s = pow2_ceil(fmaxf(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ r[j] = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f)) * s;+ }+ }+ }+ if (kPack) { // the same roundings, written as their codes and each group's scale (the readers' q * s: r exactly)+ const int g = f.quant == 2 ? 16 : 32;+ uint8_t* pr = f.pdst + (long long) blockIdx.x * f.pdst_stride;+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ float a = e < len ? fabsf(r[j]) : 0.0f;+ for (int o = g / 2; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o));+ float q = 0.0f, s = 1.0f;+ if (e < len) {+ if (f.quant == 1) {+ s = pow2_ceil(fmaxf(a, 1e-4f) * (1.0f / 448.0f));+ q = fp8_rt(fminf(fmaxf(r[j] / s, -448.0f), 448.0f));+ } else if (f.quant == 2) {+ s = fp8_rt(fmaxf(a, 6.0f * 0.001953125f) / 6.0f);+ q = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f));+ } else {+ s = pow2_ceil(fmaxf(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ q = e2m1_rt(fminf(fmaxf(r[j] / s, -6.0f), 6.0f));+ }+ r[j] = q * s;+ }+ if (f.quant == 1) {+ if (e < len) pr[e] = (uint8_t) e4m3_code(q);+ } else {+ const uint32_t c = e2m1_code(q);+ const uint32_t hi = __shfl_down_sync(0xffffffffu, c, 1); // (value e + 1: the next lane's)+ if (e < len && (e & 1) == 0) pr[e >> 1] = (uint8_t) (c | (hi << 4));+ }+ if (e < len && e % g == 0)+ pr[(f.quant == 1 ? len : len / 2) + e / g] =+ (uint8_t) (f.quant == 2 ? e4m3_code(s) : __float_as_uint(s) >> 23);+ }+ }+ uint16_t* d = f.dst != nullptr ? f.dst + (long long) blockIdx.x * f.dst_stride : nullptr;+#pragma unroll+ for (int j = 0; j < kFinMax; ++j) {+ if (j >= nj) break;+ const int e = threadIdx.x + j * kThreads;+ if (e >= len) continue;+ if (d != nullptr) d[e] = f2bf16(r[j]);+ else if (!kPack) v[e] = r[j];+ }+}++// ---------------------------------------------------------------- hyper-connections+__global__ void __launch_bounds__(kThreads) hc_mix_kernel(const float* __restrict__ h, const float* __restrict__ fn,+ int hcdim, float eps, float* __restrict__ mixes) {+ const int m = blockIdx.x, t = blockIdx.y;+ const float* w = fn + (size_t) m * hcdim;+ const float* x = h + (size_t) t * hcdim;+ float acc = 0.0f, ss = 0.0f;+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {+ const float4 hv = *(const float4*) (x + k);+ const float4 wv = *(const float4*) (w + k);+ acc += hv.x * wv.x + hv.y * wv.y + hv.z * wv.z + hv.w * wv.w;+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ acc = block_sum(acc);+ ss = block_sum(ss);+ if (threadIdx.x == 0) mixes[(size_t) t * 32 + m] = acc * rsqrtf(ss / (float) hcdim + eps);+}++// a warp a token: lane j * hc + k holds comb[j][k] (hc <= 4: lanes 0..15; wider streams fall back to lane 0 alone)+__global__ void hc_split_kernel(const float* __restrict__ mixes, const float* __restrict__ scale,+ const float* __restrict__ base, int hc, int iters, float eps, float* pre, float* post,+ float* comb) {+ const int t = blockIdx.x, lane = threadIdx.x;+ const float* mx = mixes + (size_t) t * 32;+ pre += (size_t) t * 8;+ post += (size_t) t * 8;+ comb += (size_t) t * 64;+ if (lane < hc) {+ pre[lane] = 1.0f / (1.0f + expf(-(mx[lane] * scale[0] + base[lane]))) + eps;+ post[lane] = 2.0f / (1.0f + expf(-(mx[hc + lane] * scale[1] + base[hc + lane])));+ }+ if (hc == 4) {+ // rows are the 4-lane groups (j = lane / 4), columns the lanes 4 apart (k = lane % 4)+ const bool on = lane < 16;+ float c = on ? mx[8 + lane] * scale[2] + base[8 + lane] : -FLT_MAX;+ auto row_sum = [&](float v) {+ v += __shfl_xor_sync(0xffffffffu, v, 1);+ return v + __shfl_xor_sync(0xffffffffu, v, 2);+ };+ auto col_sum = [&](float v) {+ v += __shfl_xor_sync(0xffffffffu, v, 4);+ return v + __shfl_xor_sync(0xffffffffu, v, 8);+ };+ float m = fmaxf(c, __shfl_xor_sync(0xffffffffu, c, 1));+ m = fmaxf(m, __shfl_xor_sync(0xffffffffu, m, 2));+ c = on ? expf(c - m) : 0.0f;+ c = c / row_sum(c) + eps; // softmax over each row, + eps+ c = on ? c : 0.0f;+ c = c / (col_sum(c) + eps); // each destination column over its sources+ for (int it = 1; it < iters; ++it) {+ c = c / (row_sum(c) + eps);+ c = c / (col_sum(c) + eps);+ }+ if (on) comb[lane] = c;+ return;+ }+ if (lane != 0) return;+ float c[8][8];+ for (int j = 0; j < hc; ++j) {+ float m = -FLT_MAX;+ for (int k = 0; k < hc; ++k) {+ c[j][k] = mx[2 * hc + j * hc + k] * scale[2] + base[2 * hc + j * hc + k];+ m = fmaxf(m, c[j][k]);+ }+ float s = 0.0f;+ for (int k = 0; k < hc; ++k) {+ c[j][k] = expf(c[j][k] - m);+ s += c[j][k];+ }+ for (int k = 0; k < hc; ++k) c[j][k] = c[j][k] / s + eps;+ }+ for (int k = 0; k < hc; ++k) {+ float s = 0.0f;+ for (int j = 0; j < hc; ++j) s += c[j][k];+ for (int j = 0; j < hc; ++j) c[j][k] /= s + eps;+ }+ for (int it = 1; it < iters; ++it) {+ for (int j = 0; j < hc; ++j) {+ float s = 0.0f;+ for (int k = 0; k < hc; ++k) s += c[j][k];+ for (int k = 0; k < hc; ++k) c[j][k] /= s + eps;+ }+ for (int k = 0; k < hc; ++k) {+ float s = 0.0f;+ for (int j = 0; j < hc; ++j) s += c[j][k];+ for (int j = 0; j < hc; ++j) c[j][k] /= s + eps;+ }+ }+ for (int j = 0; j < hc; ++j)+ for (int k = 0; k < hc; ++k) comb[j * hc + k] = c[j][k];+}++__global__ void __launch_bounds__(1024) hc_pre_norm_kernel(const float* __restrict__ h, const float* __restrict__ mix,+ const float* __restrict__ w, int dim, int hc, float eps,+ float* __restrict__ x) {+ const int t = blockIdx.x;+ h += (size_t) t * hc * dim;+ mix += (size_t) t * 8;+ x += (size_t) t * dim;+ float r[8];+ float ss = 0.0f;+ int n = 0;+ for (int e = threadIdx.x; e < dim; e += blockDim.x, ++n) {+ float a = 0.0f;+ for (int s = 0; s < hc; ++s) a += mix[s] * h[(size_t) s * dim + e];+ r[n] = a;+ ss += a * a;+ }+ const float inv = rsqrtf(block_sum(ss) / (float) dim + eps);+ n = 0;+ for (int e = threadIdx.x; e < dim; e += blockDim.x, ++n) x[e] = w != nullptr ? r[n] * inv * w[e] : r[n];+}++__global__ void hc_post_kernel(const float* __restrict__ y, const float* __restrict__ res,+ const float* __restrict__ post, const float* __restrict__ comb, int dim, int hc,+ float* __restrict__ out) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (i >= dim * hc) return;+ const int k = i / dim, e = i - k * dim;+ const float* r = res + (size_t) t * hc * dim;+ float a = post[(size_t) t * 8 + k] * y[(size_t) t * dim + e];+ for (int j = 0; j < hc; ++j) a += comb[(size_t) t * 64 + j * hc + k] * r[(size_t) j * dim + e];+ out[(size_t) t * hc * dim + i] = a;+}++// ---------------------------------------------------------------- gemv over F16 weights, a warp a row+__global__ void __launch_bounds__(kThreads) gemv_f16_kernel(const uint16_t* __restrict__ W, const float* __restrict__ x,+ int n, int k, float alpha, float* __restrict__ y) {+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ const int lane = threadIdx.x & 31, t = blockIdx.y;+ if (row >= n) return;+ const uint16_t* w = W + (size_t) row * k;+ const float* xv = x + (size_t) t * k;+ float acc = 0.0f;+ for (int c = lane * 8; c < k; c += 32 * 8) {+ const uint4 q = *(const uint4*) (w + c);+ const __half2* hp = (const __half2*) &q;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 f = __half22float2(hp[i]);+ acc += f.x * xv[c + 2 * i] + f.y * xv[c + 2 * i + 1];+ }+ }+ acc = warp_sum(acc);+ if (lane == 0) y[(size_t) t * n + row] = alpha * acc;+}+// ... for T activations (rows ldx apart, 16-byte aligned; outputs ldy apart), kRowsT of them a block (grid y): each+// weight row read once for the block's activations, theirs in float4s - each activation's sum gemv_f16_kernel's (the+// same products in the same order), so its bits too. (A block for all of them, 16 at most, was slower than a launch+// a token: its accumulators and guards kept the loop from unrolling - 66 against 29 us at 5 tokens on an RTX 5060 Ti,+// this 9)+constexpr int kRowsT = 2;+__global__ void __launch_bounds__(kThreads) gemv_f16_rows_kernel(const uint16_t* __restrict__ W,+ const float* __restrict__ x, size_t ldx, int n, int k,+ int T, float alpha, float* __restrict__ y, size_t ldy) {+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ const int lane = threadIdx.x & 31;+ const int t0 = blockIdx.y * kRowsT, nt = min(kRowsT, T - t0);+ if (row >= n) return;+ const uint16_t* w = W + (size_t) row * k;+ float acc[kRowsT];+#pragma unroll+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;+ for (int c = lane * 8; c < k; c += 32 * 8) {+ const uint4 q = *(const uint4*) (w + c);+ const __half2* hp = (const __half2*) &q;+ float xv[kRowsT][8];+#pragma unroll+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ const float* xp = x + (size_t) (t0 + t) * ldx + c;+ const float4 a = *(const float4*) xp, b = *(const float4*) (xp + 4);+ xv[t][0] = a.x, xv[t][1] = a.y, xv[t][2] = a.z, xv[t][3] = a.w;+ xv[t][4] = b.x, xv[t][5] = b.y, xv[t][6] = b.z, xv[t][7] = b.w;+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 f = __half22float2(hp[i]);+#pragma unroll+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];+ }+ }+#pragma unroll+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ const float a = warp_sum(acc[t]);+ if (lane == 0) y[(size_t) (t0 + t) * ldy + row] = alpha * a;+ }+}++// ---------------------------------------------------------------- gemv over F32 weights, a warp a row (the router)+__global__ void __launch_bounds__(kThreads) gemv_f32_kernel(const float* __restrict__ W, const float* __restrict__ x,+ int n, int k, float* __restrict__ y, int ldy) {+ const int row = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ const int lane = threadIdx.x & 31, t = blockIdx.y;+ if (row >= n) return;+ const float4* w = (const float4*) (W + (size_t) row * k);+ const float4* xv = (const float4*) (x + (size_t) t * k);+ float acc = 0.0f;+ for (int c = lane; c < k / 4; c += 32) {+ const float4 a = w[c], b = xv[c];+ acc = fmaf(a.x, b.x, acc);+ acc = fmaf(a.y, b.y, acc);+ acc = fmaf(a.z, b.z, acc);+ acc = fmaf(a.w, b.w, acc);+ }+ acc = warp_sum(acc);+ if (lane == 0) y[(size_t) t * ldy + row] = acc;+}++// ---------------------------------------------------------------- the router+__global__ void __launch_bounds__(512) route_kernel(const float* __restrict__ logits, const float* __restrict__ bias,+ int n, int k, float route_scale, int* idx, float* w,+ const float* __restrict__ bias_vl, const VlBits vl) {+ __shared__ float sc[1024], bs[1024];+ __shared__ float bv[32];+ __shared__ int bi[32];+ __shared__ int pick[16];+ const int t = blockIdx.x;+ logits += (size_t) t * 1024;+ if (bias_vl != nullptr && ((vl.w[t >> 5] >> (t & 31)) & 1u)) bias = bias_vl;+ for (int i = threadIdx.x; i < n; i += blockDim.x) {+ const float l = logits[i];+ const float sp = l > 20.0f ? l : log1pf(expf(l));+ sc[i] = sqrtf(sp);+ bs[i] = sc[i] + bias[i];+ }+ __syncthreads();+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ for (int r = 0; r < k; ++r) {+ float best = -FLT_MAX;+ int bix = 0x7fffffff;+ for (int i = threadIdx.x; i < n; i += blockDim.x)+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;+ }+ if (lane == 0) bv[wp] = best, bi[wp] = bix;+ __syncthreads();+ if (threadIdx.x == 0) {+ float b = -FLT_MAX;+ int ix = 0x7fffffff;+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];+ pick[r] = ix;+ bs[ix] = -FLT_MAX;+ }+ __syncthreads();+ }+ if (threadIdx.x == 0) {+ float s = 0.0f;+ for (int r = 0; r < k; ++r) s += sc[pick[r]];+ for (int r = 0; r < k; ++r) {+ idx[(size_t) t * kPicks + r] = pick[r];+ w[(size_t) t * kPicks + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;+ }+ }+}++__global__ void swiglu_kernel(const float* __restrict__ g, const float* __restrict__ u, int n, float lim,+ float* __restrict__ h) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ float gv = g[i], uv = u[i];+ if (lim > 0.0f) {+ gv = fminf(gv, lim);+ uv = fminf(fmaxf(uv, -lim), lim);+ }+ h[i] = gv / (1.0f + expf(-gv)) * uv;+}++// ---------------------------------------------------------------- the compressor (ratio > 1), a window in order+__global__ void __launch_bounds__(512) compress_kernel(const float* __restrict__ kv, const float* __restrict__ score,+ float* kv_state, float* score_state, int ratio, int p0, int T,+ int d, const float* __restrict__ w, float eps,+ float* __restrict__ latent) {+ int g = 0;+ for (int t = 0; t < T; ++t) {+ const int slot = (p0 + t) % ratio;+ const bool pool = (p0 + t + 1) % ratio == 0;+ for (int e = threadIdx.x; e < d; e += blockDim.x) {+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];+ }+ __syncthreads();+ if (!pool) continue;+ // (kv * softmax(score, over the slots)).sum: each slot's probability first, then the sum, as torch does+ float val[2] = {0.0f, 0.0f}; // d <= 2 * blockDim+ float ss = 0.0f;+ int n = 0;+ for (int e = threadIdx.x; e < d; e += blockDim.x, ++n) {+ float m = -FLT_MAX;+ for (int s = 0; s < ratio; ++s) m = fmaxf(m, score_state[(size_t) s * d + e]);+ float den = 0.0f;+ for (int s = 0; s < ratio; ++s) den += expf(score_state[(size_t) s * d + e] - m);+ float acc = 0.0f;+ for (int s = 0; s < ratio; ++s)+ acc += kv_state[(size_t) s * d + e] * (expf(score_state[(size_t) s * d + e] - m) / den);+ val[n] = acc;+ ss += acc * acc;+ }+ const float inv = rsqrtf(block_sum(ss) / (float) d + eps);+ n = 0;+ for (int e = threadIdx.x; e < d; e += blockDim.x, ++n) latent[(size_t) g * d + e] = val[n] * inv * w[e];+ ++g;+ __syncthreads();+ }+}++// ---------------------------------------------------------------- per-query sizes+__device__ __forceinline__ void rows_at(const Rows& r, int t, int& n, int& k) {+ if (r.gen == 0) {+ n = r.n[t];+ k = r.k[t];+ return;+ }+ const int nn = (int) ((r.p0 + t + 1) / r.R);+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);+ switch (r.gen) {+ case 1: n = nn, k = min(r.kcap, nn); break;+ case 2: n = nn, k = cb; break;+ case 3: n = cb * r.blk, k = min(r.kcap, nn); break;+ default: n = (nn + r.blk - 1) / r.blk, k = r.kcap; break;+ }+}++// ---------------------------------------------------------------- the indexer+// a thread a position: its key row (KF: kKvFp4p packed, or BF16) against every head's query (shared memory), relu,+// weighted sum+template <int HD, int KF>+__global__ void __launch_bounds__(kThreads) index_score_kernel(const float* __restrict__ q, const float* __restrict__ w,+ const uint8_t* __restrict__ kc, int heads, int n,+ const int* __restrict__ cand, int cand_stride, int blk,+ Rows lim, float* __restrict__ out,+ long long out_stride) {+ extern __shared__ float qs[]; // heads * HD, then heads weights+ const int t = blockIdx.y;+ for (int i = threadIdx.x; i < heads * HD; i += blockDim.x) qs[i] = q[(size_t) t * heads * HD + i];+ for (int i = threadIdx.x; i < heads; i += blockDim.x) qs[heads * HD + i] = w[(size_t) t * 64 + i];+ __syncthreads();+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ float* o = out + (long long) t * out_stride;+ int ln, lk;+ rows_at(lim, t, ln, lk);+ int p = i;+ if (cand != nullptr) {+ if (i >= lk * blk) {+ o[i] = -INFINITY;+ return;+ }+ p = cand[(size_t) t * cand_stride + i / blk] * blk + i % blk;+ }+ if (p >= ln) {+ o[i] = -INFINITY;+ return;+ }+ const uint8_t* kr = kc + (size_t) p * kv_row_bytes(KF, HD);+ float acc[32];+#pragma unroll+ for (int h = 0; h < 32; ++h) acc[h] = 0.0f;+#pragma unroll 2+ for (int c = 0; c < HD; c += 8) {+ float kv[8];+ if (KF == kKvFp4p) { // 8 codes in 4 bytes, the group's power-of-two scale: the values the finish rounded to+ const uint32_t cw = *(const uint32_t*) (kr + c / 2);+ const float ks = ue8m0_val(kr[HD / 2 + c / 32]);+#pragma unroll+ for (int x = 0; x < 8; ++x) kv[x] = e2m1_val14(cw >> (4 * x)) * 16384.0f * ks; // (any 2^k: exact)+ } else {+ const uint4 raw = *(const uint4*) (kr + c * 2);+ const uint16_t* b = (const uint16_t*) &raw;+#pragma unroll+ for (int x = 0; x < 8; ++x) kv[x] = bf16f(b[x]);+ }+#pragma unroll+ for (int h = 0; h < 32; ++h) {+ if (h >= heads) break;+ const float* qh = qs + h * HD + c;+#pragma unroll+ for (int x = 0; x < 8; ++x) acc[h] += qh[x] * kv[x];+ }+ }+ float s = 0.0f;+#pragma unroll+ for (int h = 0; h < 32; ++h) {+ if (h >= heads) break;+ s += fmaxf(acc[h], 0.0f) * qs[heads * HD + h];+ }+ o[i] = s;+}++__device__ __forceinline__ uint32_t ord_key(float f) {+ const uint32_t b = __float_as_uint(f);+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);+}++// a block a row: the k-th largest key by four 8-bit radix passes, then everything above it and enough of its ties+__global__ void __launch_bounds__(1024) topk_kernel(const float* __restrict__ vals, long long stride, Rows r,+ const int* __restrict__ cand, int cand_stride, int blk,+ int* __restrict__ out, int out_stride) {+ __shared__ unsigned hist[256];+ __shared__ uint32_t prefix_s, mask_s;+ __shared__ int remaining_s;+ __shared__ int cnt_gt, cnt_eq;+ const int t = blockIdx.x;+ int n, k;+ rows_at(r, t, n, k);+ vals += (long long) t * stride;+ out += (size_t) t * out_stride;+ const int* cd = cand != nullptr ? cand + (size_t) t * cand_stride : nullptr;+ if (n <= 0) return;+ if (k >= n) {+ for (int i = threadIdx.x; i < n; i += blockDim.x) out[i] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;+ return;+ }+ if (threadIdx.x == 0) {+ prefix_s = 0u;+ mask_s = 0u;+ remaining_s = k;+ cnt_gt = 0;+ cnt_eq = 0;+ }+ __syncthreads();+ for (int shift = 24; shift >= 0; shift -= 8) {+ for (int i = threadIdx.x; i < 256; i += blockDim.x) hist[i] = 0u;+ __syncthreads();+ const uint32_t prefix = prefix_s, mask = mask_s;+ for (int i = threadIdx.x; i < n; i += blockDim.x) {+ const uint32_t u = ord_key(vals[i]);+ if ((u & mask) == prefix) atomicAdd(&hist[(u >> shift) & 0xffu], 1u);+ }+ __syncthreads();+ if (threadIdx.x == 0) {+ const int rem = remaining_s;+ int cum = 0;+ for (int d = 255; d >= 0; --d) {+ const int h = (int) hist[d];+ if (cum + h >= rem) {+ prefix_s = prefix | ((uint32_t) d << shift);+ mask_s = mask | (0xffu << shift);+ remaining_s = rem - cum;+ break;+ }+ cum += h;+ }+ }+ __syncthreads();+ }+ const uint32_t thr = prefix_s;+ const int remaining = remaining_s;+ const int n_gt = k - remaining;+ // the picks in index order (a scan a pass of 1024 values), so the attention sums them in the same order every run+ // and the ties kept are the lowest positions+ __shared__ int wgt[32], weq[32];+ const int lane = threadIdx.x & 31, wid = threadIdx.x >> 5, nw = blockDim.x >> 5;+ const unsigned below = (1u << lane) - 1u;+ for (int i0 = 0; i0 < n; i0 += blockDim.x) {+ const int i = i0 + threadIdx.x;+ const uint32_t u = i < n ? ord_key(vals[i]) : 0u;+ const bool gt = i < n && u > thr, eq = i < n && u == thr;+ const unsigned bg = __ballot_sync(0xffffffffu, gt), be = __ballot_sync(0xffffffffu, eq);+ if (lane == 0) {+ wgt[wid] = __popc(bg);+ weq[wid] = __popc(be);+ }+ __syncthreads();+ if (wid == 0) {+ const int g0 = lane < nw ? wgt[lane] : 0, e0 = lane < nw ? weq[lane] : 0;+ int gs = g0, es = e0;+ for (int o = 1; o < 32; o <<= 1) {+ const int a = __shfl_up_sync(0xffffffffu, gs, o), b = __shfl_up_sync(0xffffffffu, es, o);+ if (lane >= o) {+ gs += a;+ es += b;+ }+ }+ if (lane < nw) {+ wgt[lane] = cnt_gt + gs - g0;+ weq[lane] = cnt_eq + es - e0;+ }+ __syncwarp();+ if (lane == 31) {+ cnt_gt += gs;+ cnt_eq += es;+ }+ }+ __syncthreads();+ int slot = -1;+ if (gt) slot = wgt[wid] + __popc(bg & below);+ else if (eq) {+ const int e = weq[wid] + __popc(be & below);+ if (e < remaining) slot = n_gt + e;+ }+ if (slot >= 0) out[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;+ __syncthreads();+ }+}++__global__ void block_max_kernel(const float* __restrict__ score, long long stride, Rows r, int blk,+ float* __restrict__ bm, long long bm_stride) {+ const int b = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ int n, k_;+ rows_at(r, t, n, k_);+ const int nb = (n + blk - 1) / blk;+ if (b >= nb) return;+ float m = -INFINITY;+ for (int j = 0; j < blk; ++j) {+ const int p = b * blk + j;+ if (p < n) m = fmaxf(m, score[(long long) t * stride + p]);+ }+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in+ bm[(long long) t * bm_stride + b] = m;+}++// ---------------------------------------------------------------- sparse attention, a block a (head, query)+constexpr int kAttTile = 64;++// kContig: query t's selected rows are ckv's rows t * sel_stride .. (the scratch of rows decoded once a query) - read+// without sel, a warp's rows of a tile loaded kPre at a time before their dots (0: as the BF16 kernel); the+// arithmetic the same+template <bool kContig, int kPre = 0>+__global__ void __launch_bounds__(kThreads) sparse_attn_kernel(const float* __restrict__ q,+ const uint16_t* __restrict__ win,+ const uint16_t* __restrict__ ckv,+ const int* __restrict__ sel, int sel_stride,+ AttnRows ar, const float* __restrict__ sink, int heads,+ int d, float scale, float* __restrict__ out) {+ // d == 512: a thread owns 2 output values, a lane 16 of a row's dot+ const int h = blockIdx.x, t = blockIdx.y;+ int n_win, n_sel, w0;+ if (ar.gen) {+ const long long ta = ar.t0 + t, pa = ar.p0 + ta;+ const long long first = max((long long) ar.win - ar.p0, ta + 1); // the window buffer row of max(0, pa - win + 1)+ w0 = (int) first;+ n_win = (int) (ar.win + ta - first + 1);+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;+ } else {+ n_win = ar.nwin[t], n_sel = ar.nsel[t], w0 = ar.win0[t];+ }+ const uint16_t* wbase = win + (size_t) w0 * d;+ const int* sl = sel + (size_t) t * sel_stride;+ __shared__ float qs[512];+ __shared__ float sc[kAttTile];+ __shared__ const uint16_t* rows[kAttTile];+ __shared__ float m_run_s, tile_scale_s, den_s;+ for (int i = threadIdx.x; i < d; i += blockDim.x) qs[i] = q[((size_t) t * heads + h) * d + i];+ if (threadIdx.x == 0) {+ m_run_s = -1e30f;+ den_s = 0.0f;+ }+ __syncthreads();+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ float o0 = 0.0f, o1 = 0.0f;+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += kAttTile) {+ const int nt = min(kAttTile, total - t0);+ if (threadIdx.x < nt) {+ const int j = t0 + threadIdx.x;+ rows[threadIdx.x] = j < n_win ? wbase + (size_t) j * d+ : ckv + (kContig ? (size_t) t * sel_stride + (j - n_win) : (size_t) sl[j - n_win]) * d;+ }+ __syncthreads();+ if (kContig && kPre > 0) { // the warp's rows of the tile (wp, wp + 8, ..), kPre loads at a time, then dots+#pragma unroll+ for (int k0 = 0; k0 < kAttTile / 8; k0 += (kPre > 0 ? kPre : 1)) {+ uint4 ra[kPre > 0 ? kPre : 1], rb[kPre > 0 ? kPre : 1];+#pragma unroll+ for (int k = 0; k < kPre; ++k) {+ const int jj = wp + 8 * (k0 + k);+ if (jj < nt) {+ ra[k] = *(const uint4*) (rows[jj] + lane * 16);+ rb[k] = *(const uint4*) (rows[jj] + lane * 16 + 8);+ }+ }+#pragma unroll+ for (int k = 0; k < kPre; ++k) {+ const int jj = wp + 8 * (k0 + k);+ if (jj < nt) {+ const uint16_t* pa = (const uint16_t*) &ra[k];+ const uint16_t* pb = (const uint16_t*) &rb[k];+ float s = 0.0f;+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + x] * bf16f(pa[x]);+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + 8 + x] * bf16f(pb[x]);+ s = warp_sum(s);+ if (lane == 0) sc[jj] = s * scale;+ }+ }+ }+ } else+ for (int jj = wp; jj < nt; jj += kThreads / 32) {+ const uint16_t* rr = rows[jj];+ const uint4 a = *(const uint4*) (rr + lane * 16);+ const uint4 b = *(const uint4*) (rr + lane * 16 + 8);+ const uint16_t* pa = (const uint16_t*) &a;+ const uint16_t* pb = (const uint16_t*) &b;+ float s = 0.0f;+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + x] * bf16f(pa[x]);+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qs[lane * 16 + 8 + x] * bf16f(pb[x]);+ s = warp_sum(s);+ if (lane == 0) sc[jj] = s * scale;+ }+ __syncthreads();+ if (wp == 0) {+ float m = -INFINITY;+ for (int j = lane; j < nt; j += 32) m = fmaxf(m, sc[j]);+ m = warp_max(m);+ const float m_old = m_run_s;+ const float m_new = fmaxf(m_old, m);+ float s = 0.0f;+ for (int j = lane; j < nt; j += 32) {+ const float p = expf(sc[j] - m_new);+ sc[j] = p;+ s += p;+ }+ s = warp_sum(s);+ __syncwarp();+ if (lane == 0) {+ tile_scale_s = expf(m_old - m_new);+ den_s = den_s * tile_scale_s + s;+ m_run_s = m_new;+ }+ }+ __syncthreads();+ const float cs = tile_scale_s;+ o0 *= cs;+ o1 *= cs;+ const int c = threadIdx.x * 2;+ if (kContig && kPre > 0) { // (eight rows' loads at once)+#pragma unroll 8+ for (int j = 0; j < nt; ++j) {+ const uint32_t pr = *(const uint32_t*) (rows[j] + c);+ o0 += sc[j] * bf16f((uint16_t) (pr & 0xffffu));+ o1 += sc[j] * bf16f((uint16_t) (pr >> 16));+ }+ } else+ for (int j = 0; j < nt; ++j) {+ const uint32_t pr = *(const uint32_t*) (rows[j] + c);+ o0 += sc[j] * bf16f((uint16_t) (pr & 0xffffu));+ o1 += sc[j] * bf16f((uint16_t) (pr >> 16));+ }+ __syncthreads();+ }+ const float den = den_s + expf(sink[h] - m_run_s);+ const int c = threadIdx.x * 2;+ float* ov = out + ((size_t) t * heads + h) * d;+ ov[c] = o0 / den;+ ov[c + 1] = o1 / den;+}++// ---------------------------------------------------------------- sparse attention over the packed compressed rows+// G heads of one query a block (8 / G warps a head): each pass stages 16 of a tile's rows in shared memory as BF16+// (the window's rows as they are; the packed rows decoded - exact in BF16), once for all G heads, through one 16-byte+// load a thread. Each head's arithmetic is sparse_attn_kernel's, operation for operation: its tiles of 64 rows, its+// dots by a warp's lanes (16 values each, in order) and the same butterfly, its softmax by one warp the same way, each+// output value's weighted sum by the same thread in row order - the same bits for any G.+constexpr int kStage = 16; // rows a pass stages (a thread: 32 values of one)++// values part * 32 .. + 32 of a packed compressed row (kKvFp4, 512 values) as BF16 into dst[0 .. 4) (exact: the+// values are E2M1 times an E4M3 scale)+__device__ __forceinline__ void unpack32(const uint8_t* rr, int part, uint4* dst) {+ const uint4 cw = *(const uint4*) (rr + part * 16); // 32 codes, two scale groups+ const float k0 = e4m3_value(rr[256 + part * 2]) * 16384.0f, k1 = e4m3_value(rr[256 + part * 2 + 1]) * 16384.0f;+ const uint32_t wv[4] = {cw.x, cw.y, cw.z, cw.w};+ uint32_t o[16];+#pragma unroll+ for (int i = 0; i < 16; ++i) { // values 2i, 2i + 1 (the low and the high nibble of byte i)+ const uint32_t b = wv[i >> 2] >> (8 * (i & 3));+ const float ks = i < 8 ? k0 : k1;+ const float lo = e2m1_val14(b) * ks, hi = e2m1_val14(b >> 4) * ks; // (exact: their BF16 bits)+ o[i] = (__float_as_uint(lo) >> 16) | (__float_as_uint(hi) & 0xffff0000u);+ }+ dst[0] = make_uint4(o[0], o[1], o[2], o[3]);+ dst[1] = make_uint4(o[4], o[5], o[6], o[7]);+ dst[2] = make_uint4(o[8], o[9], o[10], o[11]);+ dst[3] = make_uint4(o[12], o[13], o[14], o[15]);+}++// a few queries' attention (a decode step, a short window): the rows each query selects decoded once to BF16 rows -+// out[t * cap + i] = ckv[sel[t][i]] for i < the query's n_sel - for the BF16 kernel reading them in place (once a+// query, not once a head)+__global__ void __launch_bounds__(kThreads) unpack_sel_kernel(const uint8_t* __restrict__ ckv,+ const int* __restrict__ sel, int sel_stride, AttnRows ar,+ int cap, uint16_t* __restrict__ out) {+ const int t = blockIdx.y, i = blockIdx.x * kStage + (int) (threadIdx.x >> 4), part = threadIdx.x & 15;+ int n_sel;+ if (ar.gen) {+ const long long pa = ar.p0 + ar.t0 + t;+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;+ } else {+ n_sel = ar.nsel[t];+ }+ if (i >= n_sel) return;+ const size_t r = (size_t) t * cap + i;+ unpack32(ckv + (size_t) sel[(size_t) t * sel_stride + i] * kv_row_bytes(kKvFp4, 512), part, (uint4*) (out + r * 512 + part * 32));+}++template <int G>+__global__ void __launch_bounds__(kThreads) sparse_attn_packed_kernel(const float* __restrict__ q,+ const uint16_t* __restrict__ win,+ const uint8_t* __restrict__ ckv,+ const int* __restrict__ sel, int sel_stride,+ AttnRows ar, const float* __restrict__ sink,+ int heads, float scale, float* __restrict__ out) {+ constexpr int d = 512, W = 8 / G; // (kThreads / 32 warps)+ const int hb = blockIdx.x * G, t = blockIdx.y;+ int n_win, n_sel, w0;+ if (ar.gen) {+ const long long ta = ar.t0 + t, pa = ar.p0 + ta;+ const long long first = max((long long) ar.win - ar.p0, ta + 1);+ w0 = (int) first;+ n_win = (int) (ar.win + ta - first + 1);+ n_sel = ar.R > 0 ? (int) min((long long) ar.kcap, (pa + 1) / ar.R) : 0;+ } else {+ n_win = ar.nwin[t], n_sel = ar.nsel[t], w0 = ar.win0[t];+ }+ const uint16_t* wbase = win + (size_t) w0 * d;+ const int* sl = sel + (size_t) t * sel_stride;+ const size_t crow = kv_row_bytes(kKvFp4, d);+ __shared__ __align__(16) uint16_t tile[kStage * d];+ __shared__ float sc[G][kAttTile];+ __shared__ const uint8_t* rows[kAttTile];+ __shared__ float m_run_s[G], tile_scale_s[G], den_s[G];+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5, gw = wp / W, wr = wp % W;+ float qr[16]; // this lane's 16 values of its warp's head+#pragma unroll+ for (int x = 0; x < 16; ++x) qr[x] = q[((size_t) t * heads + hb + gw) * d + lane * 16 + x];+ if (threadIdx.x < G) {+ m_run_s[threadIdx.x] = -1e30f;+ den_s[threadIdx.x] = 0.0f;+ }+ float o0[G], o1[G];+#pragma unroll+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;+ const int c = threadIdx.x * 2;+ const int sr = threadIdx.x >> 4, part = threadIdx.x & 15; // the staging: a thread's row and 32 values+ // rows [s0, s0 + n) of the tile at t0 into tile[] as BF16+ auto stage = [&](int t0, int s0, int n) {+ if (sr >= n) return;+ const int j = s0 + sr;+ uint4* dst = (uint4*) (tile + sr * d + part * 32);+ if (t0 + j < n_win) { // a window row: BF16 already+ const uint4* src = (const uint4*) (rows[j] + part * 64);+ dst[0] = src[0];+ dst[1] = src[1];+ dst[2] = src[2];+ dst[3] = src[3];+ return;+ }+ unpack32(rows[j], part, dst);+ };+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += kAttTile) {+ const int nt = min(kAttTile, total - t0);+ if (threadIdx.x < nt) {+ const int j = t0 + threadIdx.x;+ rows[threadIdx.x] = j < n_win ? (const uint8_t*) (wbase + (size_t) j * d) : ckv + (size_t) sl[j - n_win] * crow;+ }+ __syncthreads();+ for (int s0 = 0; s0 < nt; s0 += kStage) { // the dots+ const int n = min(kStage, nt - s0);+ stage(t0, s0, n);+ __syncthreads();+#if !defined(STRATA_USE_HIP) // (HIP leaves a loop with shuffles and a remainder rolled anyway)+#pragma unroll 2+#endif+ for (int r = wr; r < n; r += W) { // (two rows' chains at once)+ const uint16_t* vr = tile + r * d + lane * 16;+ const uint4 a = *(const uint4*) vr;+ const uint4 b = *(const uint4*) (vr + 8);+ const uint16_t* pa = (const uint16_t*) &a;+ const uint16_t* pb = (const uint16_t*) &b;+ float s = 0.0f;+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qr[x] * bf16f(pa[x]);+#pragma unroll+ for (int x = 0; x < 8; ++x) s += qr[8 + x] * bf16f(pb[x]);+ s = warp_sum(s);+ if (lane == 0) sc[gw][s0 + r] = s * scale;+ }+ __syncthreads();+ }+ if (wp < G) { // head wp's softmax over the tile+ float* scg = sc[wp];+ float m = -INFINITY;+ for (int j = lane; j < nt; j += 32) m = fmaxf(m, scg[j]);+ m = warp_max(m);+ const float m_old = m_run_s[wp];+ const float m_new = fmaxf(m_old, m);+ float s = 0.0f;+ for (int j = lane; j < nt; j += 32) {+ const float p = expf(scg[j] - m_new);+ scg[j] = p;+ s += p;+ }+ s = warp_sum(s);+ __syncwarp();+ if (lane == 0) {+ tile_scale_s[wp] = expf(m_old - m_new);+ den_s[wp] = den_s[wp] * tile_scale_s[wp] + s;+ m_run_s[wp] = m_new;+ }+ }+ __syncthreads();+#pragma unroll+ for (int g = 0; g < G; ++g) {+ const float cs = tile_scale_s[g];+ o0[g] *= cs;+ o1[g] *= cs;+ }+ for (int s0 = 0; s0 < nt; s0 += kStage) { // the weighted sums+ const int n = min(kStage, nt - s0);+ stage(t0, s0, n);+ __syncthreads();+ for (int r = 0; r < n; ++r) {+ const uint32_t pr = *(const uint32_t*) (tile + r * d + c);+ const float v0 = bf16f((uint16_t) (pr & 0xffffu)), v1 = bf16f((uint16_t) (pr >> 16));+#pragma unroll+ for (int g = 0; g < G; ++g) {+ o0[g] += sc[g][s0 + r] * v0;+ o1[g] += sc[g][s0 + r] * v1;+ }+ }+ __syncthreads();+ }+ }+#pragma unroll+ for (int g = 0; g < G; ++g) {+ const float den = den_s[g] + expf(sink[hb + g] - m_run_s[g]);+ float* ov = out + ((size_t) t * heads + hb + g) * d;+ ov[c] = o0[g] / den;+ ov[c + 1] = o1[g] / den;+ }+}++// the ring's rows are kKvFp8: E4M3 codes, a UE8M0 scale per 32 (decoded: exactly the BF16 values they were made from)+__global__ void ring_to_ext_kernel(const uint8_t* __restrict__ ring, uint16_t* __restrict__ ext, long long p0, int n,+ int d) {+ const int r = blockIdx.x;+ const long long p = p0 - n + r;+ const uint8_t* row = ring + (size_t) (p >= 0 ? p % n : 0) * kv_row_bytes(kKvFp8, d);+ for (int e = threadIdx.x; e < d; e += blockDim.x)+ ext[(size_t) r * d + e] = p >= 0 ? f2bf16(e4m3_value(row[e]) * ue8m0_val(row[d + e / 32])) : (uint16_t) 0;+}++// a window row's BF16 values (FP8 roundings: each an E4M3 value times its group's power of two) as E4M3 codes at the+// smallest power-of-two scale 2^k their group fits under (max |v| <= 448 * 2^k = 1.75 * 2^(8 + k)): never above the+// rounding's own scale, so each v / 2^k is that E4M3 value times a power of two >= 1, at most 448 - an E4M3 value again+__global__ void __launch_bounds__(kThreads) ext_to_ring_kernel(const uint16_t* __restrict__ ext,+ uint8_t* __restrict__ ring, long long p0, int n, int T,+ int d, uint8_t* __restrict__ backup) {+ const int t = blockIdx.x;+ if (T - t > n) return; // overwritten by a later row of the window anyway+ const size_t rb = kv_row_bytes(kKvFp8, d);+ uint8_t* row = ring + (size_t) ((p0 + t) % n) * rb;+ if (backup != nullptr) {+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) backup[(size_t) t * rb + i] = row[i];+ __syncthreads();+ }+ const uint16_t* src = ext + (size_t) (n + t) * d;+ for (int e0 = 0; e0 < d; e0 += blockDim.x) { // (d a multiple of 32: a warp's 32 values one group)+ const int e = e0 + (int) threadIdx.x;+ const float v = e < d ? bf16f(src[e]) : 0.0f;+ uint32_t a = __float_as_uint(v) & 0x7fffffffu; // (the magnitudes' order as bits)+ for (int o = 16; o > 0; o >>= 1) a = max(a, (uint32_t) __shfl_xor_sync(0xffffffffu, a, o));+ const int k = a == 0u ? 0 : (int) (a >> 23) - 135 + ((a & 0x7fffffu) > 0x600000u ? 1 : 0);+ if (e >= d) continue;+ row[e] = (uint8_t) e4m3_code(v, k);+ if ((e & 31) == 0) row[d + e / 32] = (uint8_t) (k + 127);+ }+}++__global__ void ring_restore_kernel(uint8_t* __restrict__ ring, const uint8_t* __restrict__ backup, long long p0, int n,+ int keep, size_t rb) {+ const int t = keep + blockIdx.x;+ uint8_t* row = ring + (size_t) ((p0 + t) % n) * rb;+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) row[i] = backup[(size_t) t * rb + i];+}+__global__ void ring_restore_batch_kernel(const RingBatch b, long long p0, int n, int keep, size_t rb) {+ const int t = keep + blockIdx.x, li = blockIdx.y;+ uint8_t* row = b.ring[li] + (size_t) ((p0 + t) % n) * rb;+ const uint8_t* src = b.bak[li] + (size_t) t * rb;+ for (size_t i = threadIdx.x; i < rb; i += blockDim.x) row[i] = src[i];+}++__global__ void hc_mean_kernel(const float* __restrict__ h, int hc, int dim, float* __restrict__ out,+ long long out_stride) {+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (e >= dim) return;+ float a = 0.0f;+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * hc + s) * dim + e];+ out[(long long) t * out_stride + e] = a / (float) hc;+}++__global__ void iota_rows_kernel(int* out, int n, int stride) {+ const int t = blockIdx.x;+ for (int i = threadIdx.x; i < n; i += blockDim.x) out[(size_t) t * stride + i] = i;+}++// a warp a vocabulary row: E[*tok] (shared memory) . H[v]+__global__ void __launch_bounds__(kThreads) markov_kernel(float* __restrict__ logits, const uint16_t* __restrict__ E,+ const uint16_t* __restrict__ H, const int* __restrict__ tok,+ int V, int R) {+ extern __shared__ float er[];+ const uint16_t* erow = E + (size_t) (*tok) * R;+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);+ __syncthreads();+ const int lane = threadIdx.x & 31;+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ if (v >= V) return;+ const uint16_t* hr = H + (size_t) v * R;+ float a = 0.0f;+ for (int c = lane * 8; c < R; c += 32 * 8) {+ const uint4 q = *(const uint4*) (hr + c);+ const __half2* hp = (const __half2*) &q;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 f = __half22float2(hp[i]);+ a += f.x * er[c + 2 * i] + f.y * er[c + 2 * i + 1];+ }+ }+ a = warp_sum(a);+ if (lane == 0) logits[v] += a;+}++// ---------------------------------------------------------------- the markov bias over a head group (markov_share)+// a member's rows [v0, v0 + vn) of the bias (markov_kernel's sums, the prev token's E row already on the device, BF16)+__global__ void __launch_bounds__(kThreads) markov_share_kernel(float* __restrict__ logits,+ const uint16_t* __restrict__ H,+ const uint16_t* __restrict__ erow, int vn, int R) {+ extern __shared__ float er[];+ for (int i = threadIdx.x; i < R; i += blockDim.x) er[i] = bf16f(erow[i]);+ __syncthreads();+ const int lane = threadIdx.x & 31;+ const int v = blockIdx.x * (kThreads / 32) + (threadIdx.x >> 5);+ if (v >= vn) return;+ const uint16_t* hr = H + (size_t) v * R;+ float a = 0.0f;+ for (int c = lane * 8; c < R; c += 32 * 8) {+ const uint4 q = *(const uint4*) (hr + c);+ const __half2* hp = (const __half2*) &q;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 f = __half22float2(hp[i]);+ a += f.x * er[c + 2 * i] + f.y * er[c + 2 * i + 1];+ }+ }+ a = warp_sum(a);+ if (lane == 0) logits[v] += a;+}+// the share's argmax (the greedy sampler's rule: the larger value, on a tie the lower index) -> slot {value, v0 + i,+// seq}, the seq last+__global__ void __launch_bounds__(1024) argmax_share_kernel(const float* __restrict__ row, int vn, int v0,+ unsigned* slot, unsigned seq) {+ __shared__ float sv[32];+ __shared__ int si[32];+ float bv = __int_as_float(0xff800000);+ int bi = 0x7fffffff;+ for (int i = threadIdx.x; i < vn; i += blockDim.x) {+ const float x = row[i];+ if (x > bv) bv = x, bi = i; // (ascending: the lowest index of a tie stays)+ }+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, bv, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bi, o);+ if (ov > bv || (ov == bv && oi < bi)) bv = ov, bi = oi;+ }+ if ((threadIdx.x & 31) == 0) sv[threadIdx.x >> 5] = bv, si[threadIdx.x >> 5] = bi;+ __syncthreads();+ if (threadIdx.x == 0) {+ for (int w = 1; w < (int) (blockDim.x >> 5); ++w)+ if (sv[w] > bv || (sv[w] == bv && si[w] < bi)) bv = sv[w], bi = si[w];+ slot[0] = __float_as_uint(bv);+ slot[1] = (unsigned) (v0 + (bi == 0x7fffffff ? 0 : bi));+ __threadfence_system();+ *(volatile unsigned*) (slot + 2) = seq;+ }+}+// every member's share argmax of this step (slots nm apart) -> the global one into *tok, and its E row (host-mapped+// BF16 [V][R]) into erow for the next step's bias+__global__ void markov_pick_kernel(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E,+ uint16_t* erow, int R) {+ __shared__ int pick;+ if ((int) threadIdx.x < nm)+ while ((int) (*(volatile const unsigned*) (slots + (size_t) threadIdx.x * 4 + 2) - seq) < 0) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700+ __nanosleep(32);+#endif+ }+ __syncthreads();+ if (threadIdx.x == 0) {+ float bv = __int_as_float(0xff800000);+ int bi = 0x7fffffff;+ for (int m = 0; m < nm; ++m) { // (members in vocabulary order: a tie keeps the lower index)+ const float v = __uint_as_float(*(volatile const unsigned*) (slots + (size_t) m * 4));+ const int i = (int) *(volatile const unsigned*) (slots + (size_t) m * 4 + 1);+ if (v > bv || (v == bv && i < bi)) bv = v, bi = i;+ }+ pick = bi;+ *tok = bi;+ }+ __syncthreads();+#if defined(STRATA_USE_HIP) // (E is fixed once loaded: a plain load is as fresh)+ for (int i = threadIdx.x; i < R; i += blockDim.x) erow[i] = E[(size_t) pick * R + i];+#else+ for (int i = threadIdx.x; i < R; i += blockDim.x) erow[i] = __ldcv(E + (size_t) pick * R + i);+#endif+}++__global__ void __launch_bounds__(1024) confidence_kernel(const float* __restrict__ x, const uint16_t* __restrict__ E,+ const int* __restrict__ tok, const float* __restrict__ w,+ int dim, int R, float* __restrict__ out) {+ const int t = blockIdx.x;+ const float* xt = x + (size_t) t * dim;+ const uint16_t* er = E + (size_t) tok[t] * R;+ float a = 0.0f;+ for (int i = threadIdx.x; i < dim + R; i += blockDim.x) a += w[i] * (i < dim ? xt[i] : bf16f(er[i - dim]));+ a = block_sum(a);+ if (threadIdx.x == 0) out[t] = 1.0f / (1.0f + expf(-a));+}++// ---------------------------------------------------------------- engram+__global__ void engram_dequant_kernel(const uint8_t* __restrict__ rows, const uint8_t* __restrict__ scales, int hd,+ float* __restrict__ out, bool q6k) {+ const int r = blockIdx.x;+ if (q6k) { // (ggml Q6_K superblocks at the row's start)+ for (int e = threadIdx.x; e < hd; e += blockDim.x)+ out[(size_t) r * hd + e] = q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * kQ6kBytes, e % 256);+ return;+ }+ for (int e = threadIdx.x; e < hd; e += blockDim.x) {+ const float hv = e4m3_value(rows[(size_t) r * hd + e]);+ const uint8_t sb = scales[(size_t) r * (hd / 32) + e / 32];+ const float s = sb == 0xffu ? __int_as_float(0x7fc00000) : (sb == 0 ? 5.877471754111438e-39f+ : __uint_as_float((uint32_t) sb << 23));+ out[(size_t) r * hd + e] = hv * s;+ }+}++__global__ void __launch_bounds__(1024) engram_gate_kernel(float* __restrict__ h, const float* __restrict__ kv,+ const float* __restrict__ w, int hc, int dim, float eps) {+ const int k = blockIdx.x, t = blockIdx.y;+ float* hk = h + ((size_t) t * hc + k) * dim;+ const float* kvt = kv + (size_t) t * (hc + 1) * dim;+ const float* key = kvt + (size_t) k * dim;+ const float* val = kvt + (size_t) hc * dim;+ const float* wk = w + (size_t) k * dim;+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;+ for (int e = threadIdx.x; e < dim; e += blockDim.x) {+ const float hv = hk[e], kv_ = key[e];+ sh += hv * hv;+ sk += kv_ * kv_;+ sd += hv * wk[e] * kv_;+ }+ sh = block_sum(sh);+ sk = block_sum(sk);+ sd = block_sum(sd);+ const float rstd = rsqrtf(sh / (float) dim + eps) * rsqrtf(sk / (float) dim + eps);+ const float dot = sd * rstd * rsqrtf((float) dim);+ const float sq = copysignf(sqrtf(fmaxf(fabsf(dot), 1e-6f)), dot);+ const float gate = 1.0f / (1.0f + expf(-sq));+ for (int e = threadIdx.x; e < dim; e += blockDim.x) hk[e] = hk[e] + gate * val[e];+}++__global__ void gather_kernel(const float* __restrict__ x, Gather g, int cols, float* __restrict__ out) {+ const int r = blockIdx.y;+ const int c = blockIdx.x * blockDim.x + threadIdx.x;+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];+}++__global__ void combine_kernel(const float* __restrict__ ed, const float* __restrict__ cpu,+ const float* __restrict__ sh, Combine cm, int dim, float* __restrict__ out) {+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (e >= dim) return;+ float a = 0.0f;+ for (int j = 0; j < cm.n[t]; ++j) a += cm.w[t][j] * ed[(size_t) cm.row[t][j] * dim + e];+ if (cpu != nullptr && cm.cpu[t]) a += cpu[(size_t) t * dim + e];+ if (sh != nullptr) a += sh[(size_t) t * dim + e];+ out[(size_t) t * dim + e] = a;+}++__global__ void __launch_bounds__(512) route_publish_kernel(const float* __restrict__ logits,+ const float* __restrict__ bias, int n, int k,+ float route_scale, const unsigned long long* __restrict__ vram,+ const float* __restrict__ x, int dim, int* idx, float* w,+ unsigned long long* plan_ptr, Req* req,+ const float* __restrict__ bias_vl, unsigned vl) {+ __shared__ float sc[1024], bs[1024];+ __shared__ float bv[32];+ __shared__ int bi[32];+ __shared__ int pick[16];+ __shared__ int on_cpu;+ const int t = blockIdx.x;+ logits += (size_t) t * 1024;+ if (bias_vl != nullptr && ((vl >> t) & 1u)) bias = bias_vl;+ for (int i = threadIdx.x; i < n; i += blockDim.x) {+ const float l = logits[i];+ const float sp = l > 20.0f ? l : log1pf(expf(l));+ sc[i] = sqrtf(sp);+ bs[i] = sc[i] + bias[i];+ }+ __syncthreads();+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ for (int r = 0; r < k; ++r) {+ float best = -FLT_MAX;+ int bix = 0x7fffffff;+ for (int i = threadIdx.x; i < n; i += blockDim.x)+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;+ }+ if (lane == 0) bv[wp] = best, bi[wp] = bix;+ __syncthreads();+ if (threadIdx.x == 0) {+ float b = -FLT_MAX;+ int ix = 0x7fffffff;+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];+ pick[r] = ix;+ bs[ix] = -FLT_MAX;+ }+ __syncthreads();+ }+ if (threadIdx.x == 0) {+ float s = 0.0f;+ for (int r = 0; r < k; ++r) s += sc[pick[r]];+ int cpu = 0;+ for (int r = 0; r < k; ++r) {+ const float wv = sc[pick[r]] / (s + 1e-20f) * route_scale;+ idx[(size_t) t * kPicks + r] = pick[r];+ w[(size_t) t * kPicks + r] = wv;+ plan_ptr[(size_t) t * kPicks + r] = vram[pick[r]];+ cpu |= vram[pick[r]] == 0ull;+ if (req != nullptr) {+ req->ids[t][r] = pick[r];+ req->w[t][r] = wv;+ }+ }+ on_cpu = cpu;+ }+ // the token's FFN input to the host only where the CPU lane reads it (a pick of its on the CPU lane): each row+ // 20 KB over PCIe, where most windows have none+ __syncthreads();+ if (req != nullptr && on_cpu)+ for (int i = threadIdx.x; i < dim; i += blockDim.x) req->x[t][i] = x[(size_t) t * dim + i];+}++// a later layer's m likely picks (route_kernel's choice from its logits) into req->ahead[t][off ..)+__global__ void __launch_bounds__(512) route_ahead_kernel(const float* __restrict__ logits,+ const float* __restrict__ bias, int n, int m, Req* req,+ int off) {+ __shared__ float bs[1024];+ __shared__ float bv[32];+ __shared__ int bi[32];+ const int t = blockIdx.x;+ logits += (size_t) t * 1024;+ for (int i = threadIdx.x; i < n; i += blockDim.x) {+ const float l = logits[i];+ const float sp = l > 20.0f ? l : log1pf(expf(l));+ bs[i] = sqrtf(sp) + bias[i];+ }+ __syncthreads();+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ for (int r = 0; r < m; ++r) {+ float best = -FLT_MAX;+ int bix = 0x7fffffff;+ for (int i = threadIdx.x; i < n; i += blockDim.x)+ if (bs[i] > best || (bs[i] == best && i < bix)) best = bs[i], bix = i;+ for (int o = 16; o > 0; o >>= 1) {+ const float ov = __shfl_xor_sync(0xffffffffu, best, o);+ const int oi = __shfl_xor_sync(0xffffffffu, bix, o);+ if (ov > best || (ov == best && oi < bix)) best = ov, bix = oi;+ }+ if (lane == 0) bv[wp] = best, bi[wp] = bix;+ __syncthreads();+ if (threadIdx.x == 0) {+ float b = -FLT_MAX;+ int ix = 0x7fffffff;+ for (int q = 0; q < (int) (blockDim.x >> 5); ++q)+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) b = bv[q], ix = bi[q];+ req->ahead[t][off + r] = ix < n ? ix : -1;+ if (ix < n) bs[ix] = -FLT_MAX;+ }+ __syncthreads();+ }+}++__global__ void publish_kernel(Req* req, int T, unsigned seq, int na0, int na1) {+ __threadfence_system();+ req->na[0] = na0;+ req->na[1] = na1;+ req->T = T;+ __threadfence_system();+ *(volatile unsigned*) &req->seq = seq;+}++__global__ void wait_seq_kernel(const unsigned* seq_ptr, unsigned seq) {+ if (threadIdx.x != 0) return;+ while (*(volatile const unsigned*) seq_ptr != seq) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700+ __nanosleep(200);+#endif+ }+ __threadfence_system();+}++__global__ void init_pre_kernel(float* pre, int T) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;+}+++// the hand-offs through host-mapped memory: a grid-stride copy of the segments (16 bytes a thread), then (put, flag+// given) the last block to finish sets the flag after a system fence; a get first waits for the flag+__device__ __forceinline__ void xfer_copy(const Xfer& x, bool from_host) {+ size_t total = 0;+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;+ for (size_t q = (size_t) blockIdx.x * blockDim.x + threadIdx.x; q < total; q += (size_t) gridDim.x * blockDim.x) {+ size_t r = q;+ int i = 0;+ while (r >= x.bytes[i] / 16) r -= x.bytes[i++] / 16;+ const uint4* src = (const uint4*) x.src[i] + r;+ ((uint4*) x.dst[i])[r] = from_host ? ld_fresh(src) : *src;+ }+}++__global__ void __launch_bounds__(kThreads) xfer_put_kernel(const Xfer x, unsigned* flag, unsigned seq, unsigned* cnt) {+ xfer_copy(x, false);+ if (flag == nullptr) return;+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0 && atomicAdd(cnt, 1u) == gridDim.x - 1) {+ *cnt = 0u;+ __threadfence_system();+ *(volatile unsigned*) flag = seq;+ }+}++__device__ __forceinline__ void wait_at_least(const unsigned* flag, unsigned seq) {+ while ((int) (*(volatile const unsigned*) flag - seq) < 0) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700+ __nanosleep(100);+#endif+ }+}++__global__ void __launch_bounds__(kThreads) xfer_get_kernel(const Xfer x, const unsigned* flag, unsigned seq) {+ if (threadIdx.x == 0) wait_at_least(flag, seq);+ __syncthreads();+ xfer_copy(x, true);+}++__global__ void wait_flags_kernel(const unsigned* flags, int n, int stride, unsigned seq) {+ if (threadIdx.x != 0) return;+ for (int i = 0; i < n; ++i) wait_at_least(flags + (size_t) i * stride, seq);+ __threadfence_system();+}+__global__ void set_flag_kernel(unsigned* flag, unsigned v) {+ __threadfence_system();+ *(volatile unsigned*) flag = v;+}++// three members or more: a reduce-scatter and an all-gather in one launch, a block a contiguous range of the row cut+// in nm chunks. The block puts its range of this member's partial out and flags it; once every member's flag of the+// range is up it sums chunk mi of the range in member order (ar_sum_kernel's sums, bit for bit), puts that out and+// flags it; once every member's sum flag is up it reads the other chunks' sums. Each member reads about two rows over+// PCIe whatever nm is, where every member reading every other row grows with nm+__global__ void __launch_bounds__(kThreads) ar_rs_kernel(float* x, int n4, float* rows, float* sums, size_t stride,+ unsigned* bflags, int nm, int mi, int slot, int slots,+ unsigned seq) {+ float4* x4 = (float4*) x;+ const int per = (n4 + (int) gridDim.x - 1) / (int) gridDim.x;+ const int lo = blockIdx.x * per, hi = min(n4, lo + per), cs = (hi - lo + nm - 1) / nm;+ auto rflag = [&](int m) { return bflags + (((size_t) m * slots + slot) * kArBlocks + blockIdx.x) * 8; };+ auto sflag = [&](int m) {+ return bflags + ((((size_t) nm + m) * slots + slot) * kArBlocks + blockIdx.x) * 8;+ };+ auto row = [&](int m) { return (float4*) (rows + ((size_t) m * slots + slot) * stride); };+ float4* sum = (float4*) (sums + (size_t) slot * stride);+ if (lo < hi) {+ float4* mine = row(mi);+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x) mine[q] = x4[q];+ }+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0) *(volatile unsigned*) rflag(mi) = seq;+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(rflag(threadIdx.x), seq);+ __syncthreads();+ const int c0 = min(hi, lo + mi * cs), c1 = min(hi, c0 + cs);+ for (int q = c0 + threadIdx.x; q < c1; q += blockDim.x) {+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ for (int m = 0; m < nm; ++m) {+ const float4 p = m == mi ? x4[q] : ld_fresh(row(m) + q);+ if (m == 0) v = p;+ else {+ v.x = __fadd_rn(v.x, p.x);+ v.y = __fadd_rn(v.y, p.y);+ v.z = __fadd_rn(v.z, p.z);+ v.w = __fadd_rn(v.w, p.w);+ }+ }+ sum[q] = v;+ x4[q] = v;+ }+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0) *(volatile unsigned*) sflag(mi) = seq;+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(sflag(threadIdx.x), seq);+ __syncthreads();+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x)+ if (q < c0 || q >= c1) x4[q] = ld_fresh(sum + q);+}++// ar_rs_kernel with the rows and the sums in FP16 (DS4_AR_F16): half the bytes over PCIe. Every member's partial is+// rounded to FP16 (the owner's too), summed in F32 in member order, the sum rounded to FP16 - and every member, the+// owner too, takes that rounded sum, so the members' copies of the stream stay the same+__device__ __forceinline__ uint2 to_h4(float4 v) {+ const __half2 a = __floats2half2_rn(v.x, v.y), b = __floats2half2_rn(v.z, v.w);+ return make_uint2(*(const unsigned*) &a, *(const unsigned*) &b);+}+__device__ __forceinline__ float4 from_h4(uint2 u) {+ const float2 a = __half22float2(*(const __half2*) &u.x), b = __half22float2(*(const __half2*) &u.y);+ return make_float4(a.x, a.y, b.x, b.y);+}+__device__ __forceinline__ float4 sat4(float4 v) { // (FP16's range: a partial past it saturates instead of inf)+ const float m = 65504.0f;+ return make_float4(fminf(fmaxf(v.x, -m), m), fminf(fmaxf(v.y, -m), m), fminf(fmaxf(v.z, -m), m), fminf(fmaxf(v.w, -m), m));+}+__global__ void __launch_bounds__(kThreads) ar_rs16_kernel(float* x, int n4, float* rows, float* sums, size_t stride,+ unsigned* bflags, int nm, int mi, int slot, int slots,+ unsigned seq) {+ float4* x4 = (float4*) x;+ const int per = (n4 + (int) gridDim.x - 1) / (int) gridDim.x;+ const int lo = blockIdx.x * per, hi = min(n4, lo + per), cs = (hi - lo + nm - 1) / nm;+ auto rflag = [&](int m) { return bflags + (((size_t) m * slots + slot) * kArBlocks + blockIdx.x) * 8; };+ auto sflag = [&](int m) {+ return bflags + ((((size_t) nm + m) * slots + slot) * kArBlocks + blockIdx.x) * 8;+ };+ auto row = [&](int m) { return (uint2*) (rows + ((size_t) m * slots + slot) * stride); }; // (FP16: half a row)+ uint2* sum = (uint2*) (sums + (size_t) slot * stride);+ if (lo < hi) {+ uint2* mine = row(mi);+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x) mine[q] = to_h4(sat4(x4[q]));+ }+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0) *(volatile unsigned*) rflag(mi) = seq;+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(rflag(threadIdx.x), seq);+ __syncthreads();+ const int c0 = min(hi, lo + mi * cs), c1 = min(hi, c0 + cs);+ for (int q = c0 + threadIdx.x; q < c1; q += blockDim.x) {+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ for (int m = 0; m < nm; ++m) {+ const uint2 u = m == mi ? to_h4(sat4(x4[q])) : ld_fresh(row(m) + q);+ const float4 p = from_h4(u);+ if (m == 0) v = p;+ else {+ v.x = __fadd_rn(v.x, p.x);+ v.y = __fadd_rn(v.y, p.y);+ v.z = __fadd_rn(v.z, p.z);+ v.w = __fadd_rn(v.w, p.w);+ }+ }+ const uint2 s16 = to_h4(sat4(v));+ sum[q] = s16;+ x4[q] = from_h4(s16);+ }+ __threadfence_system();+ __syncthreads();+ if (threadIdx.x == 0) *(volatile unsigned*) sflag(mi) = seq;+ if ((int) threadIdx.x < nm && (int) threadIdx.x != mi) wait_at_least(sflag(threadIdx.x), seq);+ __syncthreads();+ for (int q = lo + threadIdx.x; q < hi; q += blockDim.x)+ if (q < c0 || q >= c1) x4[q] = from_h4(ld_fresh(sum + q));+}++// x[i] = the members' partials summed in member order (this member's from x, the others' from their host rows)+__global__ void __launch_bounds__(kThreads) ar_sum_kernel(float* x, int n4, const float* peers, size_t stride,+ const unsigned* flags, int nm, int mi, int slot, int slots,+ unsigned seq) {+ if (threadIdx.x == 0)+ for (int m = 0; m < nm; ++m)+ if (m != mi) wait_at_least(flags + ((size_t) m * slots + slot) * 32, seq);+ __syncthreads();+ for (int q = blockIdx.x * blockDim.x + threadIdx.x; q < n4; q += gridDim.x * blockDim.x) {+ float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ for (int m = 0; m < nm; ++m) {+ const float4 p = m == mi ? ((const float4*) x)[q]+ : ld_fresh((const float4*) (peers + ((size_t) m * slots + slot) * stride) + q);+ if (m == 0) v = p;+ else {+ v.x = __fadd_rn(v.x, p.x);+ v.y = __fadd_rn(v.y, p.y);+ v.z = __fadd_rn(v.z, p.z);+ v.w = __fadd_rn(v.w, p.w);+ }+ }+ ((float4*) x)[q] = v;+ }+}+++// a thread a (token, value): every stream's value read before any is written+__global__ void hc_post_inplace_kernel(const float* __restrict__ y, float* h, const float* __restrict__ post,+ const float* __restrict__ comb, int dim, int hc) {+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (e >= dim) return;+ float* r = h + (size_t) t * hc * dim;+ float rv[8];+ for (int j = 0; j < hc; ++j) rv[j] = r[(size_t) j * dim + e];+ const float yv = y[(size_t) t * dim + e];+ for (int k = 0; k < hc; ++k) {+ float a = post[(size_t) t * 8 + k] * yv;+ for (int j = 0; j < hc; ++j) a += comb[(size_t) t * 64 + j * hc + k] * rv[j];+ r[(size_t) k * dim + e] = a;+ }+}++__global__ void __launch_bounds__(kThreads) hc_mix_scale_kernel(const float* __restrict__ h, int hcdim, int n_mix,+ float eps, float* mixes) {+ const int t = blockIdx.x;+ const float* x = h + (size_t) t * hcdim;+ float ss = 0.0f;+ for (int k = threadIdx.x * 4; k < hcdim; k += kThreads * 4) {+ const float4 hv = *(const float4*) (x + k);+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ ss = block_sum(ss);+ const float inv = rsqrtf(ss / (float) hcdim + eps);+ if (threadIdx.x < n_mix) mixes[(size_t) t * 32 + threadIdx.x] *= inv;+}++__global__ void engram_dequant_tok_kernel(const uint8_t* __restrict__ src, size_t per, int cols, int hd,+ float* __restrict__ out, bool q6k) {+ const int r = blockIdx.x, t = r / cols, c = r - t * cols;+ const uint8_t* rows = src + (size_t) t * per + (size_t) c * hd;+ const uint8_t* scales = src + (size_t) t * per + (size_t) cols * hd + (size_t) c * (hd / 32);+ if (q6k) { // (ggml Q6_K superblocks at the row's start)+ for (int e = threadIdx.x; e < hd; e += blockDim.x)+ out[(size_t) r * hd + e] = q6k_value(rows + (size_t) (e / 256) * kQ6kBytes, e % 256);+ return;+ }+ for (int e = threadIdx.x; e < hd; e += blockDim.x) {+ const float hv = e4m3_value(rows[e]);+ const uint8_t sb = scales[e / 32];+ const float s = sb == 0xffu ? __int_as_float(0x7fc00000) : (sb == 0 ? 5.877471754111438e-39f+ : __uint_as_float((uint32_t) sb << 23));+ out[(size_t) r * hd + e] = hv * s;+ }+}++// a block a virtual slot: its picks counted (sort pass 1), or (fill) placed in token order by a block-wide scan+template <bool FILL>+__global__ void __launch_bounds__(kThreads) moe_sort_kernel(const int* __restrict__ idx, int T, int k,+ const int* __restrict__ vs, int* bounds, int* row_tok,+ int* pos) {+ const int slot = blockIdx.x;+ __shared__ int warp_n[kThreads / 32];+ __shared__ int base_s;+ if (threadIdx.x == 0) base_s = FILL ? bounds[slot] : 0;+ __syncthreads();+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;+ const int n = T * k;+ for (int q0 = 0; q0 < n; q0 += kThreads) {+ const int q = q0 + threadIdx.x;+ const int t = q / k, i = q - t * k;+ const bool hit = q < n && vs[idx[(size_t) t * kPicks + i]] == slot;+ const unsigned m = __ballot_sync(0xffffffffu, hit);+ if (lane == 0) warp_n[wp] = __popc(m);+ __syncthreads();+ int before = 0, total = 0;+ for (int w = 0; w < kThreads / 32; ++w) {+ if (w < wp) before += warp_n[w];+ total += warp_n[w];+ }+ if (FILL && hit) {+ const int r = base_s + before + __popc(m & ((1u << lane) - 1u));+ row_tok[r] = t;+ pos[(size_t) t * kPicks + i] = r;+ }+ __syncthreads();+ if (threadIdx.x == 0) base_s += total;+ __syncthreads();+ }+ if (!FILL && threadIdx.x == 0) bounds[slot + 1] = base_s; // (the counts; the scan makes them bounds)+}++__global__ void bounds_scan_kernel(int* bounds, int n_vs) { // one thread: bounds[0] = 0, bounds[s + 1] += bounds[s]+ if (threadIdx.x != 0) return;+ bounds[0] = 0;+ for (int s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];+}++__global__ void moe_combine_kernel(float* acc, const float* __restrict__ y, const int* __restrict__ pos,+ const float* __restrict__ w, int k, int r0, int r1, int dim) {+ const int e = blockIdx.x * blockDim.x + threadIdx.x, t = blockIdx.y;+ if (e >= dim) return;+ float a = acc[(size_t) t * dim + e];+ bool any = false;+ for (int i = 0; i < k; ++i) {+ const int r = pos[(size_t) t * kPicks + i];+ if (r < r0 || r >= r1) continue;+ a = __fmaf_rn(w[(size_t) t * kPicks + i], y[(size_t) (r - r0) * dim + e], a);+ any = true;+ }+ if (any) acc[(size_t) t * dim + e] = a;+}++int cdiv(int a, int b) { return (a + b - 1) / b; }++} // namespace++void route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale,+ const unsigned long long* vram, const float* x, int dim, int* idx, float* w,+ unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl,+ unsigned vl, int na0, int na1) {+ route_publish_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,+ vl != 0 ? bias_vl : nullptr, vl);+ if (req != nullptr) publish_kernel<<<1, 1, 0, s>>>(req, T, seq, na0, na1);+}+void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s) {+ if (req != nullptr && m > 0 && T > 0 && n <= 1024 && off + m <= kAhead)+ route_ahead_kernel<<<T, 512, 0, s>>>(logits, bias, n, m, req, off);+}+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { wait_seq_kernel<<<1, 32, 0, s>>>(seq_ptr, seq); }+void init_pre(float* pre, int T, cudaStream_t s) { init_pre_kernel<<<cdiv(T * 8, 128), 128, 0, s>>>(pre, T); }++void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {+ embed_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(rows, h, dim, hc);+}++void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s) {+ if (len > kFinMax * kThreads) {+ std::fprintf(stderr, "ds4 finish: a vector of %d values is more than %d\n", len, kFinMax * kThreads);+ return;+ }+ if (n_vec <= 0) return;+ if (f.pdst != nullptr) {+ if (f.quant < kKvFp8 || f.quant > kKvFp4p || len % 32 != 0) {+ std::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);+ return;+ }+ finish_kernel<true><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);+ } else+ finish_kernel<false><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);+}++void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {+ hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);+}+void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,+ float* pre, float* post, float* comb, cudaStream_t s) {+ hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);+}+void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,+ cudaStream_t s) {+ hc_pre_norm_kernel<<<T, 1024, 0, s>>>(h, mix, w, dim, hc, eps, x);+}+void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,+ float* out, cudaStream_t s) {+ hc_post_kernel<<<dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s>>>(y, res, post, comb, dim, hc, out);+}++void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,+ cudaStream_t s) {+ hc_post_inplace_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(y, h, post, comb, dim, hc);+}+void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {+ hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);+}+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, bool q6k) {+ if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, q6k);+}+void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,+ cudaStream_t s) {+ moe_sort_kernel<false><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);+ bounds_scan_kernel<<<1, 32, 0, s>>>(bounds, n_vs);+ moe_sort_kernel<true><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);+}+void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,+ cudaStream_t s) {+ if (r1 > r0 && T > 0) moe_combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(acc, y, pos, w, k, r0, r1, dim);+}++void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s) {+ gemv_f16_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, alpha, y);+}+void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,+ cudaStream_t s) {+ if (T > 0)+ gemv_f16_rows_kernel<<<dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s>>>(W, x, ldx, n, k, T, alpha,+ y, ldy);+}++void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s) {+ if (T > 0) gemv_f32_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, y, ldy);+}++void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,+ cudaStream_t s, const float* bias_vl, const VlBits* vl) {+ static const VlBits none = {};+ route_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,+ vl != nullptr ? *vl : none);+}++void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s) {+ if (n > 0) swiglu_kernel<<<cdiv(n, kThreads), kThreads, 0, s>>>(g, u, n, lim, h);+}++void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,+ int d, const float* w, float eps, float* latent, cudaStream_t s) {+ compress_kernel<<<1, 512, 0, s>>>(kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);+}++void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,+ int cand_stride, int blk, const Rows& lim, float* out, long long out_stride, cudaStream_t s,+ int kfmt) {+ if (hd != 128 || heads > 32 || (kfmt != kKvFp4p && kfmt != kKvBf16)) {+ std::fprintf(stderr, "ds4 index_score: %d heads of %d (keys %d) not covered\n", heads, hd, kfmt);+ return;+ }+ if (n <= 0) return;+ const size_t smem = (size_t) (heads * hd + heads) * sizeof(float);+ const dim3 grid(cdiv(n, kThreads), T);+ if (kfmt == kKvFp4p)+ index_score_kernel<128, kKvFp4p><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,+ cand_stride, blk, lim, out, out_stride);+ else+ index_score_kernel<128, kKvBf16><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,+ cand_stride, blk, lim, out, out_stride);+}++void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,+ int* out, int out_stride, cudaStream_t s) {+ topk_kernel<<<T, 1024, 0, s>>>(vals, stride, r, cand, cand_stride, blk, out, out_stride);+}++void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,+ int* out, int out_stride, cudaStream_t s) {+ int nb_max = 1;+ Rows rb = r;+ if (r.gen != 0) { // (a chunk: the widest query's blocks, made on the device per query)+ rb.gen = 4;+ rb.blk = blk;+ nb_max = cdiv((int) ((r.p0 + T) / r.R), blk);+ } else+ for (int t = 0; t < T; ++t) {+ rb.n[t] = cdiv(r.n[t], blk);+ rb.k[t] = r.k[t];+ nb_max = rb.n[t] > nb_max ? rb.n[t] : nb_max;+ }+ block_max_kernel<<<dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s>>>(score, stride, r, blk, bm, bm_stride);+ topk(bm, bm_stride, T, rb, nullptr, 0, 1, out, out_stride, s);+}++// the multiprocessors of the current device (cached a device)+static int sm_count() {+ static std::atomic<int> cached[64];+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) dev = 0;+ int v = cached[dev].load(std::memory_order_relaxed);+ if (v <= 0) {+ if (cudaDeviceGetAttribute(&v, cudaDevAttrMultiProcessorCount, dev) != cudaSuccess || v <= 0) v = 1;+ cached[dev].store(v, std::memory_order_relaxed);+ }+ return v;+}++void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride,+ const AttnRows& a, const float* sink, int heads, int d, int T, float scale, float* out, cudaStream_t s,+ int ckv_fmt, int heads_per_block, const AttnScratch* scr) {+ if (d != 512 || (ckv_fmt != kKvFp4 && ckv_fmt != kKvBf16)) {+ std::fprintf(stderr, "ds4 sparse_attn: head dim %d (rows %d) not covered\n", d, ckv_fmt);+ return;+ }+ if (ckv_fmt == kKvBf16) {+ sparse_attn_kernel<false><<<dim3(heads, T), kThreads, 0, s>>>(q, win, (const uint16_t*) ckv, sel, sel_stride, a,+ sink, heads, d, scale, out);+ return;+ }+ // a few queries (a decode step, a short window): their selected rows decoded once a query into the scratch, then+ // the BF16 rows' kernel itself (heads_per_block -1: this way at any T)+ const bool few = heads_per_block == -1 || (heads_per_block == 0 && T <= kAttnScratchT);+ if (few && scr != nullptr && scr->rows != nullptr && T <= scr->max_t && sel_stride <= scr->cap) {+ unpack_sel_kernel<<<dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s>>>((const uint8_t*) ckv, sel, sel_stride, a,+ scr->cap, scr->rows);+#if !defined(STRATA_USE_HIP)+ if (T == 1) // (V100: the loads ahead pay for the decode's kernel at one query, not at more)+ sparse_attn_kernel<true, 8><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,+ heads, d, scale, out);+ else+#endif+ sparse_attn_kernel<true, 0><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,+ heads, d, scale, out);+ return;+ }+ // packed rows: as many heads a block as still leaves two blocks a multiprocessor (a decoded row serves them all)+ int G = heads_per_block;+ if (G != 1 && G != 2 && G != 4 && G != 8) {+ G = 8;+ while (G > 1 && (heads % G != 0 || (heads / G) * T < 2 * sm_count())) G /= 2;+ }+ if (heads % G != 0) G = 1;+ const dim3 grid(heads / G, T);+ const uint8_t* c = (const uint8_t*) ckv;+ if (G == 8) sparse_attn_packed_kernel<8><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else if (G == 4) sparse_attn_packed_kernel<4><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else if (G == 2) sparse_attn_packed_kernel<2><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else sparse_attn_packed_kernel<1><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);+}++void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s) {+ ring_to_ext_kernel<<<n, kThreads, 0, s>>>(ring, ext, (long long) p0, n, d);+}+void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s, uint8_t* backup) {+ if (d % 32 != 0) {+ std::fprintf(stderr, "ds4 ext_to_ring: row of %d not covered\n", d);+ return;+ }+ ext_to_ring_kernel<<<T, kThreads, 0, s>>>(ext, ring, (long long) p0, n, T, d, backup);+}+void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {+ if (T > keep)+ ring_restore_kernel<<<T - keep, kThreads, 0, s>>>(ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));+}+void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {+ if (T > keep && b.n > 0)+ ring_restore_batch_kernel<<<dim3(T - keep, b.n), kThreads, 0, s>>>(b, (long long) p0, n, keep,+ kv_row_bytes(kKvFp8, d));+}+void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {+ hc_mean_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(h, hc, dim, out, out_stride);+}+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kernel<<<T, 64, 0, s>>>(out, n, stride); }+void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {+ markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);+}+void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {+ markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);+}+void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s) {+ argmax_share_kernel<<<1, 1024, 0, s>>>(row, vn, v0, slot, seq);+}+void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,+ cudaStream_t s) {+ markov_pick_kernel<<<1, 256, 0, s>>>(slots, nm, seq, tok, E, erow, R);+}+void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,+ cudaStream_t s) {+ confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);+}++void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k) {+ engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, q6k);+}++void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {+ engram_gate_kernel<<<dim3(hc, T), 1024, 0, s>>>(h, kv, w, hc, dim, eps);+}++void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s) {+ if (n > 0) gather_kernel<<<dim3(cdiv(cols, kThreads), n), kThreads, 0, s>>>(x, g, cols, out);+}++void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,+ cudaStream_t s) {+ combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(ed, cpu, sh, c, dim, out);+}++void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s) {+ size_t total = 0;+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;+ const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));+ xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);+}+void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {+ size_t total = 0;+ for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;+ const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));+ xfer_get_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq);+}+void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {+ if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);+}+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { set_flag_kernel<<<1, 1, 0, s>>>(flag, v); }+void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {+ const int slot = (int) (seq % (unsigned) a.slots);+ if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)+ static const bool f16 = [] {+ const char* e = std::getenv("DS4_AR_F16");+ return e == nullptr || std::atoi(e) != 0; // (on unless DS4_AR_F16=0)+ }();+ const int n4 = n / 4;+ const int blocks = std::max(1, std::min(kArBlocks, cdiv(n4, kThreads)));+ if (f16)+ ar_rs16_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);+ else+ ar_rs_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);+ return;+ }+ Xfer xp;+ xp.n = 1;+ xp.src[0] = x;+ xp.dst[0] = a.data + ((size_t) mi * a.slots + slot) * a.stride;+ xp.bytes[0] = (size_t) n * 4;+ xfer_put(xp, a.flags + ((size_t) mi * a.slots + slot) * 32, seq, cnt, s);+ const int n4 = n / 4;+ const int blocks = std::max(1, std::min(64, cdiv(n4, kThreads)));+ ar_sum_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.stride, a.flags, a.nm, mi, slot, a.slots, seq);+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_kernels.cuh b/src/dsv41/ds4_kernels.cuhnew file mode 100644index 000000000..f1903c679--- /dev/null+++ b/src/dsv41/ds4_kernels.cuh@@ -0,0 +1,296 @@+// src/ds4/ds4_kernels.cuh - DeepSeek-V4.1's own device kernels for a window of T tokens (1 <= T <= kWin) at positions+// p0 .. p0 + T - 1 (the EXL3 matrices go through strata::kernels::exl3). The math is DeepSeek's inference/model.py+// (MIT); tools/ds4/ref.py is the oracle the engine is checked against. Every pointer is device memory, every activation+// F32, rows of a window back to back; the KV caches keep the reference's FP8 / FP4 roundings packed as DeepSeek stores+// them (codes and scales, kv_row_bytes), the window attention's working rows BF16 (those values are exact in BF16).+// What differs from query to query (its window, how much of the compressed KV it sees, how many rows it selects) is+// known on the host and passed by value.+#pragma once++#include <cstddef>+#include <cstdint>+#include <cuda_runtime.h>++namespace strata::ds4 {++constexpr int kWin = 16; // tokens a window takes at most (the EXL3 GEMV's activations per job)+constexpr int kPicks = 8; // routed experts a token can take at most++/// The caches' rows: DeepSeek's own roundings kept as codes and scales - lossless, the values are on those grids already+/// (the numbers are Finish::quant's; a finish with that quant and a pdst writes the format), a row of d values in+/// kv_row_bytes(fmt, d) bytes, padded to 16 (the hand-offs copy 16-byte units):+/// kKvFp8 the window ring: d E4M3 codes, then d / 32 power-of-two scales (UE8M0: the F32 exponent byte)+/// kKvFp4 the compressed KV: d / 2 bytes of E2M1 codes (value 2i in the low nibble), then d / 16 E4M3 scales+/// kKvFp4p the index keys: d / 2 bytes of E2M1 codes, then d / 32 UE8M0 scales+/// kKvBf16: plain BF16 rows (a DSpark block's own rows, and the reference the tests check the packed rows against).+constexpr int kKvBf16 = 0, kKvFp8 = 1, kKvFp4 = 2, kKvFp4p = 3;+__host__ __device__ constexpr size_t kv_row_bytes(int fmt, int d) {+ return fmt == kKvBf16 ? (size_t) d * 2+ : fmt == kKvFp8 ? ((size_t) d + (size_t) d / 32 + 15) / 16 * 16+ : fmt == kKvFp4 ? ((size_t) d / 2 + (size_t) d / 16 + 15) / 16 * 16+ : ((size_t) d / 2 + (size_t) d / 32 + 15) / 16 * 16;+}++/// h[t][s][e] = BF16 rows[t][e] for each of the hc streams.+void embed(const uint16_t* rows_bf16, int T, float* h, int dim, int hc, cudaStream_t s);++/// The per-vector finish: n_vec vectors of len values at x + v * stride, each in order+/// norm (w != null): x = x * rsqrt(mean(x^2) + eps) * w+/// rope (rd > 0): the last rd values as adjacent complex pairs rotated by pos_v * freqs[i] (inverse: the conjugate),+/// pos_v = pos + (v / vpp) * pos_step+/// quant 0 none, 1 FP8 E4M3 with a power-of-two scale per 32, 2 FP4 E2M1 with an E4M3 scale per 16, 3 FP4 with a+/// power-of-two scale per 32 (the reference kernels' act_quant / fp4_act_quant, inplace)+/// and written back to x, or as BF16 to dst + v * dst_stride when dst is given, or packed (quant > 0: the cache format of+/// that number, kv_row_bytes) to pdst + v * pdst_stride bytes when pdst is given (x then left as it was).+struct Finish {+ const float* w = nullptr;+ float eps = 1e-20f;+ int rd = 0, pos = 0, inverse = 0, vpp = 1, pos_step = 1;+ const float* freqs = nullptr; // rd / 2 values+ int quant = 0;+ uint16_t* dst = nullptr;+ long long dst_stride = 0;+ uint8_t* pdst = nullptr;+ long long pdst_stride = 0;+};+void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s);++/// Hyper-Connections, row t of each [T][..] array: mixes[t][m] = (h_flat . fn[m]) * rsqrt(mean(h_flat^2) + eps), then+/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes+/// 32, pre / post 8, comb 64 floats a row.+void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);+void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,+ float* pre, float* post, float* comb, cudaStream_t s);+/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).+void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,+ cudaStream_t s);+/// out[t][k][e] = post[t][k] y[t][e] + sum_j comb[t][j][k] res[t][j][e] (out must not alias res).+void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,+ float* out, cudaStream_t s);+/// hc_post with out = res (h), the same arithmetic: a chunk keeps one copy of its streams.+void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,+ cudaStream_t s);+/// A chunk's hc_mix: the dots of each row with the n_mix weights already in mixes (a GEMM's, rows 32 apart), times+/// rsqrt(mean(h_row^2) + eps) - hc_mix's value up to the dots' summation order.+void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);++/// y[t][r] = alpha * sum_k W[r][k] x[t][k], W F16 bits [n][k] (the indexer's weights_proj).+void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s);+/// The same for T activations x[t] = x + t ldx into y + t ldy, each weight read once for all of them; each output is+/// gemv_f16's bit for bit.+void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,+ cudaStream_t s);+/// y[t ldy + r] = sum_k W[r][k] x[t][k] in F32 (W [n][k], k % 128 == 0): the router's logits, as DeepSeek's+/// inference computes them (linear(x.float(), weight.float())) - a warp a row, the same summing order for any T, so+/// the token, window and chunk paths get the same bits.+void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s);++/// The router, row t: scores = sqrt(softplus(logits[t])) (n experts, rows 1024 apart), the top `k` of scores + bias;+/// weights from the unbiased scores, normalized to sum 1 (+1e-20) and times route_scale -> idx[t][i], w[t][i] (rows+/// kPicks apart). Rows of image tokens (bit t of vl) take bias_vl instead (training's noaux_tc_for_vl).+constexpr int kVlRows = 8192;+struct VlBits {+ uint32_t w[kVlRows / 32];+};+void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,+ cudaStream_t s, const float* bias_vl = nullptr, const VlBits* vl = nullptr);++/// h = silu(min(g, lim)) * clamp(u, -lim, lim) (lim <= 0: no clamp); n values.+void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s);++/// The ratio-r compressor over a window: token t (position p0 + t) takes slot (p0 + t) % r of the state; each token that+/// completes a group writes the group's pooled latent (softmax over the slots, per value, then rmsnorm(w)) to the next+/// row of latent.+void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,+ int d, const float* w, float eps, float* latent, cudaStream_t s);++/// Per-query sizes, by value - or (gen > 0, a chunk of any length) made on the device from the query's position: query+/// t at position p0 + t has nn = (p0 + t + 1) / R compressed positions and cb = min(ccap, ceil(nn / blk)) candidate+/// blocks; gen 1: n = nn, k = min(kcap, nn) (the indexer's scores and their top-k)+/// gen 2: n = nn, k = cb (index_score over the candidates)+/// gen 3: n = cb * blk, k = min(kcap, nn) (the top-k of the candidates' scores)+/// gen 4: n = ceil(nn / blk), k = kcap (the candidate blocks' top-k)+struct Rows {+ int n[kWin];+ int k[kWin];+ int gen = 0, R = 1, kcap = 0, ccap = 0, blk = 1;+ long long p0 = 0;+};++/// The indexer's scores, row t: out[t][i] = sum_h relu(q[t][h] . k[p]) w[t][h] for i < n (the widest query's), p = i, or+/// with candidates p = cand[t][i / blk] * blk + i % blk for i < lim.k[t] * blk; positions >= lim.n[t] (or past the+/// query's candidates) -> -inf. q rows heads * hd apart, w rows 64, cand rows cand_stride, out rows out_stride; the keys+/// k rows of format kfmt (the cache's kKvFp4p, or kKvBf16).+void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,+ int cand_stride, int blk, const Rows& lim, float* out, long long out_stride, cudaStream_t s,+ int kfmt = kKvFp4p);+/// The top r.k[t] of vals[t][0, r.n[t]) (row t at vals + t * stride): their positions (unordered; with candidates+/// cand[t][i / blk] * blk + i % blk) into out[t][0, k). r.k[t] >= r.n[t] takes them all.+void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,+ int* out, int out_stride, cudaStream_t s);+/// Level one of the candidate pre-filter, row t: block b's score = the max over its blk positions of the r.n[t] the+/// query sees; the block of its newest position is pinned; the top r.k[t] blocks -> out[t] (bm: scratch rows bm_stride).+void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,+ int* out, int out_stride, cudaStream_t s);++/// Sparse attention, query t: heads x d values of q[t] against win rows win0[t] .. win0[t] + nwin[t] - 1 of the window+/// buffer (d BF16 each) and the nsel[t] compressed rows ckv[sel[t][i]] (format ckv_fmt: the cache's kKvFp4, or kKvBf16);+/// K = V, a learned sink logit per head. The packed rows, the same bits either way: up to kAttnScratchT queries (with+/// a scratch, heads_per_block 0; or -1, any T) decoded once a query into scr's BF16 rows for the BF16 rows' kernel;+/// else a kernel that decodes each staged row once for heads_per_block heads of a query (1, 2, 4, 8; 0: as many as+/// keep two blocks a multiprocessor).+struct AttnScratch {+ uint16_t* rows = nullptr; // [max_t * cap][512] BF16+ int max_t = 0, cap = 0; // queries, rows a query (at least the sel_stride)+};+constexpr int kAttnScratchT = 7; // (the staged kernel wins from 8 queries: V100, Sun)+struct AttnRows {+ int win0[kWin], nwin[kWin], nsel[kWin];+ // gen: query t is a chunk's row t0 + t at position p0 + t0 + t (the window buffer: the ring's win rows before p0,+ // then the chunk's), selecting min(kcap, (its position + 1) / R) compressed rows when R > 0+ int gen = 0, t0 = 0, win = 0, R = 0, kcap = 0;+ long long p0 = 0;+};+void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride,+ const AttnRows& a, const float* sink, int heads, int d, int T, float scale, float* out, cudaStream_t s,+ int ckv_fmt = kKvFp4, int heads_per_block = 0, const AttnScratch* scr = nullptr);+/// The window buffer: ext[r] = ring[(p0 - n + r) % n] for the n rows before p0 (zeros before position 0); afterwards+/// ring[(p0 + t) % n] = ext[n + t] for t < T. The ring's rows kKvFp8 (each group of 32 at the smallest power-of-two+/// scale its BF16 values fit under: exact for values on the FP8 grid, the window's), the window buffer's BF16 (p0 = n:+/// the ring's rows as they lie).+void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s);+void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s,+ uint8_t* backup = nullptr);+/// A speculative window's rollback: ring[(p0 + t) % n] = backup[t] for keep <= t < T (the rows ext_to_ring replaced).+void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);+/// The same for up to kMaxRingBatch layers' rings at once (one launch).+constexpr int kMaxRingBatch = 64;+struct RingBatch {+ uint8_t* ring[kMaxRingBatch] = {};+ const uint8_t* bak[kMaxRingBatch] = {};+ int n = 0;+};+void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s);++/// Engram. rows: n FP8 E4M3 rows of hd values, scales n x (hd / 32) E8M0 -> out F32 [n * hd]. q6k: the rows are+/// ggml Q6_K superblocks instead (hd / 256 of them, 210 bytes each, at the start of each row's hd bytes; no scales),+/// decoded as ggml's dequantize_row_q6_K.+void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, bool q6k = false);+/// The same over the host's per-token layout (a chunk): token t's cols rows, then their scales, `per` bytes a token;+/// T tokens -> out F32 [T * cols][hd].+void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s,+ bool q6k = false);+/// h[t][k] += gate[t][k] * value[t] with the gate from the normalized dot of h[t][k] * w[k] * key[t][k]+/// (kv[t] = [key hc x dim | value dim]).+void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s);++/// out[r] = x[src[r]] for r < n (rows of cols floats).+struct Gather {+ int src[kWin * kPicks];+};+void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s);+/// The MoE's sum, token t: out[t] = sum_j w[t][j] ed[row[t][j]] (its VRAM experts, in pick order) + cpu[t] (when its+/// cpu flag is set) + sh[t].+struct Combine {+ int n[kWin];+ int row[kWin][kPicks];+ float w[kWin][kPicks];+ int cpu[kWin];+};+void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,+ cudaStream_t s);++/// The mailboxes of a small window (decode, a draft's verification, the drafter's block): host-mapped memory, a ring of+/// kRing. The device publishes a layer's routing and its FFN inputs for the window's T tokens (seq written last, after+/// a system fence); the host answers with the CPU lane's weighted part of each token that took a CPU-lane expert.+constexpr int kRing = 64;+constexpr int kMaxDim = 8192;+constexpr int kAhead1 = 16, kAhead = kAhead1 + kPicks; // a token's guesses for the next layer, then the one after+struct Req {+ unsigned seq;+ int T;+ int na[2]; // lookahead: ahead[t][0 .. na[0]) the next layer's likely picks, ahead[t][kAhead1 ..) +na[1] the one after's+ int ids[kWin][kPicks];+ float w[kWin][kPicks];+ int ahead[kWin][kAhead];+ float x[kWin][kMaxDim];+};+struct Resp {+ unsigned seq;+ int pad[3];+ int cpu[kWin]; // 1: part[t] is the CPU lane's weighted sum for token t+ float part[kWin][kMaxDim];+};+/// route() for the T tokens of a window, plus: plan_ptr[t][i] = vram[idx[t][i]] (0: not on the device), each token's x+/// (dim floats) and picks into the request slot, then T and seq (req null: no request, the plan alone).+void route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale,+ const unsigned long long* vram, const float* x, int dim, int* idx, float* w,+ unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl = nullptr,+ unsigned vl = 0, int na0 = 0, int na1 = 0);+/// Lookahead: the m picks a later layer will likely make (its gate's logits on this layer's FFN input, the routing's+/// score + bias, the m best of n) into req->ahead[t][off ..) for the T tokens - before route_publish, which publishes them.+void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s);+/// Waits (one thread, on the device) until *seq_ptr == seq.+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s);+/// A chunk's routed experts as sorted rows: the picks (t, i < k) of T tokens (idx rows kPicks apart) by the virtual+/// slot vs[expert] (n_vs of them: the VRAM slots, then the experts streamed in), each slot's rows in token order:+/// bounds[n_vs + 1], row_tok[r], pos[t][i] (the pick's sorted row, rows kPicks apart).+void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,+ cudaStream_t s);+/// acc[t] += w[t][i] * y[pos[t][i] - r0] for each pick of t (in pick order) whose sorted row is in [r0, r1).+void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,+ cudaStream_t s);+/// rows[t][0] = 1, the rest 0 (the first layer's pre-mix: stream 0).+void init_pre(float* pre, int T, cudaStream_t s);++/// out[t][e] = mean over the hc streams of h[t][s][e] (the DSpark drafter's taps), rows out_stride floats apart.+void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s);+/// out[t][i] = i for i < n, rows `stride` apart (the drafter's block rows as every query's selection).+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s);+/// The DSpark markov head: logits[v] += E[*tok] . H[v] for v < V (E BF16 [V][R], H F16 [V][R]).+void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s);+/// The markov bias over a head group: a member's rows [v0, v0 + vn) of it added to its share of a logits row (erow:+/// the previous token's E row, BF16, on the device) - markov_add's sums; the share's argmax (the greedy sampler's rule)+/// into a host-mapped slot {value, index, seq}; every member's slot of the step -> the global argmax into *tok and its+/// E row (E host-mapped) into erow.+void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s);+void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s);+void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,+ cudaStream_t s);+/// The DSpark confidence: out[t] = sigmoid(w . [x[t] | E[tok[t]]]) (x rows dim floats, w F32 [dim + R]).+void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,+ cudaStream_t s);++/// The pipeline's hand-offs and tensor parallelism, through host-mapped memory (these cards copy peer to peer only+/// through the driver's staging). xfer_put copies up to 4 segments (16-byte multiples, 16-byte aligned) device ->+/// host-mapped, then (flag given) sets *flag = seq after a system fence (cnt: a device counter of the stream, 0+/// between uses); xfer_get waits on the device until *flag >= seq, then copies host-mapped -> device.+struct Xfer {+ const void* src[4] = {nullptr, nullptr, nullptr, nullptr};+ void* dst[4] = {nullptr, nullptr, nullptr, nullptr};+ size_t bytes[4] = {0, 0, 0, 0};+ int n = 0;+};+void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s);+void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s);+/// Waits (one thread) until flags[i * stride] >= seq for i < n (a pipelined chunk's readers have taken a buffer);+/// set_flag: *flag = v after a system fence (a reader's acknowledgement).+void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s);+void set_flag(unsigned* flag, unsigned v, cudaStream_t s);+/// A tensor-parallel group's all-reduce: host-mapped rows data[member][slot] (stride floats), flags[member][slot]+/// (32 apart). x (n floats, n % 4 == 0: this member's partial) -> the sum of the nm members' partials in member order,+/// the same bits on every member; seq counts the group's all-reduces (the same on each member), slot = seq % slots.+struct ArGroup {+ float* data = nullptr;+ unsigned* flags = nullptr;+ size_t stride = 0;+ int nm = 1, slots = 4;+ // three members or more: a reduce-scatter + all-gather (each member sums its chunk of every block's range, in+ // member order, then reads the others' sums) - [slots][stride] sums, [2][nm][slots][kArBlocks][8] flags+ float* sums = nullptr;+ unsigned* bflags = nullptr;+};+constexpr int kArBlocks = 64;+void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s);++} // namespace strata::ds4diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppnew file mode 100644index 000000000..7916a0384--- /dev/null+++ b/src/dsv41/ds4_main.cpp@@ -0,0 +1,771 @@+// src/ds4/ds4_main.cpp - maya-ds4: DeepSeek-V4.1-Flash on Maya.+//+// maya-ds4 --pack <model>/maya-ds4 --ids 0,128803,... --n 16 greedy tokens after the ids+// maya-ds4 --pack ... --ids ... --dump-logits out.f32 every position's logits (the oracle's check)+// maya-ds4 --pack ... --serve Maya's engine protocol on stdin / stdout+//+// --serve speaks what serve/server.py's StrataEngine expects: INFO lines, then `READY <context> stop`; per request+// `GEN <max_new> [key=value ...] <id,id,...>` answered by `T <id>` lines and `DONE <generated> <prompt> <prompt ms>+// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that+// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to+// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures+// (the file: see read_pics); the state is shared only where the same picture sits at the same place.+#include "ds4_model.hpp"+#include "strata/platform/env_alias.hpp"++#include <algorithm>+#include <atomic>+#include <chrono>+#include <condition_variable>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <deque>+#include <iostream>+#include <mutex>+#include <string>+#include <thread>+#include <vector>++namespace {++double now_ms() {+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();+}++// the bytes this process had the storage read for it (Linux's /proc/self/io; -1 where there is none): the Monitor's+// disk reads a token, the experts read in place from the model's files included+long long io_read_bytes() {+#ifdef __linux__+ std::FILE* f = std::fopen("/proc/self/io", "r");+ if (f == nullptr) return -1;+ char line[128];+ long long v = -1;+ while (std::fgets(line, sizeof line, f) != nullptr)+ if (std::sscanf(line, "read_bytes: %lld", &v) == 1) break;+ std::fclose(f);+ return v;+#else+ return -1;+#endif+}++std::vector<int> parse_ids(const std::string& s) {+ std::vector<int> v;+ const char* p = s.c_str();+ while (*p) {+ char* e = nullptr;+ const long x = std::strtol(p, &e, 10);+ if (e == p) break;+ v.push_back((int) x);+ p = e;+ while (*p == ',' || *p == ' ') ++p;+ }+ return v;+}++void usage() {+ std::fprintf(stderr,+ "maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"+ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--no-expert-cache] [--verbose]\n"+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"+ " [--vision on|off] [--mmap-experts on|off|auto]\n"+ " (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"+ " | --encode-image FILE.rgb --size WxH --dump-image OUT)\n");+}++// stdin lines on a thread, so a request can see STOP while it decodes+struct Lines {+ std::mutex mu;+ std::condition_variable cv;+ std::deque<std::string> q;+ bool eof = false;+ std::atomic<bool> stop{false};+ void start() {+ std::thread([this] {+ std::string l;+ while (std::getline(std::cin, l)) {+ if (!l.empty() && l.back() == '\r') l.pop_back();+ std::lock_guard<std::mutex> lk(mu);+ if (l == "STOP" || l.rfind("STOP ", 0) == 0) {+ stop.store(true);+ continue;+ }+ if (l == "QUIT") stop.store(true);+ q.push_back(l);+ cv.notify_all();+ }+ std::lock_guard<std::mutex> lk(mu);+ eof = true;+ stop.store(true);+ cv.notify_all();+ }).detach();+ }+ bool next(std::string& l) {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return !q.empty() || eof; });+ if (q.empty()) return false;+ l = q.front();+ q.pop_front();+ return true;+ }+};++// The drafts between two stats: each draft position (1st, 2nd, ..) - how often it was verified with every draft+// before it kept, and how often it was kept then+std::string per_position(const strata::ds4::Stats& s0, const strata::ds4::Stats& s1) {+ std::string pp = "per position (kept / reached):";+ for (int i = 0; i < 8; ++i) {+ const long long r = s1.draft_reached[i] - s0.draft_reached[i], k = s1.draft_kept[i] - s0.draft_kept[i];+ if (r <= 0) break;+ char b[64];+ std::snprintf(b, sizeof b, " %d: %lld/%lld %.0f%%", i + 1, k, r, 100.0 * (double) k / (double) r);+ pp += b;+ }+ return pp;+}++// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their+// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow+// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.+bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,+ float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,+ int& accepted, std::string& err) {+ out.clear();+ offered = accepted = 0;+ if (spec && room > 1 && m.draft_block() > 0) {+ std::vector<int> drafts;+ std::vector<float> conf;+ if (!m.draft(tok, drafts, conf, err, temperature, top_p, top_k, min_p, seed, counter)) return false;+ int keep = 0;+ while (keep < (int) drafts.size() && keep < room - 1 && conf[(size_t) keep] >= conf_min) ++keep;+ if (keep > 0) {+ drafts.resize((size_t) keep);+ int acc = 0, next = -1;+ if (!m.verify(tok, drafts, temperature, top_p, top_k, min_p, seed, counter, acc, next, err)) return false;+ counter += (unsigned long long) acc + 1;+ for (int i = 0; i < acc; ++i) out.push_back(drafts[(size_t) i]);+ out.push_back(next);+ offered = keep;+ accepted = acc;+ return true;+ }+ }+ if (!m.step(tok, true, err)) return false;+ out.push_back(m.sample(temperature, top_p, top_k, min_p, seed, counter++));+ return true;+}++struct Req {+ long long max_new = 0;+ float temperature = 0.0f, top_p = 1.0f, min_p = 0.0f;+ int top_k = 64;+ unsigned long long seed = 0;+ int spec = -1; // spec=0|1: this request drafts or not (-1: as the engine was started)+ float conf = -1.0f; // dspark_conf=F: the drafts' confidence floor for this request (-1: the engine's)+ std::string file; // GENI: the pictures+ std::vector<int> ids;+};++// A prompt's picture: where its span starts, its pixels (the server's resize: RGB rows, both sides multiples of the+// patch size), a hash of them; once encoded, its span's rows+struct Pic {+ int64_t start = 0;+ int width = 0, height = 0;+ uint64_t hash = 0;+ size_t len = 0; // the span (0: not known yet)+ std::vector<uint8_t> rgb;+};++// GENI's file: int32 'DSI1', int32 count, then per picture int64 start, int32 width, int32 height and its+// height x width x 3 RGB bytes, in prompt order+bool read_pics(const std::string& path, std::vector<Pic>& v, std::string& err) {+ std::FILE* f = std::fopen(path.c_str(), "rb");+ if (f == nullptr) {+ err = "cannot open the pictures' file " + path;+ return false;+ }+ char magic[4] = {};+ int32_t n = 0;+ bool ok = std::fread(magic, 1, 4, f) == 4 && std::memcmp(magic, "DSI1", 4) == 0 && std::fread(&n, 4, 1, f) == 1 &&+ n >= 0 && n <= 4096;+ for (int i = 0; ok && i < n; ++i) {+ Pic p;+ int32_t wh[2] = {};+ ok = std::fread(&p.start, 8, 1, f) == 1 && std::fread(wh, 4, 2, f) == 2 && wh[0] > 0 && wh[1] > 0 &&+ (int64_t) wh[0] * wh[1] <= (64 << 20);+ if (!ok) break;+ p.width = wh[0], p.height = wh[1];+ p.rgb.resize((size_t) p.width * p.height * 3);+ ok = std::fread(p.rgb.data(), 1, p.rgb.size(), f) == p.rgb.size();+ uint64_t h = 1469598103934665603ull; // FNV-1a over the size and the pixels+ for (int32_t x : wh)+ for (int b = 0; b < 4; ++b) h = (h ^ (uint8_t) (x >> (8 * b))) * 1099511628211ull;+ for (uint8_t b : p.rgb) h = (h ^ b) * 1099511628211ull;+ p.hash = h;+ v.push_back(std::move(p));+ }+ std::fclose(f);+ if (!ok) err = path + ": not a pictures' file (DSI1)";+ return ok;+}++// the encoded spans, the latest few (a conversation sends its pictures with every turn)+struct SpanCache {+ std::deque<std::pair<uint64_t, std::shared_ptr<const std::vector<uint16_t>>>> q;+ std::shared_ptr<const std::vector<uint16_t>> get(strata::ds4::Model& m, const Pic& p, std::string& err) {+ for (size_t i = 0; i < q.size(); ++i)+ if (q[i].first == p.hash) {+ auto r = q[i].second;+ q.erase(q.begin() + (long) i);+ q.emplace_front(p.hash, r);+ return r;+ }+ auto rows = std::make_shared<std::vector<uint16_t>>();+ if (!m.encode_image(p.rgb.data(), p.width, p.height, *rows, err)) return nullptr;+ q.emplace_front(p.hash, rows);+ if (q.size() > 8) q.pop_back();+ return rows;+ }+};++bool parse_gen(const std::string& line, Req& r, bool geni) {+ const char* p = line.c_str() + (geni ? 5 : 4);+ char* e = nullptr;+ r.max_new = std::strtoll(p, &e, 10);+ if (e == p) return false;+ p = e;+ for (;;) {+ while (*p == ' ') ++p;+ const char* s = p;+ while (*p && *p != ' ') ++p;+ if (p == s) break;+ const std::string tok(s, (size_t) (p - s));+ const size_t eq = tok.find('=');+ if (eq == std::string::npos && geni && r.file.empty()) {+ r.file = tok;+ continue;+ }+ if (eq == std::string::npos) {+ r.ids = parse_ids(tok);+ break;+ }+ const std::string k = tok.substr(0, eq), v = tok.substr(eq + 1);+ if (k == "temperature") r.temperature = std::strtof(v.c_str(), nullptr);+ else if (k == "top_p") r.top_p = std::strtof(v.c_str(), nullptr);+ else if (k == "top_k") r.top_k = std::atoi(v.c_str());+ else if (k == "min_p") r.min_p = std::strtof(v.c_str(), nullptr);+ else if (k == "seed") r.seed = std::strtoull(v.c_str(), nullptr, 10);+ else if (k == "spec") r.spec = std::atoi(v.c_str()) != 0;+ else if (k == "dspark_conf") r.conf = std::strtof(v.c_str(), nullptr);+ }+ return !r.ids.empty() && (!geni || !r.file.empty());+}++int serve(strata::ds4::Model& m, const std::string& eos_list, bool spec, float conf_min) {+ std::vector<int> eos = parse_ids(eos_list);+ for (const std::string& l : m.info_lines()) std::printf("%s\n", l.c_str());+ std::printf("READY %lld stop\n", (long long) m.max_context());+ std::fflush(stdout);+ std::fprintf(stderr, "maya ds4 serve: GEN <max_new> [keys] <ids>; GENI <max_new> [keys] <pictures> <ids>; STOP, QUIT\n");+ Lines in;+ in.start();+ std::string line;+ unsigned long long counter = 0;+ std::vector<Pic> held; // the pictures the state holds (start, hash, span)+ SpanCache spans;+ const int img_tok = m.image_token();+ while (in.next(line)) {+ if (line == "QUIT") break;+ const bool geni = line.rfind("GENI ", 0) == 0;+ if (!geni && line.rfind("GEN ", 0) != 0) {+ std::printf("ERR unsupported command '%s' (GEN, GENI, STOP and QUIT)\n", line.substr(0, line.find(' ')).c_str());+ std::fflush(stdout);+ continue;+ }+ in.stop.store(false);+ Req r;+ if (!parse_gen(line, r, geni)) {+ std::printf("ERR expected: GEN <max_new> [keys] <id,id,...> or GENI <max_new> [keys] <file> <id,id,...>\n");+ std::fflush(stdout);+ continue;+ }+ const double t0 = now_ms();+ std::vector<Pic> pics;+ {+ std::string e;+ if (geni && img_tok < 0) e = "this engine has no vision tower, so it cannot read pictures";+ else if (geni && read_pics(r.file, pics, e))+ for (const Pic& q : pics)+ if (q.start < 0 || (size_t) q.start >= r.ids.size() || r.ids[(size_t) q.start] != img_tok)+ e = "a picture's span does not start at an image token of the prompt";+ if (!e.empty()) {+ std::printf("ERR %s\n", e.c_str());+ std::printf("DONE 0 %zu 0.0 0.0 cancel 0 0 0 0 0\n", r.ids.size());+ std::fflush(stdout);+ continue;+ }+ }+ const auto& st = m.tokens();+ const size_t n_state = st.size();+ size_t c = 0;+ while (c < n_state && c < r.ids.size() && st[c] == r.ids[c]) ++c;+ // the same tokens with another picture (or none) do not share the state from that picture's span on+ auto same = [](const std::vector<Pic>& v, const Pic& q) {+ for (const Pic& x : v)+ if (x.start == q.start && x.hash == q.hash) return true;+ return false;+ };+ for (const Pic& q : pics)+ if ((size_t) q.start < c && !same(held, q)) c = (size_t) q.start;+ for (const Pic& q : held)+ if ((size_t) q.start < c && !same(pics, q)) c = (size_t) q.start;+ size_t from = 0;+ if (c == n_state && n_state < r.ids.size()) from = n_state;+ else if (m.restorable((int64_t) std::min(c, r.ids.size() - 1)) >= 0 &&+ m.restore((int64_t) std::min(c, r.ids.size() - 1)))+ from = (size_t) m.pos();+ else m.reset();+ if (from == 0) held.clear();+ if ((int64_t) r.ids.size() + std::max(0LL, r.max_new) > m.max_context()) {+ std::printf("ERR the prompt (%zu tokens) and max_new %lld are more than the context %lld\n", r.ids.size(),+ r.max_new, (long long) m.max_context());+ std::fflush(stdout);+ continue;+ }+ std::string err;+ bool ok = true;+ // the pictures whose spans the prompt reads (the rest the state holds): encoded (or the cache's), checked+ if (!pics.empty()) {+ std::vector<strata::ds4::Image> imgs;+ const double te = now_ms();+ int encoded = 0;+ for (Pic& q : pics) {+ for (const Pic& x : held)+ if (x.start == q.start && x.hash == q.hash) q.len = x.len;+ if (q.len > 0 && (size_t) q.start + q.len <= from) continue;+ std::shared_ptr<const std::vector<uint16_t>> rows = spans.get(m, q, err);+ if (!rows) {+ ok = false;+ break;+ }+ ++encoded;+ q.len = rows->size() / (size_t) m.dim();+ bool fits = (size_t) q.start + q.len <= r.ids.size() - 1;+ for (size_t i = 0; fits && i < q.len; ++i) fits = r.ids[(size_t) q.start + i] == img_tok;+ if (!fits) {+ err = "a picture's span (" + std::to_string(q.len) + " tokens at " + std::to_string(q.start) ++ ") is not a run of image tokens in the prompt";+ ok = false;+ break;+ }+ imgs.push_back({q.start, rows});+ }+ if (ok) {+ m.set_images(std::move(imgs));+ std::fprintf(stderr, "maya ds4 serve: %zu pictures, %d read (%.0f ms)\n", pics.size(), encoded,+ now_ms() - te);+ }+ for (Pic& q : pics) q.rgb = {}; // (only the start, hash and span stay)+ }+ // the prompt in windows (PP <read> <total> <ms> <tok/s> <reuse> every 64 tokens), a checkpoint before its last+ // token and one after it: the next turn re-renders this one, and shares its start up to one of them+ const size_t total = r.ids.size();+ size_t done = from, next_pp = from + 64;+ m.set_progress([&](int64_t d) { // (a run of chunks: each one through every layer)+ const double ms = now_ms() - t0;+ std::printf("PP %zu %zu %.1f %.1f %zu\n", done + (size_t) d, r.ids.size(), ms,+ ms > 0 ? 1000.0 * (double) (done + (size_t) d - from) / ms : 0.0, from);+ std::fflush(stdout);+ });+ // (DS4_LAST_WINDOW=1: the last token in a window of its own after the checkpoint, as before - for A/B runs)+ static const bool last_window = std::getenv("DS4_LAST_WINDOW") != nullptr;+ while (ok && done < total) {+ const size_t n = std::min<size_t>((size_t) m.feed_unit(), total - done);+ // (the prompt's end: the last token in the same run, the checkpoint before it kept on the way)+ const bool fin = done + n == total;+ if (fin && last_window) {+ ok = (n < 2 || m.feed(r.ids.data() + done, (int) n - 1, 0, err));+ if (ok) m.checkpoint(1);+ ok = ok && m.feed(r.ids.data() + done + n - 1, 1, 1, err);+ } else+ ok = fin ? m.feed_mark(r.ids.data() + done, (int) n, 1, 1, err)+ : m.feed(r.ids.data() + done, (int) n, 0, err);+ done += n;+ if (ok && (done >= next_pp || done == total)) {+ const double ms = now_ms() - t0;+ std::printf("PP %zu %zu %.1f %.1f %zu\n", done, r.ids.size(), ms,+ ms > 0 ? 1000.0 * (double) (done - from) / ms : 0.0, from);+ std::fflush(stdout);+ next_pp = done + 64;+ }+ }+ m.set_images({});+ if (!ok) {+ std::fprintf(stderr, "ds4: %s\n", err.c_str());+ std::printf("ERR %s\n", err.c_str());+ std::printf("DONE 0 %zu 0.0 0.0 cancel 0 0 0 0 0\n", r.ids.size());+ std::fflush(stdout);+ m.reset();+ held.clear();+ continue;+ }+ held = std::move(pics);+ m.checkpoint(0);+ const double t1 = now_ms();+ long long produced = 0;+ const char* finish = "length";+ const long long limit = r.max_new > 0 ? r.max_new : m.max_context() - (long long) r.ids.size();+ long long d_off = 0, d_acc = 0;+ bool cool = false;+ int tok = m.sample(r.temperature, r.top_p, r.top_k, r.min_p, r.seed, counter++);+ std::vector<int> got;+ // STAT for the Monitor every 8 tokens and at the end, over this answer so far: the GLM engine's keys (speed, the+ // VRAM hit, promotions a token) and DeepSeek's own - experts a token on the CPU lane, MB a token the storage+ // read (Linux), the n-gram memory's (Engram) read time a token, the drafts' acceptance+ const strata::ds4::Stats s_dec = m.stats();+ const long long io_dec = io_read_bytes();+ auto stat_line = [&]() {+ const strata::ds4::Stats& s = m.stats();+ const double ms = now_ms() - t1;+ const double fed = (double) std::max<int64_t>(1, s.tokens - s_dec.tokens);+ const double ex = (double) (s.experts - s_dec.experts), cx = (double) (s.cpu_experts - s_dec.cpu_experts);+ std::string out = "STAT";+ char b[160];+ std::snprintf(b, sizeof b, " tok_s=%.2f ms_tok=%.2f vram_hit=%.4f cpu=%.3f ram=%.3f promo=%.3f engram_ms=%.3f",+ ms > 0 ? 1000.0 * (double) produced / ms : 0.0, produced > 0 ? ms / (double) produced : 0.0,+ ex > 0 ? 1.0 - cx / ex : 1.0, cx / fed, (double) (s.ram_experts - s_dec.ram_experts) / fed,+ (double) (s.swaps - s_dec.swaps) / fed, (s.engram_ms - s_dec.engram_ms) / fed);+ out += b;+ const long long io = io_read_bytes();+ if (io >= 0 && io_dec >= 0) {+ std::snprintf(b, sizeof b, " disk_mb=%.3f", (double) (io - io_dec) / 1e6 / fed);+ out += b;+ }+ if (d_off > 0) {+ std::snprintf(b, sizeof b, " drafts=%.3f", (double) d_acc / (double) d_off);+ out += b;+ }+ if (s.skipped > s_dec.skipped) { // (STRATA_DS4_DISK_SKIP_W: experts left out a token)+ std::snprintf(b, sizeof b, " skipped=%.3f", (double) (s.skipped - s_dec.skipped) / fed);+ out += b;+ }+ std::printf("%s\n", out.c_str());+ };+ for (;;) {+ if (in.stop.load()) {+ finish = "cancel";+ break;+ }+ std::printf("T %d\n", tok);+ ++produced;+ if (produced % 8 == 0) stat_line();+ std::fflush(stdout);+ bool end = false;+ for (int e : eos) end = end || tok == e;+ if (end) {+ finish = "stop";+ break;+ }+ if (produced >= limit || m.pos() >= m.max_context()) break;+ if (got.empty()) { // the next round: the drafts (if any) and the token after them+ const int room = (int) std::min<long long>(limit - produced, m.max_context() - m.pos());+ int off = 0, acc = 0;+ // a round whose drafts were all below the confidence floor: the next one does without the drafter+ // (a request's spec / dspark_conf keys over the engine's: the speed calibration measures both ways)+ const bool try_spec = (r.spec < 0 ? spec : r.spec != 0) && !cool;+ if (!round(m, tok, room, try_spec, r.conf >= 0 ? r.conf : conf_min, r.temperature, r.top_p, r.top_k,+ r.min_p, r.seed, counter, got,+ off, acc, err)) {+ std::fprintf(stderr, "ds4: %s\n", err.c_str());+ finish = "cancel";+ break;+ }+ d_off += off;+ d_acc += acc;+ cool = try_spec && off == 0;+ std::reverse(got.begin(), got.end());+ }+ tok = got.back();+ got.pop_back();+ }+ // (a stop within accepted drafts leaves the state past it: the next prompt shares less with it, and goes back to+ // the checkpoint at this prompt's end)+ stat_line();+ const double t2 = now_ms();+ std::printf("DONE %lld %zu %.1f %.1f %s %lld %lld %zu 0 0\n", produced, r.ids.size(), t1 - t0, t2 - t1, finish,+ d_acc, d_off, from);+ std::fflush(stdout);+ const auto& s = m.stats();+ std::fprintf(stderr,+ "maya ds4 serve: prompt %zu tokens = %zu reused + %zu read in %.0f ms (%.1f tok/s), %lld generated "+ "in %.0f ms (%.2f tok/s); CPU lane %.1f%% of the experts\n",+ r.ids.size(), from, r.ids.size() - from, t1 - t0,+ t1 > t0 ? 1000.0 * (double) (r.ids.size() - from) / (t1 - t0) : 0.0, produced, t2 - t1,+ t2 > t1 ? 1000.0 * (double) produced / (t2 - t1) : 0.0,+ s.experts ? 100.0 * (double) s.cpu_experts / (double) s.experts : 0.0);+ if (d_off > 0) std::fprintf(stderr, "maya ds4 serve: drafts %lld of %lld kept; %s\n", d_acc, d_off,+ per_position(s_dec, s).c_str());+ m.save_profile();+ }+ m.save_profile();+ return 0;+}++} // namespace++int main(int argc, char** argv) {+ strata::platform::env_aliases(); // (MAYA_* settings under their older names too)+ strata::ds4::Options o;+ std::string ids_s, dump, eos = "1";+ int n = 0, window = 16;+ bool spec = true;+ float temp = 0.0f; // the CLI's sampling (0: greedy)+ unsigned long long cli_seed = 1;+ bool serve_mode = false;+ std::string images, enc_rgb, enc_out;+ int enc_w = 0, enc_h = 0;+ for (int i = 1; i < argc; ++i) {+ const std::string a = argv[i];+ auto val = [&](const char* what) -> std::string {+ if (i + 1 >= argc) {+ std::fprintf(stderr, "%s needs a value\n", what);+ std::exit(2);+ }+ return argv[++i];+ };+ if (a == "--pack") o.pack = val("--pack");+ else if (a == "--gpus") {+ for (int g : parse_ids(val("--gpus"))) o.gpus.push_back(g);+ } else if (a == "--max-context") o.max_context = std::atoll(val("--max-context").c_str());+ else if (a == "--reserve-mib") o.reserve_mib = std::atoll(val("--reserve-mib").c_str());+ else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());+ else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());+ else if (a == "--expert-profile") o.profile = val("--expert-profile");+ else if (a == "--eos") eos = val("--eos");+ else if (a == "--no-expert-cache") o.expert_cache = false;+ else if (a == "--dspark") {+ const std::string v = val("--dspark");+ spec = v != "off";+ o.dspark = v != "none"; // none: not even loaded+ } else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());+ else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));+ else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));+ else if (a == "--threads") o.threads = val("--threads") != "off";+ else if (a == "--vision") o.vision = val("--vision") != "off";+ else if (a == "--mmap-experts") {+ const std::string v = val("--mmap-experts");+ o.mmap_experts = v == "on" ? 1 : v == "off" ? 0 : -1;+ } else if (a == "--images") images = val("--images");+ else if (a == "--encode-image") enc_rgb = val("--encode-image");+ else if (a == "--dump-image") enc_out = val("--dump-image");+ else if (a == "--size") {+ const std::string v = val("--size");+ enc_w = std::atoi(v.c_str());+ enc_h = v.find('x') != std::string::npos ? std::atoi(v.c_str() + v.find('x') + 1) : 0;+ }+ else if (a == "--temperature") temp = (float) std::atof(val("--temperature").c_str());+ else if (a == "--seed") cli_seed = std::strtoull(val("--seed").c_str(), nullptr, 10);+ else if (a == "--verbose" || a == "-v") o.verbose = true;+ else if (a == "--ids") ids_s = val("--ids");+ else if (a == "--n") n = std::atoi(val("--n").c_str());+ else if (a == "--window") window = std::max(1, std::atoi(val("--window").c_str())); // (> 16: the chunked prompt path)+ else if (a == "--dump-logits") dump = val("--dump-logits");+ else if (a == "--serve") serve_mode = true;+ else if (a == "--layer-split") (void) val("--layer-split"); // the server's (this engine balances its own split)+ else if (a == "--help" || a == "-h") {+ usage();+ return 0;+ } else {+ std::fprintf(stderr, "unknown option %s\n", a.c_str());+ usage();+ return 2;+ }+ }+ if (o.pack.empty() || (!serve_mode && ids_s.empty() && enc_rgb.empty())) {+ usage();+ return 2;+ }+ if (o.profile.empty()) o.profile = o.pack + "/expert-profile.bin"; // "none": no profile+ if (o.profile == "none") o.profile.clear();+ o.vision_only = !enc_rgb.empty();+ strata::ds4::Model m;+ std::string err;+ if (!m.load(o, err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ if (!o.vision_only) std::fprintf(stderr, "%s", m.describe().c_str());+ if (serve_mode) return serve(m, eos, spec, o.dspark_conf);++ if (!enc_rgb.empty()) { // the vision tower alone: a picture's span rows (BF16 [rows][dim]) into --dump-image+ std::vector<uint8_t> px((size_t) std::max(0, enc_w) * std::max(0, enc_h) * 3);+ std::FILE* f = std::fopen(enc_rgb.c_str(), "rb");+ const bool got = f != nullptr && !px.empty() && std::fread(px.data(), 1, px.size(), f) == px.size();+ if (f != nullptr) std::fclose(f);+ if (!got) {+ std::fprintf(stderr, "maya-ds4: %s: expected %d x %d RGB bytes (--size WxH)\n", enc_rgb.c_str(), enc_w, enc_h);+ return 1;+ }+ std::vector<uint16_t> span;+ for (int k = 0; k < 2; ++k) { // (the second: warm)+ const double te = now_ms();+ if (!m.encode_image(px.data(), enc_w, enc_h, span, err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ std::fprintf(stderr, "vision: %d x %d pixels -> a span of %zu rows in %.0f ms\n", enc_w, enc_h,+ span.size() / (size_t) m.dim(), now_ms() - te);+ }+ std::FILE* o2 = enc_out.empty() ? nullptr : std::fopen(enc_out.c_str(), "wb");+ if (o2 != nullptr) {+ std::fwrite(span.data(), 2, span.size(), o2);+ std::fclose(o2);+ }+ return 0;+ }+ const std::vector<int> ids = parse_ids(ids_s);+ if (!images.empty()) { // --images: a GENI file's pictures, for the prompt's spans+ std::vector<Pic> pics;+ if (!read_pics(images, pics, err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ std::vector<strata::ds4::Image> imgs;+ for (const Pic& q : pics) {+ auto rows = std::make_shared<std::vector<uint16_t>>();+ const double te = now_ms();+ if (!m.encode_image(q.rgb.data(), q.width, q.height, *rows, err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ std::fprintf(stderr, "picture at %lld: %d x %d pixels, %zu tokens, %.0f ms\n", (long long) q.start, q.width,+ q.height, rows->size() / (size_t) m.dim(), now_ms() - te);+ imgs.push_back({q.start, rows});+ }+ m.set_images(std::move(imgs));+ }+ std::FILE* df = dump.empty() ? nullptr : std::fopen(dump.c_str(), "wb");+ std::vector<float> lg;+ const double t0 = now_ms();+ for (size_t a = 0; a < ids.size();) {+ const int T = (int) std::min<size_t>((size_t) window, ids.size() - a);+ const bool last = a + (size_t) T == ids.size();+ if (!m.feed(ids.data() + a, T, df != nullptr ? T : (last ? 1 : 0), err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ if (df != nullptr && m.logits(lg)) std::fwrite(lg.data(), 4, lg.size(), df);+ a += (size_t) T;+ }+ m.set_images({});+ const double t1 = now_ms();+ std::fprintf(stderr, "prompt: %zu tokens in %.0f ms (%.2f tok/s, windows of %d)\n", ids.size(), t1 - t0,+ 1000.0 * (double) ids.size() / (t1 - t0), window);+ const strata::ds4::Stats s0 = m.stats();+ if (s0.eg_rows > 0)+ std::fprintf(stderr, "prompt's Engram rows: %lld, %.1f%% without a read (repeats and the row cache), %lld read\n",+ (long long) s0.eg_rows, 100.0 * (double) s0.eg_hits / (double) s0.eg_rows, (long long) s0.eg_reads);+ const long long io0 = io_read_bytes();+ std::vector<int> out;+ unsigned long long counter = 0;+ long long d_off = 0, d_acc = 0, rounds = 0;+ bool cool = false;+ if (n > 0) {+ int tok = m.sample(temp, 1.0f, temp > 0 ? 64 : 1, 0.0f, cli_seed, counter++);+ std::vector<int> got;+ for (;;) {+ out.push_back(tok);+ std::printf("%d,", tok);+ std::fflush(stdout);+ if (tok == 1 || (int) out.size() >= n) break;+ if (got.empty()) {+ int off = 0, acc = 0;+ const bool try_spec = spec && !cool;+ if (!round(m, tok, n - (int) out.size(), try_spec, o.dspark_conf, temp, 1.0f, temp > 0 ? 64 : 1, 0.0f,+ cli_seed, counter, got, off, acc, err)) {+ std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());+ return 1;+ }+ d_off += off;+ d_acc += acc;+ ++rounds;+ cool = try_spec && off == 0;+ std::reverse(got.begin(), got.end());+ }+ tok = got.back();+ got.pop_back();+ }+ std::printf("\n");+ if (df != nullptr && m.logits(lg)) std::fwrite(lg.data(), 4, lg.size(), df);+ }+ if (spec && rounds > 0) {+ const auto& s1 = m.stats();+ const double nv = (double) std::max<int64_t>(1, s1.drafts_rounds - s0.drafts_rounds);+ std::fprintf(stderr, "dspark: %lld rounds, %lld drafts offered, %lld accepted (%.2f tokens a round); drafting "+ "%.2f ms a round, verifying %.1f ms in all (%lld windows)\n",+ rounds, d_off, d_acc, (double) out.size() / (double) rounds, (s1.draft_ms - s0.draft_ms) / nv,+ s1.verify_ms - s0.verify_ms, (long long) (s1.drafts_offered > 0 ? 0 : 0));+ std::fprintf(stderr, "dspark %s\n", per_position(s0, s1).c_str());+ }+ const double t2 = now_ms();+ const auto& s = m.stats();+ const double nt = std::max<double>(1.0, (double) (s.tokens - s0.tokens));+ std::fprintf(stderr, "decode: %zu tokens in %.0f ms (%.2f tok/s); CPU lane: %lld of %lld experts; the cache moved %lld "+ "(%lld in all)\n", out.size(), t2 - t1,+ out.size() > 1 ? 1000.0 * (double) (out.size() - 1) / (t2 - t1) : 0.0,+ (long long) (s.cpu_experts - s0.cpu_experts), (long long) (s.experts - s0.experts),+ (long long) (s.swaps - s0.swaps), (long long) s.swaps);+ std::fprintf(stderr, "decode per token: layers %.2f ms (of it: routing waits %.2f, CPU lane %.2f, the rest host launches), "+ "head wait %.2f, Engram reads %.2f\n",+ (s.layer_ms - s0.layer_ms) / nt, (s.gpu_wait_ms - s0.gpu_wait_ms) / nt, (s.cpu_ms - s0.cpu_ms) / nt,+ (s.head_ms - s0.head_ms) / nt, (s.engram_ms - s0.engram_ms) / nt);+ if (s.eg_rows > s0.eg_rows) { // the Engram rows: how many the row cache served, the reads' time behind the layers+ const double rows = (double) (s.eg_rows - s0.eg_rows);+ std::fprintf(stderr, "Engram per token: %.0f rows, %.1f%% without a read (the row cache holds %lld of %lld), "+ "%.1f read, the reads %.2f ms from asking to the last one (left to wait: %.2f)\n",+ rows / nt, 100.0 * (double) (s.eg_hits - s0.eg_hits) / rows, (long long) s.eg_cached,+ (long long) s.eg_cache_cap, (double) (s.eg_reads - s0.eg_reads) / nt, (s.eg_io_ms - s0.eg_io_ms) / nt,+ (s.engram_ms - s0.engram_ms) / nt);+ }+ const long long io1 = io_read_bytes();+ if (io0 >= 0 && io1 >= 0)+ std::fprintf(stderr, "decode reads from the disk: %.1f MB a token\n", (double) (io1 - io0) / 1e6 / nt);+ if (s.skipped > s0.skipped) // STRATA_DS4_DISK_SKIP_W+ std::fprintf(stderr, "decode left out %.1f experts a token not in RAM (%.1f%% of the routed)\n",+ (double) (s.skipped - s0.skipped) / nt,+ 100.0 * (double) (s.skipped - s0.skipped) / std::max<double>(1.0, (double) (s.experts - s0.experts)));+ if (s.ahead_picks > s0.ahead_picks) { // lookahead+ const double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);+ std::fprintf(stderr, "lookahead: the guesses a layer before held %.1f%% of the picks (their top k %.1f%%, two layers "+ "before %.1f%%); file-read picks asked for early %.1f%% (%lld of %lld); %.1f experts asked "+ "for early a token\n",+ 100.0 * (double) (s.ahead_hit - s0.ahead_hit) / p, 100.0 * (double) (s.ahead_hit_k - s0.ahead_hit_k) / p,+ 100.0 * (double) (s.ahead_hit2 - s0.ahead_hit2) / p,+ c > 0 ? 100.0 * (double) (s.ahead_cold_hit - s0.ahead_cold_hit) / c : 0.0,+ (long long) (s.ahead_cold_hit - s0.ahead_cold_hit), (long long) (s.ahead_cold - s0.ahead_cold),+ (double) (s.ahead_pf - s0.ahead_pf) / nt);+ std::string rk = "lookahead: picked by the guess's rank (1 ..):";+ for (int i = 0; i < 16; ++i) {+ const long long n = (long long) (s.ahead_rank_n[i] - s0.ahead_rank_n[i]);+ if (n <= 0) break;+ char b[24];+ std::snprintf(b, sizeof b, " %.0f%%", 100.0 * (double) (s.ahead_rank_hit[i] - s0.ahead_rank_hit[i]) / (double) n);+ rk += b;+ }+ std::fprintf(stderr, "%s\n", rk.c_str());+ }+ if (s.prof_tokens > s0.prof_tokens) { // DS4_PROF+ const double pt = (double) (s.prof_tokens - s0.prof_tokens);+ double sum = 0;+ for (int i = 0; i < 6; ++i) sum += (s.prof_ms[i] - s0.prof_ms[i]) / pt;+ std::fprintf(stderr, "GPU per token (DS4_PROF): %.2f ms - between layers %.2f, attention %.2f, routing + shared "+ "expert %.2f, experts gate/up %.2f, waiting for the CPU lane %.2f, down + post %.2f\n", sum,+ (s.prof_ms[0] - s0.prof_ms[0]) / pt, (s.prof_ms[1] - s0.prof_ms[1]) / pt,+ (s.prof_ms[2] - s0.prof_ms[2]) / pt, (s.prof_ms[3] - s0.prof_ms[3]) / pt,+ (s.prof_ms[4] - s0.prof_ms[4]) / pt, (s.prof_ms[5] - s0.prof_ms[5]) / pt);+ }+ if (df != nullptr) std::fclose(df);+ m.save_profile();+ return 0;+}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cunew file mode 100644index 000000000..eb86182de--- /dev/null+++ b/src/dsv41/ds4_model.cu@@ -0,0 +1,7786 @@+// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).+//+// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):+// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables+// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)+// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus+// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the+// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b+// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])+// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert+// h = post * moe + comb^T h; pre = f_pre+// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).+#include "ds4_model.hpp"+#include "ds4_engram_io.hpp"++#include "ds4_gguf.cuh"+#include "ds4_gguf_cpu.hpp"+#include "ds4_kernels.cuh"+#include "ds4_os.hpp"+#include "ds4_pool.hpp"+#include "ds4_vision.cuh"+#include "strata/kernels/cpu/exl3_cpu.hpp"+#include "strata/kernels/exl3.hpp"+#include "strata/kernels/f16_bits.hpp"+#include "strata/kernels/glm_fast.hpp"+#include "strata/kernels/sampler.hpp"++#include <cublas_v2.h>+#include <cuda_runtime.h>++#include <algorithm>+#include <array>+#include <condition_variable>+#include <deque>+#include <atomic>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <fstream>+#include <functional>+#include <future>+#include <map>+#include <mutex>+#include <numeric>+#include <sstream>+#include <thread>+#include <unordered_map>++#if defined(_MSC_VER)+#define strtok_r strtok_s+#endif++namespace strata::ds4 {+namespace x3 = strata::kernels::exl3;+namespace kc = strata::kernels::cpu;+namespace gf = strata::kernels::glmf;+namespace gg = strata::ds4::gguf;++#define CK(call) \+ do { \+ cudaError_t e_ = (call); \+ if (e_ != cudaSuccess) { \+ err = std::string(#call) + ": " + cudaGetErrorString(e_); \+ return false; \+ } \+ } while (0)++namespace {++double now_ms() {+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();+}++size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }++// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:+// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):+// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before+void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {+ const int b = whole / 128, q = b / np, r = b % np;+ auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };+ int at = 0;+ for (int i = 0; i < p; ++i) at += blocks(i);+ off = 128 * at;+ n = 128 * blocks(p);+}++std::string read_file(const std::string& p) {+ std::ifstream f(p);+ std::stringstream ss;+ ss << f.rdbuf();+ return ss.str();+}+std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"+ std::vector<int> v;+ std::stringstream ss(s);+ std::string part;+ while (std::getline(ss, part, ',')) {+ if (part.empty() || part == "\n") continue;+ const size_t dash = part.find('-');+ const int a = std::atoi(part.c_str());+ const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);+ for (int c = a; c <= b; ++c) v.push_back(c);+ }+ return v;+}+// each NUMA node's physical cores (the first logical CPU of each core), in order+std::vector<std::vector<int>> node_cores() {+#if defined(_WIN32)+ std::vector<std::vector<int>> nodes = os::node_cores();+#else+ std::vector<std::vector<int>> nodes;+ for (int n = 0; n < 64; ++n) {+ const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");+ if (cl.empty()) break;+ std::vector<int> cores;+ for (int c : cpu_list(cl)) {+ const std::vector<int> sib = cpu_list(+ read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));+ if (sib.empty() || sib[0] == c) cores.push_back(c);+ }+ if (!cores.empty()) nodes.push_back(cores);+ }+#endif+ if (nodes.empty()) {+ std::vector<int> all;+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);+ nodes.push_back(all);+ }+ return nodes;+}+// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages+void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }+void pin_self(int cpu) { os::pin_self(cpu); }+// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root+// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known+uint64_t ram_avail() {+#if defined(_WIN32)+ return os::ram_avail();+#endif+ uint64_t avail = ~uint64_t{0};+ {+ std::istringstream mi(read_file("/proc/meminfo"));+ std::string k;+ uint64_t kb = 0;+ while (mi >> k >> kb) {+ if (k == "MemAvailable:") avail = kb << 10;+ mi.ignore(64, '\n');+ }+ }+ std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."+ const size_t at = cg.find("0::");+ if (at == std::string::npos) return avail;+ cg = cg.substr(at + 3);+ cg = cg.substr(0, cg.find('\n'));+ for (;;) {+ const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);+ const std::string mx = read_file(dir + "/memory.max");+ if (!mx.empty() && mx.compare(0, 3, "max") != 0) {+ const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);+ const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);+ uint64_t file = 0;+ std::istringstream st(read_file(dir + "/memory.stat"));+ std::string k;+ uint64_t v = 0;+ while (st >> k >> v)+ if (k == "file") file = v;+ const uint64_t used = cur > file ? cur - file : 0;+ avail = std::min(avail, lim > used ? lim - used : 0);+ }+ if (cg.empty() || cg == "/") break;+ const size_t sl = cg.rfind('/');+ cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);+ }+ return avail;+}++float bf16_to_f32(uint16_t b) {+ uint32_t u = (uint32_t) b << 16;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}+float f16_to_f32(uint16_t h) {+ const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;+ uint32_t u;+ if (e == 0) {+ if (m == 0) u = s;+ else { // subnormal+ int ee = -1;+ uint32_t mm = m;+ do {+ ++ee;+ mm <<= 1;+ } while ((mm & 0x400) == 0);+ u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;+ }+ } else if (e == 31) u = s | 0x7f800000u | m << 13;+ else u = s | (e - 15 + 127) << 23 | m << 13;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}++// ------------------------------------------------------------------ the pack+struct Tensor {+ std::string dtype;+ int file = -1;+ uint64_t off = 0, bytes = 0;+ std::vector<int64_t> shape;+};++struct Pack {+ std::string dir, model_dir;+ std::unordered_map<std::string, std::string> kv;+ std::vector<std::string> files;+ std::vector<os::fd_t> fds;+ std::unordered_map<std::string, Tensor> t;+ int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;+ std::vector<int32_t> eg_map;+ std::vector<int64_t> eg_mult, eg_primes, eg_off;+ // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the+ // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)+ bool gguf = false;++ std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)+ std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)+ std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)++ ~Pack() {+ for (auto& m : maps) os::unmap(m.first, m.second);+ for (os::fd_t fd : fds) os::close_fd(fd);+ for (os::fd_t fd : dfds) os::close_fd(fd);+ for (os::fd_t fd : pfds) os::close_fd(fd);+ }+ // file i mapped whole, read-only (once; not thread-safe: called while loading)+ const uint8_t* map(int file) {+ if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});+ auto& m = maps[(size_t) file];+ if (m.first == nullptr) {+ const uint64_t size = os::file_size(fds[(size_t) file]);+ void* p = os::map_ro(fds[(size_t) file], size);+ if (p == nullptr) return nullptr;+ m = {(uint8_t*) p, (size_t) size};+ }+ return m.first;+ }++ bool load(const std::string& d, std::string& err) {+ dir = d;+ {+ std::ifstream f(d + "/ds4.txt");+ if (!f) {+ err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";+ return false;+ }+ std::string line;+ while (std::getline(f, line)) {+ const size_t sp = line.find(' ');+ if (sp == std::string::npos) continue;+ kv[line.substr(0, sp)] = line.substr(sp + 1);+ }+ }+ model_dir = str("model_dir");+ std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");+ if (f == nullptr) {+ err = d + "/tensors.txt: not found";+ return false;+ }+ std::vector<char> buf(1 << 16);+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {+ char* s = buf.data();+ if (s[0] == 'f') { // file <i> <name>+ char name[4096];+ int i = 0;+ if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {+ if ((int) files.size() <= i) files.resize((size_t) i + 1);+ files[(size_t) i] = name;+ }+ continue;+ }+ if (s[0] != 't') continue;+ char* save = nullptr;+ ::strtok_r(s, " \n", &save);+ const char* name = ::strtok_r(nullptr, " \n", &save);+ const char* dt = ::strtok_r(nullptr, " \n", &save);+ const char* fi = ::strtok_r(nullptr, " \n", &save);+ const char* off = ::strtok_r(nullptr, " \n", &save);+ const char* nb = ::strtok_r(nullptr, " \n", &save);+ if (name == nullptr || nb == nullptr) continue;+ Tensor x;+ x.dtype = dt;+ x.file = std::atoi(fi);+ x.off = std::strtoull(off, nullptr, 10);+ x.bytes = std::strtoull(nb, nullptr, 10);+ while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));+ t.emplace(name, std::move(x));+ }+ std::fclose(f);+ for (const std::string& fn : files) {+ const std::string p = model_dir + "/" + fn;+ const os::fd_t fd = os::open_ro(p);+ if (fd == os::kBadFd) {+ err = p + ": cannot open";+ return false;+ }+ fds.push_back(fd);+ }+ gguf = str("format") == "gguf";+ if (gguf)+ for (size_t i = 0; i < fds.size(); ++i) {+ if (map((int) i) == nullptr) {+ err = model_dir + "/" + files[i] + ": cannot map";+ return false;+ }+ dfds.push_back(os::open_direct(model_dir + "/" + files[i]));+ pfds.push_back(os::open_par(model_dir + "/" + files[i]));+ }+ // the Engram tables+ std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");+ if (e == nullptr) {+ err = d + "/engram.bin: not found";+ return false;+ }+ char magic[4];+ int32_t hdr[6];+ bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&+ std::fread(hdr, 4, 6, e) == 6;+ if (ok) {+ eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];+ const int cols = (eg_ngram - 1) * eg_heads;+ eg_map.resize((size_t) eg_vocab);+ eg_mult.resize((size_t) eg_layers * eg_ngram);+ eg_primes.resize((size_t) eg_layers * cols);+ eg_off.resize((size_t) eg_layers * cols);+ ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&+ std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&+ std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&+ std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();+ }+ std::fclose(e);+ if (!ok) {+ err = d + "/engram.bin: unreadable";+ return false;+ }+ return true;+ }++ std::string str(const std::string& k, const std::string& def = "") const {+ auto it = kv.find(k);+ return it == kv.end() ? def : it->second;+ }+ double num(const std::string& k, double def) const {+ auto it = kv.find(k);+ return it == kv.end() ? def : std::atof(it->second.c_str());+ }+ std::vector<int> list(const std::string& k) const {+ std::vector<int> v;+ std::stringstream ss(str(k));+ std::string x;+ while (std::getline(ss, x, ','))+ if (!x.empty()) v.push_back(std::atoi(x.c_str()));+ return v;+ }+ const Tensor* get(const std::string& n) const {+ auto it = t.find(n);+ return it == t.end() ? nullptr : &it->second;+ }+ bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)+ if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)+ return os::pread_par(pfds[(size_t) file], dst, n, off);+ return os::pread_full(fds[(size_t) file], dst, n, off);+ }+ bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }+ const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)+};++// ------------------------------------------------------------------ geometry+struct Geo {+ int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,+ k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,+ cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;+ float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;+ double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;+ std::vector<int> ratios, kv_src, ix_src, eg_layers;+ // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks+ int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;+ std::vector<int> d_targets;++ void from(const Pack& p) {+ vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);+ nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);+ hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);+ q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);+ o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);+ k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);+ win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);+ sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);+ ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);+ cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);+ cand_blk = (int) p.num("candidate_block_size", 0);+ orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);+ eos = (int) p.num("eos_token_id", 1);+ eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);+ route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);+ rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);+ factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);+ beta_slow = p.num("rope_scaling.beta_slow", 1);+ ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");+ ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");+ d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);+ d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);+ d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");+ // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)+ n_mtp = d_block > 0 && (int) ratios.size() > nl &&+ (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)+ ? (int) ratios.size() - nl : 0;+ }+};++// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them+std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {+ std::vector<float> f((size_t) dim / 2);+ for (int i = 0; i < dim / 2; ++i) {+ const float e = (float) (2 * i) / (float) dim;+ f[(size_t) i] = 1.0f / std::pow((float) base, e);+ }+ if (original > 0) {+ auto corr = [&](double rot) {+ return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)+ };+ const double low = std::max(std::floor(corr(beta_fast)), 0.0);+ const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);+ const float den = (float) std::max(high - low, 1e-3);+ for (int i = 0; i < dim / 2; ++i) {+ float ramp = ((float) i - (float) low) / den;+ ramp = std::min(1.0f, std::max(0.0f, ramp));+ const float smooth = 1.0f - ramp;+ f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;+ }+ }+ return f;+}++// ------------------------------------------------------------------ devices, layers+struct Dev {+ int id = 0;+ cudaStream_t st = nullptr;+ cudaEvent_t ev = nullptr;+ size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;+ std::vector<void*> allocs;+ // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)+ float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,+ *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,+ *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,+ *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,+ *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,+ *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,+ *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,+ *ed = nullptr;+ float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;+ unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)+ uint8_t* eg_rows = nullptr;+ uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows+ AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)+ // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the+ // driver's staging, ~1.7 ms a copy): h | pre | topk | cand+ uint8_t* bounce = nullptr;+ cudaEvent_t ev_out = nullptr;+ float* tap = nullptr; // a window's stream means before a DSpark target layer+ // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's+ // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window+ // MoE's grouping scratch+ int grp = 0, mi = 0;+ unsigned ar_seq = 0;+ unsigned* xcnt = nullptr;+ // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;+ // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)+ x3::Mat head;+ float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;+ unsigned head_seq = 0;+ // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the+ // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)+ uint16_t *mk_share = nullptr, *mk_erow = nullptr;+ int* mk_ints = nullptr;+ int mk_v0 = 0, mk_vn = 0;+ uint8_t* hand = nullptr;+ unsigned* hand_flag = nullptr;+ unsigned long long* grp_ptr = nullptr;+ int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;+};++// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of+// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS+struct Chunk {+ int C = 0, A = 0, AI = 0;+ float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,+ *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,+ *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,+ *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,+ *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;+ uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,+ *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;+ size_t w16_elems = 0;+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;+ uint8_t* eg_rows = nullptr;+ int* bounds_pin = nullptr; // pinned: the sort's bounds, read back+ // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,+ // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)+ static constexpr int kParts = 4;+ uint8_t* ring = nullptr;+ size_t ring_stride = 0;+ int ring_n = 0;+ cudaStream_t cs = nullptr;+ cudaEvent_t filled[kParts] = {}, freed[kParts] = {};+ // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of+ // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,+ // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s+ uint8_t* stage = nullptr;+ size_t stage_stride = 0;+ std::unique_ptr<Pool> readers;+ cublasHandle_t blas = nullptr;+ // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each+ // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)+ uint8_t* hand[2] = {nullptr, nullptr};+ unsigned* hand_flag[2] = {nullptr, nullptr};+ unsigned* hand_ack = nullptr;+ unsigned arb_seq = 0; // the group's chunk all-reduces+ // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)+ uint16_t* emb_host[2] = {nullptr, nullptr};+ uint8_t* eg_host[2] = {nullptr, nullptr};+ cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};+ double stream_bytes = 0; // the experts streamed in, in all+ // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a+ // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the+ // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once+ // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io+ // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits+ // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot+ // i % ring_n; the compute takes the items as they land, in runs of what has.+ struct Item {+ int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)+ int mi = 0; // the device's member index in the layer's group+ const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read+ const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)+ uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there+ int64_t after = -1; // (layer-major) the items up to this one read first+ std::atomic<int> left{0}; // parts still being read+ std::atomic<bool> bad{false};+ };+ bool flow = false;+ int dev = -1; // the device (index into devs) this chunk state belongs to+ std::unique_ptr<IoQueue> io;+ std::deque<Item> items; // items[0] is item `base`; the indices below are absolute+ size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;+ uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)+ int S = 0; // staging slots, stage_stride apart+ std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)+ std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use+ int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where+ size_t spec_begin = 0;+ std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes+ int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count+ int *bc_pin = nullptr, *bc_dev = nullptr;+ uint64_t n_batch = 0;+ double wait_ms = 0; // the compute waiting for a read+ uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;+ uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started+ // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through+ // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in+ // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for+ // its later chunks, so each is read once a run+ bool lm = false, lm_pinned = false;+ uint8_t* lm_state = nullptr;+ size_t lm_state_bytes = 0;+ uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side+ size_t lc_bytes = 0;+ int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all+ size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read+ std::vector<int> lc_of; // expert -> its slot (-1: none)+ std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it+ uint64_t it_lc = 0; // items taken from the layer cache+};++struct Layer {+ int il = 0, dev = 0, ratio = 0;+ // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the+ // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert+ // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)+ int grp = 0, nm = 1, mi = 0;+ std::vector<Layer> mates;+ x3::ExpertLayout hlay;+ size_t hblob = 0;+ int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)+ // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more+ // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their+ // attention adds nothing to the sum+ int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;+ // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's+ // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member+ int* vs_pin = nullptr;+ int* vs_dev = nullptr;+ std::vector<int> host_list;+ std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."+ int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)+ bool mtp = false; // a DSpark block (index stage = il - n_layers)+ bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;+ int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)+ int ik_src = -1; // the kv source whose index keys its indexer reads+ int eg = -1; // engram layer index, or -1+ x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;+ float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,+ *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,+ *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,+ *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)+ uint16_t* i_wproj = nullptr;+ float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)+ uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)+ float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)+ // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row+ uint8_t* snap_win = nullptr;+ float *snap_kv = nullptr, *snap_sc = nullptr;+ // routed experts+ x3::ExpertLayout lay;+ size_t blob = 0; // bytes of one expert's blob+ int n_vram = 0;+ uint8_t* vram_base = nullptr;+ std::vector<int> slot; // expert -> VRAM slot or -1+ std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)+ std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)+ // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),+ // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)+ struct Half {+ size_t trellis[3], suh[3], svh[3], bytes;+ int k[3], n[3];+ } hl;+ std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes+ // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it+ // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)+ int n_slots = 0, spare_vram = -1;+ std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};+ std::vector<float> heat;+ std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked+ bool swap_pending = false;+ unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes+ // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and+ // the window's raw rows (kv, then score) to replay the accepted ones+ uint8_t* ring_bak = nullptr;+ float *c_snap = nullptr, *c_rows = nullptr;+ unsigned long long* vram_tab = nullptr; // device [n_exp]+ std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob+ std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)+ std::vector<uint64_t> region_bytes;+ std::vector<int> region_file;+ size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region+ size_t piece_bytes[3][3];+ size_t dense_bytes = 0;+ // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] ++ // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane+ int mx_file[3] = {-1, -1, -1};+ uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};+};++struct Src {+ int layer = 0, ratio = 1;+ int64_t cap = 0;+ // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)+ std::vector<uint8_t*> ckv, ik;+ uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices+ cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;+ // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows+ // (ckv rows, then ik rows at kWin * hd) and its flag+ bool remote = false;+ uint8_t* x_stage = nullptr;+ unsigned* x_flag = nullptr;+ // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements+ // [slot][device] (32 apart); readers: the devices outside the source's group that read it+ uint8_t* x_stage1 = nullptr;+ unsigned* x_flag1 = nullptr;+ unsigned* x_ack = nullptr;+ std::vector<int> readers;+ uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }+ unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }+};++// a host thread that enqueues one device's work (threads mode)+struct Worker {+ std::thread th;+ std::mutex mu;+ std::condition_variable cv;+ std::function<void()> job;+ bool busy = false, quit = false;+ void start() {+ th = std::thread([this] {+ for (;;) {+ std::function<void()> j;+ {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return quit || job != nullptr; });+ if (quit) return;+ j = std::move(job);+ job = nullptr;+ }+ j();+ {+ std::lock_guard<std::mutex> lk(mu);+ busy = false;+ }+ cv.notify_all();+ }+ });+ }+ void submit(std::function<void()> f) {+ {+ std::lock_guard<std::mutex> lk(mu);+ job = std::move(f);+ busy = true;+ }+ cv.notify_all();+ }+ void wait() {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return !busy; });+ }+ bool idle() {+ std::lock_guard<std::mutex> lk(mu);+ return !busy;+ }+ void stop() {+ {+ std::lock_guard<std::mutex> lk(mu);+ quit = true;+ }+ cv.notify_all();+ if (th.joinable()) th.join();+ }+};++struct Group {+ std::vector<int> mem; // device indices, member order (member 0 the lead)+ ArGroup ar, arb; // a window's all-reduces, a chunk's+ // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the+ // lead's acknowledgements (host-mapped)+ float* lg_stage = nullptr;+ unsigned *lg_flag = nullptr, *lg_ack = nullptr;+};++constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers++// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the+// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate+// blocks as each layer sees them, the window's hand-off seq+struct WinPlan {+ bool par = false, async = false;+ unsigned win = 0;+ int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2+ int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)+ int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)+ unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)+ unsigned seq[kMaxL] = {};+ int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};+};++} // namespace++// ------------------------------------------------------------------ the model+struct Model::Impl {+ Options o;+ Pack pk;+ Geo g;+ std::vector<Dev> devs;+ std::vector<Layer> L;+ std::vector<Src> srcs;+ std::vector<Group> groups;+ std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device+ bool par = false; // threads mode (the windows: decode)+ bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked+ unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)+ // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows+ int chunk = 0, chunk_min = 512;+ static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)+ // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk+ // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers+ // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token+ static constexpr int kMaxChunk = 4 * kVlRows;+ std::vector<std::unique_ptr<Chunk>> chunks; // per device+ size_t eg_cap = kWin; // tokens h_eg holds+ int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)+ bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)+ std::vector<float> chunk_logits;+ std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)+ std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer+ std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded+ // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this+ // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took+ struct PathModel {+ bool on = false;+ double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)+ // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and+ // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)+ double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};+ int64_t kn[2][4] = {};+ double win_s = 0; // windows: seconds a window took, so far+ int64_t win_n = 0;+ int last_cls = 0, last_nw = 0; // the last pick's size class, its windows+ double m = 1; // the picks' spread, against the profile's+ int64_t n_m = 0;+ int last = -1; // the last pick: 1 the flow, 0 windows+ double last_pred = 0; // its prediction (s, before its k)+ } pm;+ static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)+ static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)+ std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through+ WinPlan wplan;+ int head_dev = 0;+ x3::Mat head;+ float* norm_w = nullptr; // on the head device+ // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the+ // confidence head (on the head device: the drafts' logits come from the trunk's head)+ x3::Mat d_main_proj;+ float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;+ uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]+ // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its+ // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole+ // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq+ bool mk_split = false;+ uint16_t* h_mk_embed = nullptr;+ unsigned* mk_slots = nullptr;+ unsigned mk_seq = 0;+ int d_dev = 0; // the drafter's first block's device+ bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)+ float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps+ cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it+ float* d_confv = nullptr; // on the head device+ int* d_ints = nullptr; // on the head device: drafts in, picks out+ float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;+ // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for+ // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()+ // checks it is its own)+ strata::kernels::SpecList* d_specq = nullptr;+ std::vector<strata::kernels::SpecList> h_specq = std::vector<strata::kernels::SpecList>(kWin);+ bool drafts_q = false;+ float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;+ int dq_top_k = 0;+ uint64_t dq_seed = 0, dq_counter = 0;+ // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the+ // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5+ // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the+ // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs+ // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)+ bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;+ float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;+ int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;+ std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;+ float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)+ uint16_t* h_dump_ring = nullptr;++ uint16_t* embed_pinned = nullptr;+ std::vector<float> freqs_win, freqs_comp;+ std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores+ std::vector<std::vector<int>> cores; // each node's physical cores+ int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)+ // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps+ // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)+ bool file_experts = false;+ uint64_t warm_bytes = 0;+ static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)+ uint8_t* file_part(const Layer& l, int e) const {+ return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];+ }+ // host side of the CPU lane, a window's rows+ float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;+ int* h_idx = nullptr;+ uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales+ // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one+ // an Engram layer: asked for when the window's tokens are known, collected right before that layer)+ EgReader eg_io;+ EgRowCache eg_cache;+ std::mutex eg_cache_mu;+ std::vector<std::unique_ptr<EgBatch>> eg_win;+ std::vector<uint8_t> eg_q6k; // Engram layer li's table in ggml Q6_K (a row one superblock a 256 values), else FP8+ std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};+ uint16_t* h_emb = nullptr; // the window's embedding rows+ std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;+ // the state+ int64_t pos = 0;+ std::vector<int32_t> toks, ctoks;+ struct Check { // the state at a position: every window ring, the compressors' partial groups+ int64_t pos = -1;+ std::vector<std::vector<uint8_t>> win;+ std::vector<std::vector<float>> comp;+ };+ Check ck[2]; // the end of the last prompt, and the position before it+ std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits+ Stats st;+ int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query+ // (plan_window's: the windows' plans are made in order)+ int n_logits = 0; // the last forward's logit rows+ // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread+ // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum+ Req* reqs = nullptr;+ Resp* resps = nullptr;+ unsigned seq_issued = 0;+ std::atomic<int> layer_of[kRing];+ std::thread service;+ std::atomic<bool> quit{false};+ bool sync_decode = std::getenv("DS4_SYNC") != nullptr;+ // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run+ // before the wait for the CPU lane - the same bits, no faster)+ bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;+ int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this+ // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same+ // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files+ // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next+ // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the+ // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's+ // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).+ int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;+ int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;+ // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)+ bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;+ std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;+ // 8 asked for (prefetch)+ std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)+ std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for+ std::thread pf_th;+ std::mutex pf_mu;+ std::condition_variable pf_cv;+ std::vector<std::pair<const void*, size_t>> pf_q;+ // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster+ // on one V100 - more for a PC to try: each its staging buffers and+ // copy streams), switched between windows+ struct Swap {+ int layer = 0, in = -1, out = -1;+ int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM+ };+ static constexpr int kMaxSwaps = 8;+ // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay+ // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)+ static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }+ const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),+ c_decay = env_f("DS4_CACHE_DECAY", 0.9f);+ const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));+ // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the+ // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the+ // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the+ // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in+ // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4+ // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.+ struct RamSwap {+ int layer = 0, in = -1, out = -1, slot = -1;+ };+ std::vector<uint8_t*> ram_slots, ram_chunks;+ std::vector<size_t> ram_chunk_bytes;+ bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)+ size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers+ std::vector<int> ram_free;+ std::deque<RamSwap> ram_queue;+ std::vector<RamSwap> ram_ready;+ std::condition_variable ram_cv;+ std::thread ram_th;+ int ram_inflight = 0, ram_held = 0;+ bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;+ // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a+ // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route+ // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every+ // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the+ // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left+ // out still counts as used (the expert cache, the profile).+ const float disk_skip_w = [] {+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");+ return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;+ }();+ const int disk_skip_from = [] {+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");+ return v != nullptr ? std::max(0, std::atoi(v)) : 0;+ }();+ bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;+ if (l.lay.fmt == gg::kFmtMXFP4) {+ for (int r = 0; r < 3; ++r)+ if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;+ return true;+ }+ return !file_experts || os::in_ram(file_part(l, e), l.blob);+ }+ // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads+ // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between+ // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of+ // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'+ // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes+ // the RAM place of the one it takes in)+ // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private+ // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever+ // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the+ // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds+ // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the+ // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold+ // them (the log's WARNING, a NOTE and INFO for the server)+ double commit_kept = 0, pagefile_hint = 0;+ double commit_margin() const {+ const os::Commit c = os::commit();+ return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);+ }+ // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)+ double commit_room() const {+ const os::Commit c = os::commit();+ double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;+ if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);+ return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());+ }+ // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)+ bool commit_charged(const Dev& d) const {+ if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;+#if defined(_WIN32)+ int v = 0;+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {+ (void) cudaGetLastError();+ return false;+ }+ return v != 0;+#else+ (void) d;+ return false;+#endif+ }+ bool copy_mode = false;+ std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts+ std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes+ std::vector<int> copy_spare; // layer -> its spare slot (-1: none)+ std::mutex sw_mu;+ std::condition_variable sw_cv;+ std::deque<Swap> sw_queue;+ std::vector<Swap> sw_ready;+ struct SwapBufs {+ uint8_t *a = nullptr, *b = nullptr, *h = nullptr;+ std::vector<cudaStream_t> st; // a copy stream a device+ };+ std::vector<SwapBufs> sw_bufs;+ std::vector<std::thread> cache_ths;+ int sw_inflight = 0;+ int64_t decay_tokens = 0;+ bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;+ std::vector<float> host_logits;+ std::vector<void*> host_allocs;+ std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;+ // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or+ // chunk path (tests: where the two part)+ std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;+ // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),+ // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the+ // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias+ // [n_exp] (F32)+ std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;+ // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,+ // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]+ // (F16) and y rows [rows dim] (F32)+ int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;+ std::FILE* dump_y = [] {+ const char* v = std::getenv("DS4_DUMP_Y");+ const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;+ return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;+ }();+ std::mutex dump_mu;+ // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its+ // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU+ // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it+ bool prof_on = std::getenv("DS4_PROF") != nullptr;+ struct ProfMark {+ const Dev* d;+ cudaEvent_t ev;+ int stage;+ };+ std::mutex prof_mu;+ std::vector<ProfMark> prof_marks;+ std::vector<cudaEvent_t> prof_pool;+ bool prof_window = false; // (decode windows only: T == 1)+ // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared+ // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run+ bool prof_chunk = false;+ double cprof_ms[4] = {};+ void prof_mark(const Dev& d, int stage) {+ if (!prof_window && !prof_chunk) return;+ std::lock_guard<std::mutex> lk(prof_mu);+ cudaEvent_t e = nullptr;+ if (prof_pool.empty()) cudaEventCreate(&e);+ else {+ e = prof_pool.back();+ prof_pool.pop_back();+ }+ cudaEventRecord(e, d.st);+ prof_marks.push_back({&d, e, stage});+ }+ void prof_collect() {+ if (!prof_window && !prof_chunk) return;+ std::lock_guard<std::mutex> lk(prof_mu);+ std::vector<std::pair<const Dev*, cudaEvent_t>> last;+ for (const ProfMark& m : prof_marks) {+ auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });+ if (it != last.end()) {+ float ms = 0.0f;+ if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)+ (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;+ it->second = m.ev;+ } else {+ last.push_back({m.d, m.ev});+ }+ }+ for (const ProfMark& m : prof_marks) prof_pool.push_back(m.ev);+ prof_marks.clear();+ if (!prof_chunk) ++st.prof_tokens;+ }+ void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {+ std::vector<float> v((size_t) T * g.hc * g.dim);+ cudaSetDevice(d.id);+ cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[3] = {il, p0, T};+ std::fwrite(hdr, 8, 3, dump_l);+ std::fwrite(v.data(), 4, v.size(), dump_l);+ std::fflush(dump_l);+ }++ ~Impl() {+ eg_io.stop(); // (its threads write into h_eg)+ for (auto& w : workers) w->stop();+ for (size_t i = 0; i < chunks.size(); ++i)+ if (chunks[i] && chunks[i]->blas) {+ cudaSetDevice(devs[i].id);+ cublasDestroy(chunks[i]->blas);+ }+ quit.store(true);+ if (service.joinable()) service.join();+ {+ std::lock_guard<std::mutex> lk(pf_mu);+ }+ pf_cv.notify_all();+ if (pf_th.joinable()) pf_th.join();+ {+ std::lock_guard<std::mutex> lk(sw_mu);+ }+ sw_cv.notify_all();+ ram_cv.notify_all();+ for (std::thread& t : cache_ths)+ if (t.joinable()) t.join();+ if (ram_th.joinable()) ram_th.join();+ for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ for (void* p : d.allocs) cudaFree(p);+ if (d.st) cudaStreamDestroy(d.st);+ if (d.ev) cudaEventDestroy(d.ev);+ }+ for (void* p : host_allocs) cudaFreeHost(p);+ for (Layer& l : L)+ for (uint8_t* b : l.host)+ if (b != nullptr) {+ (void) b; // host blobs live in one mapping per layer, unmapped below+ }+ for (auto& m : host_maps) {+ if (chunk > 0) cudaHostUnregister(m.first);+ os::anon_free(m.first, m.second);+ }+ }+ std::vector<std::pair<void*, size_t>> host_maps;++ template <class T>+ bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {+ void* v = nullptr;+ cudaSetDevice(d.id);+ const size_t b = align256(n * sizeof(T));+ if (cudaMalloc(&v, b) != cudaSuccess) {+ err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st+ // does not order against - a zeroing still queued there could wipe the weights just copied+ if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {+ cudaFree(v);+ err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ d.allocs.push_back(v);+ if (acct) *acct += b;+ else d.scratch += b;+ p = (T*) v;+ return true;+ }+ static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }+ static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }+ // bytes of a VRAM slot of l's experts (a member's part of each, in a group)+ static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }+ // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64+ // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share+ int head_members(int nm) const {+ int na = nm;+ while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;+ return na;+ }+ // host-mapped memory (the same address on the host and every device), zeroed+ template <class T>+ bool hmap(T*& p, size_t n, std::string& err) {+ void* v = nullptr;+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";+ return false;+ }+ std::memset(v, 0, n * sizeof(T) + 64);+ host_allocs.push_back(v);+ p = (T*) v;+ return true;+ }+ template <class T>+ bool halloc(T*& p, size_t n, std::string& err) {+ void* v = nullptr;+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {+ err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";+ return false;+ }+ host_allocs.push_back(v);+ p = (T*) v;+ return true;+ }++ // ---- weights+ bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {+ std::vector<uint8_t> tmp(t.bytes);+ if (!pk.read(t, tmp.data())) {+ err = "read failed";+ return false;+ }+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));+ return true;+ }+ // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its+ // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a+ // standalone [n][k] matrix+ bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,+ int np, bool cols) {+ const bool q8 = t.dtype == "Q8_0";+ if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {+ err = pre + ": a " + t.dtype + " tensor that is not a matrix";+ return false;+ }+ int n = (int) t.shape[0], k = (int) t.shape[1];+ // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)+ auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };+ std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));+ if (t.bytes != w.size() || !pk.read(t, w.data())) {+ err = pre + ": read failed";+ return false;+ }+ if (np > 1) {+ const int whole = cols ? n : k, wd = whole / np;+ if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " ++ std::to_string(np) + " parts";+ return false;+ }+ const size_t rb = bytes_of((size_t) k);+ std::vector<uint8_t> p;+ if (cols) {+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));+ n = wd;+ } else {+ const size_t pb = bytes_of((size_t) wd);+ p.resize((size_t) n * pb);+ for (size_t r = 0; r < (size_t) n; ++r)+ std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);+ k = wd;+ }+ w.swap(p);+ }+ uint8_t* dp = nullptr;+ if (!dalloc(d, dp, w.size(), err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));+ m = x3::Mat{};+ m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;+ m.raw = dp;+ m.k = k;+ m.n = n;+ m.ld = q8 ? (int) bytes_of((size_t) k) : k;+ return true;+ }+ // a Q8_0 tensor's values as F32 (host)+ static std::vector<float> q8_values(const uint8_t* q, size_t n) {+ std::vector<float> v(n);+ for (size_t b = 0; b < n / 32; ++b) {+ const uint8_t* p = q + b * gg::kQ8Bytes;+ const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));+ for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];+ }+ return v;+ }+ // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone+ // matrix of its own (the rotations' 128-blocks stay whole)+ // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix+ // splits evenly, its groups only as wide as that goes)+ bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,+ bool cols = true, int rot = 0) {+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0"))+ return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);+ const Tensor* tr = pk.get(pre + ".trellis");+ const Tensor* su = pk.get(pre + ".suh");+ const Tensor* sv = pk.get(pre + ".svh");+ if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {+ err = pre + ": not an EXL3 matrix in the pack";+ return false;+ }+ if (pk.get(pre + ".mul1") == nullptr) {+ err = pre + ": only the mul1 codebook is covered";+ return false;+ }+ m.k = (int) tr->shape[0] * 16;+ m.n = (int) tr->shape[1] * 16;+ m.K = (int) tr->shape[2] / 16;+ m.cb = 2;+ m.ld = 0;+ if (np > 1) {+ const int whole = cols ? m.n : m.k;+ int at = 0, w = 0;+ if (whole % 128 != 0 || whole / 128 < np) {+ err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " ++ std::to_string(np) + " parts of 128-blocks";+ return false;+ }+ split128(whole, np, part, at, w, rot);+ std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);+ if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {+ err = pre + ": read failed";+ return false;+ }+ const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));+ std::vector<uint8_t> t1, u1, v1;+ if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)+ const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;+ t1.resize((size_t) (m.k / 16) * nth * tile);+ for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)+ std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);+ u1 = u0;+ v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));+ m.n = w;+ } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)+ const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;+ t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));+ u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));+ v1 = v0;+ m.k = w;+ }+ uint32_t* dt = nullptr;+ uint16_t *ds = nullptr, *dv = nullptr;+ if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||+ !dalloc(d, dv, v1.size() / 2, err, acct))+ return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));+ CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));+ CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));+ m.trellis = dt;+ m.suh = ds;+ m.svh = dv;+ return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);+ }+ uint32_t* dt = nullptr;+ uint16_t *ds = nullptr, *dv = nullptr;+ if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||+ !dalloc(d, dv, sv->bytes / 2, err, acct))+ return false;+ if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;+ m.trellis = dt;+ m.suh = ds;+ m.svh = dv;+ if (!x3::supported(m)) {+ err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) ++ " not covered by the EXL3 kernels";+ return false;+ }+ return true;+ }+ // a float tensor (F32 / F16 / BF16) as F32 on the device+ bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {+ const Tensor* t = pk.get(name);+ if (t == nullptr) {+ err = name + ": not in the pack";+ return false;+ }+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ size_t n = 0;+ std::vector<float> v;+ if (t->dtype == "F32") {+ n = t->bytes / 4;+ v.resize(n);+ std::memcpy(v.data(), raw.data(), t->bytes);+ } else if (t->dtype == "F16" || t->dtype == "BF16") {+ n = t->bytes / 2;+ v.resize(n);+ const uint16_t* s = (const uint16_t*) raw.data();+ for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);+ } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)+ n = t->bytes / gg::kQ8Bytes * 32;+ v = q8_values(raw.data(), n);+ } else {+ err = name + ": dtype " + t->dtype + " is not a float";+ return false;+ }+ if (want != 0 && n != want) {+ err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);+ return false;+ }+ if (!dalloc(d, p, n, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));+ return true;+ }+ bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {+ const Tensor* t = pk.get(name);+ if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)+ const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();+ std::vector<uint16_t> h(n);+ for (size_t i = 0; i < n; ++i) {+ float v;+ if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);+ else if (t->dtype == "Q8_0") v = q[i];+ else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);+ h[i] = strata::kernels::f16_from_f32(v);+ }+ if (!dalloc(d, p, n, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));+ return true;+ }+ if (t == nullptr || t->dtype != "F16") {+ err = name + ": not an F16 tensor in the pack";+ return false;+ }+ if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;+ return upload(d, *t, p, err);+ }++ size_t mat_bytes(const std::string& pre) const {+ if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);+ size_t b = 0;+ for (const char* s : {".trellis", ".suh", ".svh"})+ if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);+ return b;+ }+ size_t tensor_bytes(const std::string& name) const {+ const Tensor* t = pk.get(name);+ return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;+ }++ // ---- expert regions and blob layout+ // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every+ // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)+ // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)+ static int gg_type_of(const std::string& dtype) {+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23})+ if (dtype == gg::type_name(t)) return t;+ return -1;+ }+ bool expert_layout_mx(Layer& l, std::string& err) {+ const std::string P = l.name + "ffn.experts.";+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;+ const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");+ const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;+ const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;+ const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;+ if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {+ err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) ++ "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";+ return false;+ }+ l.mx_file[r] = t0->file;+ l.mx_off[r] = t0->off;+ l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;+ for (int e = 0; e < l.n_exp; ++e) {+ const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");+ if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||+ t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {+ err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";+ return false;+ }+ }+ l.lay.trellis[r] = off;+ l.lay.suh[r] = l.lay.svh[r] = off;+ l.lay.K[r] = ty;+ off += align256(bytes);+ }+ l.lay.fmt = gg::kFmtMXFP4;+ if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||+ gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||+ gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {+ err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) ++ " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";+ return false;+ }+ l.blob = off;+ return true;+ }+ const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)+ return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]+ : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];+ return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);+ }+ size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)+ return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);+ }+ // (copy_mode) matrix r's rows, and the rows of one NUMA part of it+ int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }+ size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }+ // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them+ const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {+ if (copy_mode && l.ram[(size_t) e] >= 0)+ return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];+ return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ }+ // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)+ void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);+ }+ }+ void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);+ }+ }+ // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down+ // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back+ bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {+ if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);+ else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);+ return true;+ }+ return read_mx_file(l, e, blob);+ }+ bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)+ for (int r = 0; r < 3; ++r)+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),+ blob + l.lay.trellis[r]))+ return false;+ return true;+ }+ void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);+ for (size_t row = 0; row < (size_t) g.dim; ++row)+ std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);+ }+ void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);+ for (size_t row = 0; row < (size_t) g.dim; ++row)+ std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);+ }++ bool expert_layout(Layer& l, std::string& err) {+ if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)+ return expert_layout_mx(l, err);+ const std::string P = l.name + "ffn.experts.";+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down+ const char* pieces[3] = {".trellis", ".suh", ".svh"};+ l.region_off.resize((size_t) l.n_exp);+ l.region_bytes.resize((size_t) l.n_exp);+ l.region_file.resize((size_t) l.n_exp);+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");+ if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {+ err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";+ return false;+ }+ l.lay.K[r] = (int) tr->shape[2] / 16;+ l.lay.trellis[r] = off;+ off += align256(tr->bytes);+ l.lay.suh[r] = off;+ off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);+ l.lay.svh[r] = off;+ off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);+ }+ l.lay.cb = 2;+ l.blob = off;+ for (int e = 0; e < l.n_exp; ++e) {+ const std::string E = P + std::to_string(e) + ".";+ uint64_t lo = UINT64_MAX, hi = 0;+ int file = -1;+ for (int r = 0; r < 3; ++r)+ for (int q = 0; q < 3; ++q) {+ const Tensor* t = pk.get(E + parts[r] + pieces[q]);+ if (t == nullptr) {+ err = E + parts[r] + pieces[q] + ": missing";+ return false;+ }+ if (file < 0) file = t->file;+ if (t->file != file) {+ err = E + ": split across files";+ return false;+ }+ lo = std::min(lo, t->off);+ hi = std::max(hi, t->off + t->bytes);+ if (e == 0) l.piece_bytes[r][q] = t->bytes;+ else if (t->bytes != l.piece_bytes[r][q]) {+ err = E + parts[r] + ": not the shape of expert 0's";+ return false;+ }+ }+ l.region_off[(size_t) e] = lo;+ l.region_bytes[(size_t) e] = hi - lo;+ l.region_file[(size_t) e] = file;+ for (int r = 0; r < 3; ++r)+ for (int q = 0; q < 3; ++q) {+ const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;+ if (e == 0) l.piece_src[r][q] = src;+ else if (src != l.piece_src[r][q]) {+ err = E + ": the pieces lie differently from expert 0's";+ return false;+ }+ }+ }+ return true;+ }+ // the region of expert e (read from the file) -> its blob+ void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};+ for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);+ }+ }++ // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)+ void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ const uint8_t* tr = region + l.piece_src[r][0];+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ (size_t) nth * tile);+ std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);+ std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);+ }+ }++ // ---- loading+ bool load(const Options& opt, std::string& err) {+ o = opt;+ const double t0 = now_ms();+ if (!pk.load(o.pack, err)) return false;+ g.from(pk);+ if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {+ err = o.pack + "/ds4.txt: incomplete geometry";+ return false;+ }+ if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {+ err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " ++ std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";+ return false;+ }+ freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);+ freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);++ // devices+ int ndev = 0;+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {+ err = "no CUDA device";+ return false;+ }+ std::vector<int> ids = o.gpus;+ if (ids.empty())+ for (int i = 0; i < ndev; ++i) ids.push_back(i);+ devs.resize(ids.size());+ for (size_t i = 0; i < ids.size(); ++i) {+ Dev& d = devs[i];+ d.id = ids[i];+ CK(cudaSetDevice(d.id));+ CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));+ CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));+ if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {+ err = "exl3 scratch on CUDA" + std::to_string(d.id);+ return false;+ }+ CK(cudaMemGetInfo(&d.free0, &d.total));+ }+ // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each+ // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)+ int tp = o.tp;+ if (pk.gguf && tp > 1) {+ int blk = 32;+ for (const auto& kv : pk.t) {+ const std::string& n = kv.first;+ if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||+ n.compare(n.size() - 9, 9, "w2.weight") != 0)+ continue;+ const int ty = gg_type_of(kv.second.dtype);+ if (ty >= 0) blk = std::max(blk, gg::type_block(ty));+ }+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % 128 != 0 || g.heads % tp != 0 ||+ g.o_groups % tp != 0))+ --tp;+ if (tp != o.tp)+ std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "+ "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);+ }+ // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone+ // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the+ // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the+ // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down+ // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16+ par = o.threads || tp > 1;+ prompt_par = par || o.tp > 1;+ const bool tp_all = tp > 1 && tp == (int) devs.size();+ for (size_t i = 0; i < devs.size();) {+ Group G;+ G.mem.push_back((int) i);+ size_t j = i + 1;+ while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))+ G.mem.push_back((int) j++);+ if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own+ for (int m : G.mem) {+ Group one;+ one.mem.push_back(m);+ groups.push_back(one);+ }+ } else groups.push_back(G);+ i = j;+ }+ for (size_t gi = 0; gi < groups.size(); ++gi)+ for (size_t m = 0; m < groups[gi].mem.size(); ++m) {+ devs[(size_t) groups[gi].mem[m]].grp = (int) gi;+ devs[(size_t) groups[gi].mem[m]].mi = (int) m;+ }+ // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the+ // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM+ // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place+ // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times+ if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM+ // after VRAM's are in, by --mmap-experts)+ else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;+ else {+ double exp_b = 0, other_b = 0, room = 0;+ for (const auto& kt : pk.t) {+ const std::string& n = kt.first;+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||+ (!o.dspark && n.rfind("mtp.", 0) == 0))+ continue;+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;+ }+ for (const Dev& d : devs)+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -+ (prompt_par ? 1.5 : 0.4) * (1 << 30)) *+ o.vram_experts;+ file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();+ }+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;+ const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)+ if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;+ if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));+ if (chunk > 0) {+ chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));+ eg_cap = (size_t) chunk;+ tap_cap = std::max(kWin, 128);+ } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)+ std::fprintf(stderr,+ "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"+ " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"+ " 461 / 1010 in chunks of %d) - %s\n",+ kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,+ o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"+ : "leave --tp 1 out, or pass --threads on, for chunks");+ // the vision tower on the first GPU, before the split: the layers get what it leaves+ if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,+ if (o.vision_only) return false; // and the server is told pictures are off - info_lines)+ std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "+ "text only\n", err.c_str());+ vision_err = err.empty() ? "unknown reason" : err;+ err.clear();+ vt.release();+ img_tok = -1;+ }+ if (o.vision_only) return true;++ // layers: roles and the sources they read+ if (!o.dspark) g.n_mtp = 0;+ L.resize((size_t) (g.nl + g.n_mtp));+ int last_src = -1;+ for (int il = 0; il < g.nl + g.n_mtp; ++il) {+ Layer& l = L[(size_t) il];+ l.il = il;+ l.mtp = il >= g.nl;+ l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";+ l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;+ l.k_exp = l.mtp ? g.d_k : g.k_exp;+ l.ratio = g.ratios[(size_t) il];+ if (l.mtp) { // window attention only: no compressor, indexer or Engram+ if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {+ err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " ++ std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";+ return false;+ }+ if (!expert_layout(l, err)) return false;+ continue;+ }+ l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();+ l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();+ l.owns_k = l.kv_src;+ l.cand_src = il == g.cand_src;+ l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;+ if (l.kv_src) {+ Src s;+ s.layer = il;+ s.ratio = l.ratio;+ s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;+ srcs.push_back(s);+ last_src = (int) srcs.size() - 1;+ }+ if (l.ratio > 0) {+ if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {+ err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) ++ " without a KV source of that ratio before it";+ return false;+ }+ l.src = last_src;+ l.ik_src = last_src;+ }+ auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);+ if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());+ if (!expert_layout(l, err)) return false;+ }++ // the split: each layer's dense weights + its experts, the head on the last GPU+ for (Layer& l : L) {+ const std::string& P = l.name;+ size_t b = 0;+ if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");+ for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",+ "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",+ "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})+ b += mat_bytes(P + m);+ for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));+ b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");+ b += (size_t) g.win * win_rb() + (2u << 20);+ l.dense_bytes = b;+ }+ const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);+ const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;+ const size_t kv_per_src_dev = 0; // counted after the split+ (void) kv_per_src_dev;+ std::vector<double> avail(groups.size(), 0.0);+ double avail_sum = 0;+ for (size_t gi = 0; gi < groups.size(); ++gi) {+ for (int m : groups[gi].mem)+ avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));+ avail_sum += avail[gi];+ }+ double need_sum = (double) head_b;+ for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;+ {+ // contiguous layer ranges, each group's share of the total by its free memory+ size_t d = 0;+ double cum = 0, target = avail[0] / avail_sum * need_sum;+ std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers+ for (Layer& l : L)+ if (l.mtp) order.push_back(&l);+ for (Layer& l : L)+ if (!l.mtp) order.push_back(&l);+ for (Layer* lp : order) {+ Layer& l = *lp;+ const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;+ if (cum + lb / 2 > target && d + 1 < groups.size()) {+ ++d;+ target += avail[d] / avail_sum * need_sum;+ }+ l.grp = (int) d;+ cum += lb;+ }+ // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its+ // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage+ // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all+ if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {+ const int G = (int) groups.size(), NL = g.nl;+ std::vector<double> cap(groups.size());+ for (int gi = 0; gi < G; ++gi) {+ const int nm = (int) groups[(size_t) gi].mem.size();+ double mn = 1e300;+ for (int m : groups[(size_t) gi].mem)+ mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -+ 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)+ cap[(size_t) gi] = mn * nm;+ }+ for (const Layer& l : L)+ if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;+ cap[(size_t) G - 1] -= (double) head_b;+ std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense+ for (int il = 0; il < NL; ++il) {+ pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;+ pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;+ }+ auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi+ const int nm = (int) groups[(size_t) gi].mem.size();+ const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);+ return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;+ };+ // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)+ std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));+ std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));+ for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);+ for (int gi = 1; gi < G; ++gi)+ for (int b = gi + 1; b <= NL; ++b)+ for (int a = gi; a < b; ++a) {+ const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));+ if (v < best[(size_t) gi][(size_t) b]) {+ best[(size_t) gi][(size_t) b] = v;+ cut[(size_t) gi][(size_t) b] = a;+ }+ }+ int b = NL;+ for (int gi = G - 1; gi >= 0; --gi) {+ const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;+ for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;+ b = a;+ }+ }+ // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)+ if (const char* sp = std::getenv("DS4_SPLIT")) {+ std::vector<int> n;+ for (const char* q = sp; *q;) {+ n.push_back(std::atoi(q));+ while (*q && *q != ',') ++q;+ if (*q == ',') ++q;+ }+ int il = 0;+ for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)+ for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;+ for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;+ }+ for (Layer& l : L) {+ l.dev = groups[(size_t) l.grp].mem[0];+ l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)+ }+ head_dev = L[(size_t) g.nl - 1].dev;+ d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;+ }+ // the groups' members: copies of each layer's device state, a member's part of an expert+ for (Layer& l : L) {+ l.ff0 = 0;+ l.ffn = g.ff;+ l.ah0 = l.og0 = 0;+ l.ahn = g.heads;+ l.ogn = g.o_groups;+ if (l.nm == 1) continue;+ const int na = head_members(l.nm);+ if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {+ err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";+ return false;+ }+ for (int m = 1; m < l.nm; ++m) {+ Layer c = l;+ c.mates.clear();+ c.mi = m;+ c.dev = groups[(size_t) l.grp].mem[(size_t) m];+ l.mates.push_back(std::move(c));+ }+ // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even+ // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its+ // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)+ const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ const int ha = m - (l.nm - na); // (its place among the members with heads)+ x.ahn = ha < 0 ? 0 : g.heads / na;+ x.ah0 = ha < 0 ? 0 : ha * x.ahn;+ x.ogn = ha < 0 ? 0 : g.o_groups / na;+ x.og0 = ha < 0 ? 0 : ha * x.ogn;+ if (gguf_x) {+ x.ffn = g.ff / l.nm;+ x.ff0 = m * x.ffn;+ } else+ split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;+ x.hlay.K[r] = l.lay.K[r];+ x.hlay.trellis[r] = off;+ if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks+ const int ty = gg::expert_type(l.lay, r);+ if (k % gg::type_block(ty) != 0) {+ err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) ++ " blocks do not split";+ return false;+ }+ x.hlay.suh[r] = x.hlay.svh[r] = off;+ off += align256((size_t) n * gg::type_row_bytes(ty, k));+ continue;+ }+ off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));+ x.hlay.suh[r] = off;+ off += align256((size_t) k * 2);+ x.hlay.svh[r] = off;+ off += align256((size_t) n * 2);+ }+ x.hlay.cb = l.lay.cb;+ x.hlay.fmt = l.lay.fmt;+ x.hblob = off;+ }+ }+ // which devices hold each source's compressed KV / index keys+ for (Src& s : srcs) {+ s.ckv.assign(devs.size(), nullptr);+ s.ik.assign(devs.size(), nullptr);+ }+ for (const Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m) {+ const Layer& l = member(l0, m);+ if (l.src >= 0) {+ srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;+ if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;+ }+ if (l.ix_src && l.ik_src >= 0) {+ srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;+ if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;+ }+ }++ // per device: the scratch, the caches, the dense weights+ for (size_t di = 0; di < devs.size(); ++di) {+ Dev& d = devs[di];+ const int D = g.dim, H = g.hc;+ const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;+ bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&+ dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&+ dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&+ dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&+ dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&+ dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&+ dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&+ dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&+ dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&+ dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&+ dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&+ dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&+ dalloc(d, d.rw, W * kPicks, err) &&+ dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.moe_out, W * D, err) &&+ dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&+ dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&+ dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&+ dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&+ dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&+ dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&+ dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&+ dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);+ if (!ok) return false;+ d.attn_scr.max_t = kAttnScratchT;+ d.attn_scr.cap = (int) topk_stride();+ if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;+ if (!g.eg_layers.empty()) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;+ ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&+ dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);+ if (!ok) return false;+ }+ if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;+ if (!halloc(d.bounce, bounce_bytes(), err)) return false;+ if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||+ !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||+ !dalloc(d, d.n_groups, 4, err))+ return false;+ if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;+ if (g.n_mtp > 0 && (int) di == head_dev &&+ (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))+ return false;+ CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));+ cudaSetDevice(d.id);+ CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));+ CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));+ for (Src& s : srcs) {+ if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)+ if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))+ return false;+ CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));+ CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));+ CK(cudaSetDevice(d.id));+ }+ if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))+ return false;+ if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))+ return false;+ }+ }+ for (Src& sr : srcs) {+ if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))+ return false;+ if (sr.remote && chunk > 0 &&+ (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||+ !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))+ return false;+ const int sg = L[(size_t) sr.layer].grp;+ for (size_t di = 0; di < devs.size(); ++di)+ if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);+ }+ for (Group& G : groups) {+ G.ar.nm = (int) G.mem.size();+ G.ar.slots = 4;+ G.ar.stride = (size_t) kWin * g.dim;+ if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||+ !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))+ return false;+ // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)+ if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||+ !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))+ return false;+ if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&+ (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||+ !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))+ return false;+ G.arb = G.ar; // (a chunk's: two slots of chunk rows)+ G.arb.slots = 2;+ G.arb.stride = (size_t) chunk * g.dim;+ if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||+ !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))+ return false;+ if (chunk > 0 && G.arb.nm > 2 &&+ (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||+ !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))+ return false;+ }+ if (g.n_mtp > 0) {+ if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;+ CK(cudaSetDevice(devs[(size_t) d_dev].id));+ CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));+ CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));+ }+ // the dense weights, each device's layers by a thread of its own+ {+ std::vector<std::string> errs(devs.size());+ std::vector<std::thread> th;+ for (size_t di = 0; di < devs.size(); ++di)+ th.emplace_back([this, di, &errs] {+ std::string e;+ if (!load_dense((int) di, e)) errs[di] = e;+ });+ for (auto& t : th) t.join();+ for (auto& e : errs)+ if (!e.empty()) {+ err = e;+ return false;+ }+ }+ if (o.verbose)+ std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);+ // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E+ // host-mapped, the steps' slots+ // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)+ bool mk_rows = g.n_mtp > 0 && head_nm() > 1;+ for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;+ if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {+ const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");+ if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {+ err = "the markov embed: not a BF16 [vocab][rank] table";+ return false;+ }+ if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||+ !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {+ if (err.empty()) err = "the markov embed: read failed";+ return false;+ }+ devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)+ mk_split = true;+ }++ // the embedding table, pinned on the host+ {+ const Tensor* t = pk.get("embed.weight");+ const size_t nv = (size_t) g.vocab * g.dim;+ const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;+ if (t == nullptr || (!q8 && (t->dtype != "BF16" || t->bytes != nv * 2))) {+ err = "embed.weight: not a BF16 (or Q8_0) [vocab][dim] table";+ return false;+ }+ if (!halloc(embed_pinned, nv, err)) return false;+ if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)+ std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {+ if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {+ err = "embed.weight: read failed";+ return false;+ }+ const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);+ for (size_t c = 0; c < (size_t) g.dim; ++c) {+ uint32_t u;+ std::memcpy(&u, &v[c], 4);+ u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even+ embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);+ }+ }+ } else if (!pk.read(*t, embed_pinned)) {+ err = "embed.weight: read failed";+ return false;+ }+ }+ // the host side of the CPU lane and the Engram rows+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;+ if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||+ !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) ||+ !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||+ !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))+ return false;+ if (pk.eg_layers > 0) {+ // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);+ // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)+ const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;+ const unsigned hw = std::max(1u, std::thread::hardware_concurrency());+ int nth = (int) std::min(16u, std::max(4u, hw / 2));+ if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));+ const uint64_t avail = ram_avail();+ uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));+ if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);+ eg_cache.init((size_t) ((mb << 20) / row), row);+ std::vector<std::string> paths;+ for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);+ eg_io.start(paths, nth);+ for (int li = 0; li < pk.eg_layers; ++li) {+ eg_win.emplace_back(new EgBatch());+ const Tensor* tw = pk.get("layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.weight");+ const bool q6k = tw != nullptr && tw->dtype == "Q6_K";+ if (q6k && pk.eg_hd % 256 != 0) {+ err = "Engram layer " + std::to_string(g.eg_layers[(size_t) li]) + ": Q6_K rows of " ++ std::to_string(pk.eg_hd) + " values (not whole superblocks of 256)";+ return false;+ }+ eg_q6k.push_back(q6k ? 1 : 0);+ }+ if (o.verbose)+ std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "+ "their layer; row cache %llu MB (%zu rows)\n",+ nth, (unsigned long long) mb, eg_cache.capacity());+ }+ prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);+ if (!o.profile.empty()) {+ std::FILE* f = std::fopen(o.profile.c_str(), "rb");+ if (f != nullptr) {+ std::vector<uint32_t> p(prof.size());+ const size_t n = std::fread(p.data(), 4, p.size(), f);+ if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());+ std::fclose(f);+ }+ }++ // the chunked prompt path's buffers (before the experts: they take what is left)+ if (chunk > 0 && !alloc_chunks(err)) return false;+ // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM+ // now leaves over+ if (file_experts) {+ double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ size_t fr = 0, tot = 0;+ CK(cudaMemGetInfo(&fr, &tot));+ room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;+ }+ for (const Layer& l : L) all += (double) l.blob * l.n_exp;+ const double need = std::max(0.0, all - room);+ const uint64_t avail = ram_avail();+ warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;+ if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;+ std::fprintf(stderr,+ "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "+ "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",+ need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"+ : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",+ avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);+ }+ // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in+ // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)+ cores = node_cores();+ NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());+ if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;+ if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)+ {+ const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;+ c_xh.resize((size_t) NH * S * 2 * E);+ c_xsb.resize((size_t) NH * S * 2 * (E / 128));+ c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));+ c_xhd.resize(S * FF);+ c_xsd.resize(S * (FF / 128));+ c_accd.resize((size_t) NH * S * DQ * (E / NH));+ c_yd.resize(S * E);+ }+ // the experts: VRAM slots in what each device has left, the rest on the host+ if (!place_experts(err)) return false;+ if (!load_experts(err)) return false;+ if (pk.gguf) make_pools();+ if (chunk > 0) {+ // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots+ const double tr = now_ms();+ for (auto& m : host_maps)+ if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {+ err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";+ return false;+ }+ if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);+ if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)+ size_t mr = 0;+ for (const Layer& l : L)+ if (!l.mtp) {+ for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);+ if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)+ mr = std::max(mr, l.blob);+ for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));+ }+ }+ const char* nr = std::getenv("DS4_READERS");+ const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);+ const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;+ // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the+ // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the+ // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)+ const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;+ for (size_t di = 0; di < chunks.size(); ++di) {+ auto& cp = chunks[di];+ if (cp == nullptr || cp->ring == nullptr) continue;+ cp->stage_stride = (mr + 4095) & ~(size_t) 4095;+ // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)+ bool fl = flow_on;+ int n_exp_max = 0;+ for (const Layer& l : L)+ if (!l.mtp)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di) {+ // (a group's members: each its share of the down's blocks in every row)+ fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&+ (l.nm == 1 || (flow_tp && (g.ff / l.nm) %+ std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));+ n_exp_max = std::max(n_exp_max, l.n_exp);+ }+ if (fl) // (each part read unbuffered from its offset rounded down: a page more each)+ cp->stage_stride = std::max(cp->stage_stride, flow_stride());+ cp->S = cp->ring_n;+ if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;+ cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);+ if (!fl) continue;+ cudaSetDevice(devs[di].id);+ cp->flow = true;+ cp->dev = (int) di;+ cp->io = std::make_unique<IoQueue>(n_readers);+ cp->stage_item.assign((size_t) cp->S, -1);+ cp->stage_ev.resize((size_t) cp->S);+ cp->ring_fill.resize((size_t) cp->ring_n);+ cp->ring_free.resize((size_t) cp->ring_n);+ for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})+ for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));+ // (a region is written again only bc_n batches later - more than a layer's items, so after that+ // layer's bounds wait, which every copy from it precedes)+ cp->bc_n = n_exp_max + 2;+ cp->bc_w = cp->ring_n + 1;+ if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||+ !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))+ return false;+ }+ // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one+ // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each+ // window, a page fault at a time) - DS4_CHUNK_MIN still overrides+ bool all_flow = true;+ for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);+ if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;+ }+ for (Layer& l : L) {+ if (l.mtp) continue;+ for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)+ Layer& x = member(l, m);+ if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;+ if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;+ }+ }+ }+ {+ void* m = nullptr;+ if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory for the decode path's requests";+ return false;+ }+ host_allocs.push_back(m);+ reqs = (Req*) m;+ if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory for the decode path's answers";+ return false;+ }+ host_allocs.push_back(m);+ resps = (Resp*) m;+ std::memset((void*) reqs, 0, sizeof(Req) * kRing);+ std::memset((void*) resps, 0, sizeof(Resp) * kRing);+ for (auto& x : layer_of) x.store(-1);+ ahead_fl.assign(L.size() * 1024, 0);+ ahead_rank.assign(L.size() * 1024, 0);+ ahead_for.assign(L.size(), 0);+ const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;+ service = std::thread([this, core] {+ if (core >= 0) pin_self(core);+ serve_lane();+ });+ if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });+ }+ if (o.expert_cache) {+ size_t maxb = 0;+ for (const Layer& l : L) maxb = std::max(maxb, l.blob);+ const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));+ sw_bufs.resize((size_t) nt);+ for (SwapBufs& B : sw_bufs) {+ if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStream_t cs = nullptr;+ CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));+ B.st.push_back(cs);+ }+ }+ for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ CK(cudaStreamSynchronize(d.st));+ }+ if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)+ for (size_t di = 0; di < devs.size(); ++di) {+ workers.push_back(std::make_unique<Worker>());+ workers.back()->start();+ }+ calibrate_paths();+ if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());+ reset();+ return true;+ }+++ // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed+ // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)+ size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }+ size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }+ size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }++ // ---- the chunked prompt path+ size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows+ size_t chunk_hand_bytes() const {+ return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);+ }++ bool alloc_chunks(std::string& err) {+ chunks.resize(devs.size());+ const int D = g.dim, H = g.hc, HD = g.hd;+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ for (size_t di = 0; di < devs.size(); ++di) {+ Dev& d = devs[di];+ cudaSetDevice(d.id);+ chunks[di] = std::make_unique<Chunk>();+ Chunk& c = *chunks[di];+ const int nm = (int) groups[(size_t) d.grp].mem.size();+ const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);+ int FF = 0, ff0_ = 0;+ split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)+ const size_t C = (size_t) chunk;+ c.C = chunk;+ c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);+ // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)+ c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));+ c.w16_elems = (size_t) 8 << 20;+ bool has_eg = false, has_first = false;+ size_t stride = 0;+ for (const Layer& l : L)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di && !l.mtp) {+ has_eg = has_eg || l.eg >= 0;+ has_first = has_first || l.il == 0;+ stride = std::max(stride, align256(unit(member(l, m))));+ }+ const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;+ bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&+ dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&+ dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&+ dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&+ dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&+ dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&+ dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&+ dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&+ dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&+ dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&+ dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&+ dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&+ dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&+ dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&+ dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&+ dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&+ dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&+ dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&+ dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&+ halloc(c.bounds_pin, 2048, err);+ if (!ok) return false;+ if (has_first && !dalloc(d, c.emb, C * D, err)) return false;+ if (has_eg) {+ const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);+ if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||+ !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))+ return false;+ }+ if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches+ const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;+ c.ring_stride = stride;+ c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);+ if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;+ }+ CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));+ for (int i = 0; i < Chunk::kParts; ++i) {+ CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));+ }+ if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {+ err = "cuBLAS on CUDA" + std::to_string(d.id);+ return false;+ }+ for (int sl = 0; sl < 2; ++sl) {+ if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;+ if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;+ if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;+ CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));+ }+ if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;+ }+ return true;+ }++ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS+ bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,+ std::string& err) {+ if (rows <= 0) return true;+ const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);+ for (int c0 = 0; c0 < m.n; c0 += cols) {+ const int nc = std::min(cols, m.n - c0);+ if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)+ else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);+ const float one = 1.0f;+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,+ m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);+ if (st != CUBLAS_STATUS_SUCCESS) {+ err = "cublasGemmEx: status " + std::to_string((int) st);+ return false;+ }+ }+ return true;+ }+ // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)+ bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,+ float* y, int ldy, std::string& err) {+ const float zero = 0.0f;+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,+ x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP);+ if (st != CUBLAS_STATUS_SUCCESS) {+ err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);+ return false;+ }+ return true;+ }+ // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale+ bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {+ const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;+ const float one = 1.0f, zero = 0.0f;+ if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=+ CUBLAS_STATUS_SUCCESS) {+ err = "cublasSgemm (mHC mixes)";+ return false;+ }+ hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);+ return true;+ }++ // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full+ // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file+ // region read into pinned staging)+ bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,+ const uint8_t* src = nullptr) {+ if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into+ // pinned staging as its blob) or from the file's mapping+ auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };+ bool ok = true;+ if (l.nm == 1) {+ if (src != nullptr)+ ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ else+ for (int r = 0; r < 3; ++r)+ ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),+ cudaMemcpyHostToDevice, cs) == cudaSuccess;+ } else {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,+ (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,+ (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ }+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ return ok;+ }+ const auto& parts = l.hh[(size_t) e];+ const int D = g.dim, FF = g.ff;+ auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {+ return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ };+ auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {+ return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ };+ bool ok = true;+ if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets+ const uint8_t* p = src != nullptr ? src : parts[0];+ if (l.nm == 1) {+ for (int r = 0; r < 3; ++r)+ ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&+ cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&+ cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);+ } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;+ ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,+ nt * tile, nth * tile, (size_t) D / 16) &&+ cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&+ cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];+ ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,+ (size_t) (fn / 16) * row) &&+ cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&+ cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);+ }+ } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh+ for (int X = 0; X < 2; ++X) {+ ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,+ (size_t) (k / 16));+ ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);+ }+ ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);+ }+ } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of+ // both parts' columns+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)+ for (int X = 0; X < 2; ++X) {+ const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);+ if (a >= b) continue;+ ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,+ (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&+ cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,+ (size_t) (b - a) * 2);+ }+ ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);+ }+ const size_t tile = (size_t) 32 * l.lay.K[2];+ const int nh = D / 2;+ for (int X = 0; X < 2; ++X) {+ ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,+ (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));+ ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);+ }+ ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ return ok;+ }++ // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk+ // in layer order; the copies run at most kParts batches ahead of the compute+ struct Batch {+ int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range+ };+ struct Streamer {+ std::vector<Batch> b;+ size_t cp = 0, kp = 0; // the next batch to copy, to compute+ std::atomic<size_t> done{0}; // cp, for read_ahead's thread+ };+ // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel+ // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -+ // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone+ void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {+ static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *+ 1048576.0;+ std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied+ double ahead = 0;+ for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {+ const size_t done = S.done.load(std::memory_order_relaxed);+ while (!issued.empty() && issued.front().first < done) {+ ahead -= issued.front().second;+ issued.pop_front();+ }+ if (ahead > cap) {+ std::this_thread::sleep_for(std::chrono::microseconds(500));+ continue;+ }+ const Batch& bt = S.b[q];+ const Layer& l0 = L[(size_t) bt.il];+ double nb = 0;+ if (l0.dev == di)+ for (int i = 0; i < bt.n; ++i) {+ const int e = l0.host_list[(size_t) (bt.j0 + i)];+ if (l0.lay.fmt == gg::kFmtMXFP4) {+ if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);+ nb += (double) l0.blob;+ continue;+ }+ os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);+ nb += (double) l0.region_bytes[(size_t) e];+ }+ issued.push_back({q++, nb});+ ahead += nb;+ }+ }+ bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {+ const int P = Chunk::kParts;+ while (S.cp < S.b.size() && S.cp < S.kp + P) {+ const Batch& bt = S.b[S.cp];+ const int half = (int) (S.cp % P);+ const Layer& l0 = L[(size_t) bt.il];+ int mi = 0;+ for (int m = 0; m < l0.nm; ++m)+ if (member(l0, m).dev == di) mi = m;+ if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));+ uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots+ if (c.stage != nullptr) {+ if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)+ sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;+ std::atomic<bool> bad{false};+ c.readers->run(bt.n, [&](int i) {+ const int e = l0.host_list[(size_t) (bt.j0 + i)];+ if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)+ : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],+ l0.region_bytes[(size_t) e],+ sb + (size_t) i * c.stage_stride))+ bad.store(true);+ });+ if (bad.load()) {+ err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";+ return false;+ }+ }+ for (int i = 0; i < bt.n; ++i) {+ uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;+ if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,+ sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))+ return false;+ c.stream_bytes += (double) unit(member(l0, mi));+ }+ CK(cudaEventRecord(c.filled[half], c.cs));+ ++S.cp;+ S.done.store(S.cp, std::memory_order_relaxed);+ }+ return true;+ }++ // ---- the flow (see Chunk)+ // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute+ // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing+ static void kick(cudaStream_t s) {+#if defined(_WIN32) && defined(STRATA_USE_HIP)+ (void) cudaStreamQuery(s);+ (void) cudaGetLastError();+#else+ (void) s;+#endif+ }+ size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)+ return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;+ }+ size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }+ static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }+ // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from+ // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of+ // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole+ size_t mx_read_off(const Layer& l, int m, int r) const {+ return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : 0;+ }+ size_t mx_read_len(const Layer& l, int r) const {+ return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : mx_part_bytes(l, r);+ }+ // the device's member index in L0's group (0 alone)+ int flow_mi(const Chunk& c, const Layer& L0) const {+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == c.dev) return m;+ return 0;+ }+ void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {+ c.items.emplace_back();+ Chunk::Item& it = c.items.back();+ it.il = l.il;+ it.mi = mi;+ it.k = k;+ it.e = e;+ it.v = v;+ const bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;+ if (in_ram && ram_pinned) { // (DMA'd from where it lies)+ const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];+ for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);+ ++c.it_ram;+ return;+ }+ if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)+ it.ram = copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];+ bool hit = false;+ if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after+ const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;+ if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again+ c.lc_il = l.il; // once that layer's items are all read - `after` below)+ c.lc_used = 0;+ c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);+ c.lc_first = i;+ std::fill(c.lc_of.begin(), c.lc_of.end(), -1);+ }+ const int j = c.lc_of[(size_t) e];+ if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)+ it.ram = c.lc + (size_t) j * stride;+ it.after = c.lc_writer[(size_t) j];+ hit = true;+ } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)+ const int jn = c.lc_used++;+ c.lc_of[(size_t) e] = jn;+ c.lc_writer[(size_t) jn] = (int64_t) i;+ it.keep = c.lc + (size_t) jn * stride;+ it.after = (int64_t) c.lc_first - 1;+ }+ }+ it.stage = (int) (c.n_file++ % (uint64_t) c.S);+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;+ for (int r = 0; r < 3; ++r) {+ const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);+ it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);+ sb += flow_region(r);+ }+ it.left.store(in_ram && copy_mode ? 3 * NH : 3, std::memory_order_relaxed);+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);+ }+ // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance+ // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the+ // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -+ // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is+ // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer+ // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the+ // rest of its picks are streamed in)+ // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same+ // profile and the same routing - and each reads only its part of every expert)+ void flow_spec(Chunk& c, int k, int il, int T) {+ static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;+ Layer& l = L[(size_t) il];+ const int mi = flow_mi(c, l);+ int* vs = member(l, mi).vs_pin;+ c.spec_k = k;+ c.spec_il = il;+ c.spec_begin = c.base + c.items.size();+ const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += pr[e];+ std::vector<std::pair<double, int>> cand;+ const double picks = (double) l.k_exp * T;+ // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE+ // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole+ // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376+ // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and+ // 21 of 1568, the early reads 9-12% faster there)+ static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;+ const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;+ for (int e : l.host_list) {+ if (!early) break;+ const double p = sum > 0 ? pr[e] / sum : 0.0;+ const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));+ if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {+ return a.first > b.first;+ });+ if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));+ std::vector<char> is((size_t) l.n_exp, 0);+ int j = 0;+ for (const auto& x : cand) {+ vs[x.second] = l.n_slots + j++;+ is[(size_t) x.second] = 1;+ }+ for (int e : l.host_list)+ if (!is[(size_t) e]) vs[e] = l.n_slots + j++;+ for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);+ c.n_spec = (int) cand.size();+ c.it_spec += cand.size();+ }+ // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones+ void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {+ const Layer& l = L[(size_t) il];+ const int mi = flow_mi(c, l);+ const int* vs = member(l, mi).vs_pin;+ for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {+ const int v = fit(c, i).v;+ if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;+ }+ for (int e : l.host_list) { // (index order: their virtual slots ascending)+ const int v = vs[e];+ if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);+ }+ }+ // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)+ bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {+ const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);+ const size_t i = (size_t) (it - c.fl_layers.begin());+ if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)+ if (k + 1 < (int) c.fl_T.size()) {+ k2 = k + 1;+ il2 = il;+ return true;+ }+ if (i + 1 >= c.fl_layers.size()) return false;+ k2 = 0;+ il2 = c.fl_layers[i + 1];+ return true;+ }+ if (i + 1 < c.fl_layers.size()) {+ k2 = k;+ il2 = c.fl_layers[i + 1];+ return true;+ }+ if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {+ k2 = k + 1;+ il2 = c.fl_layers[0];+ return true;+ }+ return false;+ }+ // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each+ // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now+ bool flow_pump(Chunk& c, std::string& err) {+ const size_t end = c.base + c.items.size();+ IoQueue::Job jobs[6];+ while (c.n_read < end) {+ Chunk::Item& it = fit(c, c.n_read);+ if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot+ // written again once the layer before's items are all read (none still copies out of it)+ bool wait = false;+ if (it.keep == nullptr)+ wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;+ else {+ while (c.lc_chk < c.n_read &&+ (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))+ ++c.lc_chk;+ wait = c.lc_chk <= (size_t) it.after;+ }+ if (wait) break;+ }+ if (it.stage >= 0) {+ const int64_t prev = c.stage_item[(size_t) it.stage];+ if (prev >= 0) {+ if ((size_t) prev >= c.n_dma) break;+ const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);+ if (q == cudaErrorNotReady) break;+ CK(q);+ }+ c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;+ const Layer& l = L[(size_t) it.il];+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;+ const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)+ int nj = 0;+ for (int r = 0; r < 3; ++r) {+ if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X) {+ IoQueue::Job& j = jobs[nj++];+ j = IoQueue::Job{};+ j.len = nb;+ j.dst = sb + (size_t) X * nb;+ j.left = &it.left;+ j.bad = &it.bad;+ j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];+ }+ sb += flow_region(r);+ continue;+ }+ IoQueue::Job& j = jobs[nj++];+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];+ j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);+ j.len = mx_read_len(l, r);+ j.dst = sb;+ j.left = &it.left;+ j.bad = &it.bad;+ j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;+ j.map = pk.at(l.mx_file[r], j.off);+ j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;+ sb += flow_region(r);+ }+ c.io->push(jobs, nj);+ }+ ++c.n_read;+ }+ const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;+ struct Kick { // (every way out: what was queued is submitted)+ cudaStream_t s;+ const size_t &a, b;+ ~Kick() {+ if (a != b) kick(s);+ }+ } kq{c.cs, c.n_dma, dma0};+ while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {+ Chunk::Item& it = fit(c, c.n_dma);+ if (it.left.load(std::memory_order_acquire) != 0) break;+ if (it.bad.load(std::memory_order_acquire)) {+ err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) ++ " from the model's files failed";+ return false;+ }+ const size_t r = c.n_dma % R;+ if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));+ const Layer& l = L[(size_t) it.il];+ uint8_t* dst = c.ring + r * c.ring_stride;+ const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;+ for (int p = 0; p < 2; ++p)+ CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));+ if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)+ const int t2 = gg::expert_type(l.lay, 2);+ const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);+ CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,+ cudaMemcpyHostToDevice, c.cs));+ } else+ CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));+ CK(cudaEventRecord(c.ring_fill[r], c.cs));+ if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));+ c.stream_bytes += (double) unit(l);+ ++c.n_dma;+ }+ return true;+ }+ // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM+ // ones in shares between them (the device works while the reads run)+ bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {+ cudaStream_t s = d.st;+ const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)+ if (c.spec_k != k || c.spec_il != L0.il) {+ err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";+ return false;+ }+ flow_rest(c, k, L0.il, b);+ const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;+ int k2 = 0, il2 = 0;+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);+ const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);+ int v_done = 0;+ while (c.n_comp < le) {+ // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what+ // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so+ // the sums come out the same on every run whatever the reads' timing+ const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});+ if (!flow_pump(c, err)) return false;+ if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy+ const double tw = now_ms();+ kick(s);+ if (c.n_read > c.n_dma) c.io->wait_any(500);+ else {+ const Chunk::Item& it = fit(c, c.n_read);+ if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));+ }+ c.wait_ms += now_ms() - tw;+ continue;+ }+ const int n = (int) (i1 - i0);+ int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;+ int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;+ ++c.n_batch;+ // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)+ for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];+ bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];+ if (bc[n] > bc[0]) {+ CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));+ CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));+ if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))+ return false;+ }+ for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));+ kick(s);+ c.n_comp = i1;+ if (!flow_pump(c, err)) return false;+ const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;+ v_done = v_to;+ }+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;+ while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)+ c.items.pop_front();+ ++c.base;+ }+ return true;+ }++ // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)+ // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order+ bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,+ const int* b, int i0, int i1, int T, std::string& err) {+ const int D = g.dim, FF = l.ffn;+ while (i0 < i1) {+ int j = i0 + 1;+ while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;+ const int a = b[i0], nrows = b[j] - a;+ int max_rows = 0;+ for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);+ if (nrows > 0) {+ x3::SetArgs sa;+ sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)+ sa.stride = stride;+ sa.lay = l.nm > 1 ? l.hlay : l.lay;+ sa.n_exp = j - i0;+ sa.bounds = d_bounds + i0;+ sa.row0 = a;+ sa.max_rows = max_rows;+ sa.n_embd = D;+ sa.n_ff = FF;+ sa.limit = g.lim;+ if (sa.lay.fmt == gg::kFmtMXFP4) {+ gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ gg::set_down(sa, c.h16, FF, c.y, D, d.st);+ } else {+ x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ x3::set_down(sa, c.h16, FF, c.y, D, d.st);+ }+ if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {+ std::vector<int> bb(b + i0, b + j + 1);+ std::vector<uint16_t> hv((size_t) nrows * FF);+ std::vector<float> yv((size_t) nrows * D);+ CK(cudaStreamSynchronize(d.st));+ CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&+ wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};+ std::fwrite(hdr, 8, 6, dump_y);+ std::fwrite(bb.data(), 4, bb.size(), dump_y);+ std::fwrite(hv.data(), 2, hv.size(), dump_y);+ std::fwrite(yv.data(), 4, yv.size(), dump_y);+ std::fflush(dump_y);+ }+ moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);+ }+ i0 = j;+ }+ (void) err;+ return true;+ }++ bool load_dense(int di, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ size_t* acct = &d.weights;+ for (Layer& l0 : L)+ for (int mi = 0; mi < l0.nm; ++mi) {+ Layer& l = member(l0, mi);+ if (l.dev != di) continue;+ // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,+ // of the shared expert's FFN; the rest whole)+ const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);+ const std::string& P = l.name;+ const std::string A = P + "attn.";+ bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&+ (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&+ mat(d, A + "wkv", l.wkv, err, acct) &&+ (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&+ f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&+ f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&+ f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&+ f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&+ f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&+ f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&+ f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&+ f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&+ f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&+ f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&+ f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&+ (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||+ f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&+ mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&+ mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&+ mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);+ if (!ok) return false;+ l.sink += (size_t) l.ah0;+ for (int gi = 0; gi < l.ogn; ++gi)+ if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;+ if (l.kv_src) {+ const std::string C = A + "compressor.";+ if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;+ if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;+ if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||+ !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))+ return false;+ }+ if (l.ix_src) {+ const std::string I = A + "indexer.";+ if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))+ return false;+ if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))+ return false;+ }+ if (l.eg >= 0) {+ if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;+ if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already+ if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;+ } else {+ float *qw = nullptr, *kw = nullptr;+ if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))+ return false;+ std::vector<float> a((size_t) g.hc * g.dim), b(a.size());+ CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));+ for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product+ CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));+ l.eg_w = qw;+ }+ }+ if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;+ if (g.n_mtp > 0 && !l.mtp) {+ if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;+ if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||+ !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))+ return false;+ }+ if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;+ }+ if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns+ const int hn = head_nm();+ if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))+ return false;+ if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))+ return false;+ if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;+ if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head+ const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");+ const int R = g.d_rank;+ head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)+ // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same+ // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)+ if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {+ std::vector<uint16_t> rows((size_t) d.mk_vn * R);+ if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {+ err = "the markov head: read failed";+ return false;+ }+ if (th->dtype == "BF16")+ for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));+ if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||+ !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))+ return false;+ CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));+ }+ }+ }+ if (di == head_dev) {+ head = d.head;+ norm_w = d.norm_w;+ d_norm = d.d_norm;+ if (g.n_mtp > 0) {+ const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";+ if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))+ return false;+ const Tensor* te = pk.get(Z + "markov_head.embed.weight");+ const Tensor* th = pk.get(Z + "markov_head.head.weight");+ if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {+ err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";+ return false;+ }+ if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||+ !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))+ return false;+ }+ }+ if (g.n_mtp > 0 && di == d_dev) {+ if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||+ !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))+ return false;+ }+ return true;+ }++ void set_vm(Layer& l, int e, int slot) {+ uint8_t* b = l.vram_base + (size_t) slot * l.blob;+ if (l.lay.fmt == gg::kFmtMXFP4) {+ for (int r = 0; r < 3; ++r) {+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];+ m = x3::Mat{};+ m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;+ m.raw = b + l.lay.trellis[r];+ m.k = r < 2 ? g.dim : g.ff;+ m.n = r < 2 ? g.ff : g.dim;+ }+ return;+ }+ for (int r = 0; r < 3; ++r) {+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];+ m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);+ m.suh = (const uint16_t*) (b + l.lay.suh[r]);+ m.svh = (const uint16_t*) (b + l.lay.svh[r]);+ m.k = r < 2 ? g.dim : g.ff;+ m.n = r < 2 ? g.ff : g.dim;+ m.K = l.lay.K[r];+ m.cb = 2;+ m.ld = 0;+ }+ }+ // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)+ void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ const uint8_t* tr = blob + l.lay.trellis[r];+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ (size_t) nth * tile);+ std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);+ std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);+ }+ }+ void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ uint8_t* tr = blob + l.lay.trellis[r];+ for (int X = 0; X < NH; ++X)+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);+ std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);+ for (int X = 0; X < NH; ++X)+ std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],+ (size_t) l.hl.n[r] * 2);+ }+ }++ // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back+ void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)+ std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,+ blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);+ std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);+ std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];+ std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);+ std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);+ std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);+ }+ void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)+ std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,+ src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);+ if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);+ std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);+ }+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];+ std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);+ std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);+ if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);+ }++ bool place_experts(std::string& err) {+ // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)+ double room = 1e300, kept = 0;+ for (const Dev& d : devs)+ if (commit_charged(d)) room = commit_room();+ const double room0 = room;+ // per group: what its tightest member has left (each member holds its part of every VRAM expert of the+ // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round+ // the members layer by layer, so each member's total comes to about the mean's)+ auto ucount = [](const Layer& l) {+ double s = 0;+ for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));+ return s / l.nm;+ };+ for (size_t gi = 0; gi < groups.size(); ++gi) {+ double budget = 1e300;+ bool charged = false;+ for (int m : groups[gi].mem) {+ cudaSetDevice(devs[(size_t) m].id);+ size_t fr = 0, tot = 0;+ CK(cudaMemGetInfo(&fr, &tot));+ budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *+ o.vram_experts);+ charged = charged || commit_charged(devs[(size_t) m]);+ }+ if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)+ double all = 0;+ for (const Layer& l : L)+ if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;+ const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);+ kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;+ budget = capped;+ room = std::max(0.0, room - std::min(budget, all) * nm);+ }+ double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)+ for (Layer& l : L)+ if (l.grp == (int) gi && l.mtp) {+ l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));+ budget_t -= (double) l.n_vram * ucount(l);+ }+ double want = 0;+ for (const Layer& l : L)+ if (l.grp == (int) gi && !l.mtp) want += (double) ucount(l) * l.n_exp;+ const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;+ for (Layer& l : L) {+ if (l.grp != (int) gi || l.mtp) continue;+ l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));+ // never more than fits: the floor of each layer's share can still round up the total+ while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;+ }+ double used = 0;+ for (Layer& l : L)+ if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;+ // what the floors left: whole experts, layer by layer+ for (Layer& l : L) {+ if (l.grp != (int) gi) continue;+ while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {+ ++l.n_vram;+ used += (double) ucount(l);+ }+ }+ }+ if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it+ const os::Commit c = os::commit();+ const double gib = (double) (1ull << 30);+ const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;+ commit_kept = kept;+ pagefile_hint = std::ceil((pf + kept) / gib);+ std::fprintf(stderr,+ "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "+ "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "+ "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "+ "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",+ kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);+ } else if (room0 < 1e300 && o.verbose) {+ std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "+ "programs)\n", room0 / 1e9, commit_margin() / 1e9);+ }+ for (Layer& l : L) {+ // the hottest experts of the profile first, then by index+ std::vector<int> order((size_t) l.n_exp);+ std::iota(order.begin(), order.end(), 0);+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });+ l.slot.assign((size_t) l.n_exp, -1);+ l.host.assign((size_t) l.n_exp, nullptr);+ // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps+ l.n_slots = std::min(l.n_vram, l.n_exp);+ if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)+ l.n_vram = l.n_slots - 1;+ l.spare_vram = l.n_slots - 1;+ } else {+ l.n_vram = l.n_slots;+ l.spare_vram = -1;+ }+ for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;+ for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {+ Layer& x = member(l, m);+ Dev& d = devs[(size_t) x.dev];+ cudaSetDevice(d.id);+ void* v = nullptr;+ if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {+ err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) ++ " experts on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ d.allocs.push_back(v);+ d.experts += (size_t) l.n_slots * unit(x);+ x.vram_base = (uint8_t*) v;+ }+ l.heat.assign((size_t) l.n_exp, 0.0f);+ { // the profile as the starting heat: its counts scaled to ~512 hits a layer+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ const double sc = sum > 0 ? 512.0 / sum : 0.0;+ for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);+ }+ const int n_host = l.n_exp - l.n_vram;+ { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;+ l.hl.k[r] = k;+ l.hl.n[r] = n;+ l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;+ off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);+ l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;+ off += align256((size_t) k * 2);+ l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;+ off += align256((size_t) n * 2);+ }+ l.hl.bytes = off;+ }+ l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});+ if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);+ } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)+ for (int e = 0; e < l.n_exp; ++e)+ if (pk.map(l.region_file[(size_t) e]) == nullptr) {+ err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";+ return false;+ }+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) {+ l.hh[(size_t) e][0] = file_part(l, e);+ l.host[(size_t) e] = l.hh[(size_t) e][0];+ }+ } else if (n_host > 0) {+ const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)+ const size_t bytes = (size_t) n_parts * l.hl.bytes;+ for (int X = 0; X < NH; ++X) {+ void* m = node_map(bytes, X, NH);+ if (m == nullptr) {+ err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) ++ " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";+ return false;+ }+ host_maps.emplace_back(m, bytes);+ int hi = 0;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;+ if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;+ }+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];+ }+ l.vm.assign((size_t) l.n_exp, {});+ if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);+ if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0)+ tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));+ cudaSetDevice(devs[(size_t) x.dev].id);+ CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));+ }+ }+ return true;+ }++ // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the+ // file: the mapping reads them on first use)+ bool load_experts_mx(std::string& err) {+ struct Job {+ int layer, e;+ };+ std::vector<Job> jobs;+ size_t maxb = 0;+ for (const Layer& l : L) {+ maxb = std::max(maxb, l.blob);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];+ });+ std::atomic<size_t> next{0};+ std::atomic<bool> failed{false};+ std::mutex emu;+ std::string ferr;+ std::atomic<int64_t> done_bytes{0};+ const double t0 = now_ms();+ std::vector<std::thread> th;+ for (int t = 0; t < 12; ++t)+ th.emplace_back([&] {+ uint8_t *stage = nullptr, *stage2 = nullptr;+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);+ for (;;) {+ const size_t i = next.fetch_add(1);+ if (i >= jobs.size() || failed.load()) break;+ const Layer& l = L[(size_t) jobs[i].layer];+ const int e = jobs[i].e;+ bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);+ for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);+ ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,+ unit(l), cudaMemcpyHostToDevice) == cudaSuccess;+ }+ if (!ok) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ failed.store(true);+ break;+ }+ const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;+ if (o.verbose && (i % 1024) == 0)+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,+ db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ if (stage) cudaFreeHost(stage);+ if (stage2) cudaFreeHost(stage2);+ });+ for (auto& t : th) t.join();+ if (failed.load()) {+ err = ferr;+ return false;+ }+ // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the+ // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them+ struct Warm {+ int layer, e;+ double share;+ };+ std::vector<Warm> warm;+ for (const Layer& l : L) {+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0) {+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);+ } else {+ warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});+ }+ }+ }+ std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });+ if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left+ std::vector<std::pair<int, int>> hot;+ for (const Warm& w : warm) hot.push_back({w.layer, w.e});+ if (!copy_experts(hot, err)) return false;+ if (copy_mode) {+ warm.erase(std::remove_if(warm.begin(), warm.end(),+ [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),+ warm.end());+ const uint64_t av = ram_avail();+ if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;+ }+ }+ if (ram_tier && warm_bytes > 0 && !warm.empty()) {+ // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for+ // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not+ // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on+ // Windows - the engine's own pinned buffers come after this+ const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));+ while (ram_slots.size() < want + 1) {+ const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());+ uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);+ if (c == nullptr) break;+ ram_chunks.push_back(c);+ ram_chunk_bytes.push_back(n * maxb);+ for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);+ }+ ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;+ for (Layer& l : L)+ if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);+ std::atomic<size_t> ri{0};+ std::atomic<bool> rfail{false};+ std::vector<std::thread> rt;+ for (int t = 0; t < 12 && ram_held > 0; ++t)+ rt.emplace_back([&] {+ for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {+ const Layer& l = L[(size_t) warm[i].layer];+ if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],+ mx_part_bytes(l, r), false);+ }+ });+ for (auto& t : rt) t.join();+ if (rfail.load()) {+ err = "reading the RAM tier's experts from the model's files failed";+ return false;+ }+ for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;+ if (ram_held > 0) ram_free.push_back(ram_held);+ if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });+ if (o.verbose)+ std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "+ "read from the model's files (%.1f s)\n",+ ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,+ (now_ms() - t0) / 1000.0);+ return true;+ }+ size_t nw = 0;+ for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)+ b += (double) L[(size_t) warm[nw].layer].blob;+ std::atomic<size_t> wi{0};+ std::vector<std::thread> wt;+ for (int t = 0; t < 12 && nw > 0; ++t)+ wt.emplace_back([&] {+ std::vector<uint8_t> buf(maxb);+ for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());+ });+ for (auto& t : wt) t.join();+ if (o.verbose && nw > 0)+ std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,+ nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);+ return true;+ }++ // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU+ // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the+ // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:+ // none - read in place: the page cache follows a conversation's experts where a static set (the profile's+ // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the+ // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file+ // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what+ // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place+ bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {+ if (o.mmap_experts > 0 || hot.empty()) return true;+ const double t0 = now_ms();+ auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };+ double all = 0;+ {+ std::vector<char> seen(L.size(), 0);+ for (const auto& h : hot) {+ const Layer& l = L[(size_t) h.first];+ all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));+ }+ }+ const uint64_t avail = ram_avail();+ // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),+ commit_room());+ // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are+ // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed+ // on 64 GB and on 32 GB of RAM)+ double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;+ if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;+ if (o.mmap_experts == 0 && all > can)+ std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "+ "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "+ "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);+ if (budget <= 0) {+ if (o.verbose)+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "+ "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",+ all / 1e9, can / 1e9, kRamMargin / 1e9);+ return true;+ }+ // NUMA parts: whole rows of each matrix, so any row count the nodes divide+ int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));+ if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)+ nh = std::atoi(v) >= 2 ? 2 : 1;+ if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;+ NH = nh;+ cp_lay.assign(L.size(), {0, 0, 0, 0});+ for (const Layer& l : L) {+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ cp_lay[(size_t) l.il][(size_t) r] = off;+ off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));+ }+ cp_lay[(size_t) l.il][3] = off;+ }+ // the hottest that fit the budget (a layer's first one brings its spare)+ std::vector<int> n_copy(L.size(), 0);+ double used = 0;+ size_t take = 0;+ for (; take < hot.size(); ++take) {+ const Layer& l = L[(size_t) hot[take].first];+ const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));+ if (used + add > budget) break;+ used += add;+ ++n_copy[(size_t) l.il];+ }+ // each layer's slots: one block per node of (copies + spare) parts+ std::vector<int> first(L.size(), -1);+ copy_spare.assign(L.size(), -1);+ for (Layer& l : L) {+ l.ram.assign((size_t) l.n_exp, -1);+ const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);+ if (ns == 0) continue;+ const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;+ std::array<uint8_t*, 2> base = {nullptr, nullptr};+ bool ok = true;+ for (int X = 0; X < NH && ok; ++X) {+ base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);+ ok = base[(size_t) X] != nullptr;+ if (ok) {+ ram_chunks.push_back(base[(size_t) X]);+ ram_chunk_bytes.push_back(bytes);+ }+ }+ if (!ok) {+ std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,+ ns, (double) bytes * NH / 1e9);+ n_copy[(size_t) l.il] = 0;+ continue;+ }+ first[(size_t) l.il] = (int) ram_part.size();+ for (int i = 0; i < ns; ++i)+ ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});+ if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;+ }+ struct Job {+ int layer, e, slot;+ };+ std::vector<Job> jobs;+ {+ std::vector<int> next = first;+ for (size_t i = 0; i < take; ++i) {+ const int il = hot[i].first;+ if (n_copy[(size_t) il] == 0) continue;+ jobs.push_back({il, hot[i].second, next[(size_t) il]++});+ }+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];+ });+ std::atomic<size_t> ji{0};+ std::atomic<bool> bad{false};+ std::atomic<int64_t> done_b{0};+ std::vector<std::thread> th;+ for (int t = 0; t < 12 && !jobs.empty(); ++t)+ th.emplace_back([&] {+ for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {+ const Job& j = jobs[i];+ const Layer& l = L[(size_t) j.layer];+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,+ ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))+ bad.store(true);+ }+ const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;+ if (o.verbose && i % 2048 == 0)+ std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,+ db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ });+ for (auto& t : th) t.join();+ if (bad.load()) {+ err = "copying the CPU lane's experts from the model's files failed";+ return false;+ }+ for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;+ copy_mode = true;+ ram_held = (int) jobs.size();+ double cb = 0;+ for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;+ std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",+ jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,+ jobs.size() < hot.size() ? "; the others read in place" : "");+ return true;+ }++ bool load_experts(std::string& err) {+ if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);+ // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place+ // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their+ // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are+ // dropped from it again+ struct Job {+ int layer, e;+ };+ std::vector<Job> jobs;+ for (const Layer& l : L)+ for (int e = 0; e < l.n_exp; ++e)+ if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});+ uint64_t warm = 0;+ if (file_experts) {+ std::vector<std::pair<double, Job>> cand;+ for (const Layer& l : L) {+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });+ for (const auto& c : cand) {+ const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];+ if (warm + b > warm_bytes) break;+ warm += b;+ jobs.push_back(c.second);+ }+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])+ return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];+ return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];+ });+ std::atomic<size_t> next{0};+ std::atomic<bool> failed{false};+ std::mutex emu;+ std::string ferr;+ const int nth = 12;+ size_t maxr = 0, maxb = 0;+ for (const Layer& l : L) {+ maxb = std::max(maxb, l.blob);+ for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);+ }+ std::atomic<int64_t> done_bytes{0};+ const double t0 = now_ms();+ std::vector<std::thread> th;+ for (int t = 0; t < nth; ++t)+ th.emplace_back([&] {+ std::vector<uint8_t> region(maxr);+ uint8_t *stage = nullptr, *stage2 = nullptr;+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);+ int cur_dev = -1;+ for (;;) {+ const size_t i = next.fetch_add(1);+ if (i >= jobs.size() || failed.load()) break;+ const Layer& l = L[(size_t) jobs[i].layer];+ const int e = jobs[i].e;+ if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],+ region.data())) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ failed.store(true);+ break;+ }+ if (l.slot[(size_t) e] >= 0) {+ repack(l, region.data(), stage);+ bool up = true;+ for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)+ const Layer& x = member(l, m);+ const Dev& d = devs[(size_t) x.dev];+ if (cur_dev != d.id) {+ cudaSetDevice(d.id);+ cur_dev = d.id;+ }+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);+ up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,+ unit(x), cudaMemcpyHostToDevice) == cudaSuccess;+ }+ if (!up) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);+ failed.store(true);+ break;+ }+ if (file_experts)+ os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);+ } else if (!file_experts) {+ for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);+ }+ const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) ++ (int64_t) l.region_bytes[(size_t) e];+ if (o.verbose && (i % 1024) == 0)+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),+ db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ if (stage) cudaFreeHost(stage);+ if (stage2) cudaFreeHost(stage2);+ });+ for (auto& t : th) t.join();+ if (failed.load()) {+ err = ferr;+ return false;+ }+ if (file_experts)+ std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",+ (now_ms() - t0) / 1000.0, warm / 1e9);+ return true;+ }++ std::string describe() const {+ std::ostringstream s;+ for (size_t di = 0; di < devs.size(); ++di) {+ const Dev& d = devs[di];+ int a = -1, b = -1, ev = 0, et = 0;+ for (const Layer& l : L)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di) {+ if (a < 0) a = l.il;+ b = l.il;+ ev += l.n_vram;+ et += l.n_exp;+ }+ size_t fr = 0, tot = 0;+ cudaSetDevice(d.id);+ cudaMemGetInfo(&fr, &tot);+ const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();+ char tp[64] = "";+ if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);+ char line[384];+ std::snprintf(line, sizeof line,+ "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "+ "%.2f GB, %.0f MiB free\n",+ d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,+ d.caches / 1e9, fr / 1048576.0);+ s << line;+ }+ int nh = 0;+ size_t hb_ = 0;+ for (const Layer& l : L) {+ nh += l.n_exp - l.n_vram;+ hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;+ }+ char line[200];+ if (copy_mode)+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "+ "parts), the rest in place in the model's files; %d threads\n",+ nh, hb_ / 1e9, ram_held, NH, cpu_threads());+ else if (file_experts)+ std::snprintf(line, sizeof line,+ "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",+ nh, hb_ / 1e9, cpu_threads());+ else+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,+ hb_ / 1e9, NH, cpu_threads());+ s << line;+ return s.str();+ }++ int cpu_threads() const {+ int n = 0;+ for (const auto& pl : pools) n += pl->size() - 1;+ return n + 1;+ }+ // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over+ // every node's cores (the pages lie where the cache put them)+ void make_pools() {+ std::vector<std::vector<int>> cs = cores;+ if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)+ std::vector<int> all = cs[0];+ cs.assign((size_t) NH, {});+ for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);+ }+ for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {+ std::vector<int> c = cs[(size_t) X];+ if (file_experts && NH == 1)+ for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());+ if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)+ int nw = (int) c.size();+ if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));+ pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)+ }+ (void) gg::mx_tier(); // (measured here, not on the first token)+ if (o.verbose) {+ int nthr = 0;+ for (auto& p : pools) nthr += p->size();+ std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),+ gg::mx_tier_report());+ }+ }++ void reset() {+ pos = 0;+ toks.clear();+ ctoks.clear();+ ck[0].pos = ck[1].pos = -1;+ for (Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m) {+ Layer& l = member(l0, m);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);+ if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);+ if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ }++ // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the+ // state is indexed by position, and what lies past a restored position is written again)+ void checkpoint(int slot) {+ Check& c = ck[slot & 1];+ c.pos = pos;+ c.win.resize(L.size());+ c.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ c.win[i].resize((size_t) g.win * win_rb());+ cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);+ if (l.c_kv) {+ c.comp[2 * i].resize((size_t) l.ratio * g.hd);+ c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);+ cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);+ cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,+ cudaMemcpyDeviceToHost, d.st);+ }+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ }++ // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1+ int best_check(int64_t limit) const {+ int b = -1;+ for (int i = 0; i < 2; ++i)+ if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;+ return b;+ }+ int64_t restorable(int64_t limit) const {+ const int b = best_check(limit);+ return b < 0 ? -1 : ck[b].pos;+ }++ bool restore(int64_t limit) {+ const int b = best_check(limit);+ if (b < 0) return false;+ const Check& c = ck[b];+ for (size_t i = 0; i < L.size(); ++i)+ for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)+ Layer& l = member(L[i], m);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);+ if (l.c_kv) {+ cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);+ cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,+ cudaMemcpyHostToDevice, d.st);+ }+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ pos = c.pos;+ toks.resize((size_t) pos);+ ctoks.resize((size_t) pos);+ for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has+ if (o2.pos > pos) o2.pos = -1;+ return true;+ }++ size_t bounce_bytes() const {+ return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);+ }+ size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }+ size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }+ size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }+ static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need++ // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the+ // host; a prompt's pictures as their spans' rows (set_images)+ VisionTower vt;+ VisionGeo vg;+ int img_tok = -1;+ std::string vision_err; // why the tower did not load (pictures off), or empty+ std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]+ std::vector<Image> imgs; // the pictures of the prompt being fed, by start+ bool load_vision(std::string& err) {+ vg.layers = (int) pk.num("vision.num_hidden_layers", 0);+ const int tok = (int) pk.num("image_token_id", -1);+ if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none+ vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);+ vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);+ vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);+ vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;+ const char* mark[3] = {"image_start", "image_newline", "image_end"};+ for (int i = 0; i < 3; ++i) {+ const Tensor* t = pk.get(mark[i]);+ img_mark[i].resize((size_t) g.dim);+ if (t == nullptr || t->dtype != "BF16" || t->bytes != (uint64_t) g.dim * 2 || !pk.read(*t, img_mark[i].data())) {+ err = std::string(mark[i]) + ": not a BF16 row of the model's width";+ return false;+ }+ }+ const double t0 = now_ms();+ Dev& d = devs[0];+ auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {+ const Tensor* t = pk.get(n);+ if (t == nullptr || t->dtype != "BF16") return false;+ shape = t->shape;+ return dst == nullptr || pk.read(*t, dst);+ };+ if (!vt.load(vg, d.id, read, err)) return false;+ img_tok = tok;+ CK(cudaSetDevice(d.id));+ CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)+ if (o.verbose)+ std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",+ vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),+ (now_ms() - t0) / 1000.0);+ return true;+ }+ bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {+ if (img_tok < 0) {+ err = "this model has no vision tower (or it is off)";+ return false;+ }+ std::vector<uint16_t> rows;+ int nh = 0, nw = 0;+ if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;+ const size_t D = (size_t) g.dim;+ span.resize(((size_t) nh * (nw + 1) + 2) * D);+ uint16_t* o = span.data();+ auto put = [&](const uint16_t* r) {+ std::memcpy(o, r, D * 2);+ o += D;+ };+ put(img_mark[0].data());+ for (int i = 0; i < nh; ++i) {+ for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);+ put(img_mark[1].data());+ }+ put(img_mark[2].data());+ return true;+ }+ // the row of a picture's span at position p (null: p is in none)+ const uint16_t* image_row(int64_t p) const {+ if (imgs.empty()) return nullptr;+ auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });+ if (it == imgs.begin()) return nullptr;+ const Image& im = *(it - 1);+ const int64_t r = p - im.start;+ if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;+ return im.rows->data() + (size_t) r * g.dim;+ }+ // token tk's embedding row at position p: inside a picture's span, the picture's+ const uint16_t* emb_row(int64_t p, int tk) const {+ const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;+ return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;+ }+ bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }+ // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)+ int32_t ctok(int64_t p, int tk) const {+ if (tk == img_tok && image_row(p) != nullptr) return -1;+ return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;+ }+ // a chunk's image rows (route()'s bits); false: none+ bool vl_bits(int64_t p0, int T, VlBits& b) const {+ if (imgs.empty()) return false;+ std::memset(&b, 0, sizeof b);+ bool any = false;+ for (int t = 0; t < T && t < kVlRows; ++t)+ if (in_image(p0 + t)) {+ b.w[t >> 5] |= 1u << (t & 31);+ any = true;+ }+ return any;+ }++ // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]+ // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a+ // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,+ // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.+ void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;+ const size_t per = (size_t) cols * (hd + hd / 32);+ b.clear();+ b.reads.reserve((size_t) T * cols);+ const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";+ const Tensor* tw = pk.get(P + "weight");+ const Tensor* ts = pk.get(P + "scale");+ const bool mx8 = tw->dtype == "MXFP8"; // (a GGUF: the scales between the codes, one per 32)+ const bool q6k = eg_q6k[(size_t) li] != 0; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read+ int64_t hits = 0;+ {+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ for (int t = 0; t < T; ++t) {+ const int64_t p = p0 + t;+ uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;+ uint8_t* scales = rows + (size_t) cols * hd;+ if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)+ std::memset(rows, 0, per);+ continue;+ }+ int64_t prod[8];+ bool blocked = false;+ for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)+ blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;+ const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];+ prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];+ }+ int64_t rolling = prod[0];+ for (int n = 1; n < pk.eg_ngram; ++n) {+ rolling ^= prod[n];+ for (int h = 0; h < pk.eg_heads; ++h) {+ const int c = (n - 1) * pk.eg_heads + h;+ const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];+ uint8_t* rd = rows + (size_t) c * hd;+ uint8_t* sd = scales + (size_t) c * (hd / 32);+ const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;+ if (const uint8_t* hit = eg_cache.find(key)) {+ std::memcpy(rd, hit, (size_t) hd);+ std::memcpy(sd, hit + hd, (size_t) hd / 32);+ ++hits;+ continue;+ }+ const auto fi = first.find(key);+ if (fi != first.end()) {+ b.dups.push_back({fi->second, {rd, sd}});+ continue;+ }+ first.emplace(key, (int) b.reads.size());+ EgRead r;+ r.file = tw->file;+ r.dst = rd;+ r.scales = sd;+ r.key = key;+ if (mx8) {+ r.mx8 = true;+ r.n = (uint32_t) (hd / 32 * 33);+ r.off = tw->off + (uint64_t) id * r.n;+ } else if (q6k) { // (into the row's code bytes; its scale bytes unused)+ r.n = (uint32_t) (hd / 256 * 210);+ r.off = tw->off + (uint64_t) id * r.n;+ } else {+ r.n = (uint32_t) hd;+ r.off = tw->off + (uint64_t) id * hd;+ r.file2 = ts->file;+ r.n2 = (uint32_t) (hd / 32);+ r.off2 = ts->off + (uint64_t) id * (hd / 32);+ }+ b.reads.push_back(r);+ }+ }+ }+ }+ eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();+ eg_hits_n += hits + (int64_t) b.dups.size();+ eg_reads_n += (int64_t) b.reads.size();+ eg_io.submit(b);+ }+ // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)+ void eg_finish(EgBatch& b) {+ if (b.post.load(std::memory_order_acquire) == 2) return;+ const int64_t t0 = now_ns();+ eg_io.wait(b);+ int want = 0;+ if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {+ const int hd = pk.eg_hd;+ for (const auto& d : b.dups) {+ const EgRead& r = b.reads[(size_t) d.first];+ std::memcpy(d.second.first, r.dst, (size_t) hd);+ std::memcpy(d.second.second, r.scales, (size_t) hd / 32);+ }+ {+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ for (size_t i = 0; i < b.reads.size(); ++i) {+ const EgRead& r = b.reads[i];+ if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)+ std::memset(r.dst, 0, (size_t) hd);+ std::memset(r.scales, 0, (size_t) hd / 32);+ continue;+ }+ if (uint8_t* c = eg_cache.put(r.key)) {+ std::memcpy(c, r.dst, (size_t) hd);+ std::memcpy(c + hd, r.scales, (size_t) hd / 32);+ }+ }+ }+ eg_io_ns += b.t1.load() - b.t0;+ b.live = false;+ b.post.store(2, std::memory_order_release);+ } else {+ while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();+ }+ eg_wait_ns += now_ns() - t0;+ }+ void eg_stats() {+ st.eg_rows = eg_rows_n.load();+ st.eg_hits = eg_hits_n.load();+ st.eg_reads = eg_reads_n.load();+ st.eg_io_ms = (double) eg_io_ns.load() / 1e6;+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ st.eg_cached = (int64_t) eg_cache.size();+ st.eg_cache_cap = (int64_t) eg_cache.capacity();+ }+ // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)+ void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {+ EgBatch b;+ eg_issue(p0, T, only, dst, b);+ eg_finish(b);+ }++ // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its+ // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,+ // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's+ // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down+ // pieces; the down bands). yd[slot] gets each row's expert output.+ struct Ent {+ int e = 0, n = 0, slot0 = 0;+ int tok[kc::kExl3CpuMaxN] = {};+ };+ void run_nodes(int n, const std::function<void(int)>* f) {+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);+ pools[0]->help();+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();+ }++ // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each+ // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),+ // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits+ // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds+ // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever+ // number of entries take it, and each job its band of h straight into the down's int8 input.+ // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)+ void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {+ if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);+ constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();+ const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);+ const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);+ const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)+ const float lim = g.lim;+ static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");+ { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at+ // once, so the SSD reads them with large requests side by side while the first entries compute+ // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost+ // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)+ std::vector<std::pair<const void*, size_t>> want;+ for (const Ent& en : ents)+ if (l.ram.empty() || l.ram[(size_t) en.e] < 0)+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});+ os::prefetch_cold(want);+ }+ // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token+ // index past the window (none on the decode or chunk paths) gets a row of its own after them+ uint8_t* xq = (uint8_t*) c_xh.data();+ uint8_t* hq = xq + (size_t) kWin * xa;+ uint8_t* xq_extra = hq + (size_t) kSlots * ha;+ auto x_act = [&](const Ent& en, int i) -> const void* {+ const int t = en.tok[i];+ return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;+ };+ {+ bool done[kWin] = {};+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i) {+ const int t = en.tok[i];+ if (t < kWin && done[t]) continue;+ if (t < kWin) done[t] = true;+ gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));+ }+ }+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);+ const void* ap[gg::kMxCpuMaxN];+ float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];+ float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = x_act(en, i);+ gp[i] = yg[i];+ up[i] = yu[i];+ }+ // (part X's rows, from where they are: a copy's part on node X, or the file's)+ gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);+ gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);+ for (int i = 0; i < en.n; ++i) {+ float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;+ for (int r = r0; r < r1; ++r) {+ float gv = yg[i][r - r0], uv = yu[i][r - r0];+ if (lim > 0.0f) {+ gv = std::min(gv, lim);+ uv = std::min(std::max(uv, -lim), lim);+ }+ h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;+ }+ if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);+ }+ };+ run_nodes(ne * GB, f);+ if (!band_q)+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i)+ gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);+ const void* ap[gg::kMxCpuMaxN];+ float* yp[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = hq + (size_t) (en.slot0 + i) * ha;+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;+ }+ gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);+ };+ run_nodes(ne * DB, f);+ }+ // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input+ // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same+ void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {+ constexpr int RB = 64; // rows a job+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();+ gg::IqCpu fm;+ std::string ferr;+ if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||+ fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {+ static std::atomic<bool> said{false};+ if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());+ for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);+ return;+ }+ const float lim = g.lim;+ { // (os::prefetch, as above)+ std::vector<std::pair<const void*, size_t>> want;+ for (const Ent& en : ents)+ if (l.ram.empty() || l.ram[(size_t) en.e] < 0)+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});+ os::prefetch(want);+ }+ // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of+ // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or+ // chunk paths), then (x no longer needed) the slot's h+ uint8_t* aq = (uint8_t*) c_xh.data();+ auto x_act = [&](const Ent& en, int i) -> uint8_t* {+ const int t = en.tok[i];+ return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;+ };+ {+ bool done[kWin] = {};+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i) {+ const int t = en.tok[i];+ if (t < kWin && done[t]) continue;+ if (t < kWin) done[t] = true;+ gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));+ }+ }+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);+ const void* ap[gg::kMxCpuMaxN];+ float* hp[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = x_act(en, i);+ hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)+ }+ // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any+ // block size of the type)+ gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,+ r0 - X * FFh, r1 - X * FFh, lim);+ };+ run_nodes(ne * GB, f);+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i)+ gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,+ aq + (size_t) (kWin + en.slot0 + i) * E * 4);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);+ const void* hp[gg::kMxCpuMaxN];+ float* yp[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)+ }+ gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);+ };+ run_nodes(ne * DB, f);+ }++ void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,+ const int* ord = nullptr) {+ if (ldx == 0) ldx = (size_t) g.dim;+ if (l.lay.fmt == gg::kFmtMXFP4) return cpu_experts_mx(l, x, ents, ldx, ord);+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;+ const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;+ const int ne = (int) ents.size();+ const float lim = g.lim;+ auto part = [&](int e, int X, int r) {+ const uint8_t* b = l.hh[(size_t) e][(size_t) X];+ kc::Exl3Part p;+ p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);+ p.suh = (const uint16_t*) (b + l.hl.suh[r]);+ p.svh = (const uint16_t*) (b + l.hl.svh[r]);+ p.k = l.hl.k[r];+ p.n = l.hl.n[r];+ p.K = l.lay.K[r];+ return p;+ };+ auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };+ auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };+ auto acc = [&](int X, int sl, int r, int q) {+ return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;+ };+ auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };+ auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };+ auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / (2 * GQ))];+ const int r = (j / GQ) % 2, q = j % GQ;+ const kc::Exl3Part p = part(en.e, X, r);+ const float* xp[kc::kExl3CpuMaxN];+ float* ap[kc::kExl3CpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ const float* xt = x + (size_t) en.tok[i] * ldx;+ for (int b = 2 * q; b < 2 * q + 2; ++b)+ xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);+ xp[i] = xh(X, sl, r);+ ap[i] = acc(X, sl, r, q);+ }+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);+ };+ run_nodes(ne * 2 * GQ, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / GB)];+ const int b = j % GB;+ const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ float y[2][128], a[128], hb[128];+ for (int r = 0; r < 2; ++r) {+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;+ for (int q = 0; q < GQ; ++q) {+ const float* aq = acc(X, sl, r, q) + b * 128;+ for (int c = 0; c < 128; ++c) a[c] += aq[c];+ }+ double xs = 0.0;+ for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];+ kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);+ }+ for (int c = 0; c < 128; ++c) {+ float gv = y[0][c], uv = y[1][c];+ if (lim > 0.0f) {+ gv = std::min(gv, lim);+ uv = std::min(std::max(uv, -lim), lim);+ }+ hb[c] = gv / (1.0f + std::exp(-gv)) * uv;+ }+ const int gb = X * GB + b;+ xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);+ }+ };+ run_nodes(ne * GB, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / DQ)];+ const int q = j % DQ;+ const kc::Exl3Part p = part(en.e, X, 2);+ const float* xp[kc::kExl3CpuMaxN];+ float* ap[kc::kExl3CpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ xp[i] = xhd(en.slot0 + i);+ ap[i] = accd(X, en.slot0 + i, q);+ }+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);+ };+ run_nodes(ne * DQ, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / DB)];+ const int b = j % DB;+ const kc::Exl3Part pd = part(en.e, X, 2);+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ float a[128];+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;+ for (int q = 0; q < DQ; ++q) {+ const float* aq = accd(X, sl, q) + b * 128;+ for (int c = 0; c < 128; ++c) a[c] += aq[c];+ }+ double xs = 0.0;+ for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];+ kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);+ }+ };+ run_nodes(ne * DB, f);+ }++ // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it+ // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs+ // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's+ // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:+ // the CPU lane reads it from the file again.+ void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot+ for (;;) {+ RamSwap rs;+ {+ std::unique_lock<std::mutex> lk(sw_mu);+ ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });+ if (quit.load()) return;+ rs = ram_queue.front();+ ram_queue.pop_front();+ }+ const Layer& l = L[(size_t) rs.layer];+ if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {+ for (int r = 0; r < 3; ++r) // (one copy of it in RAM)+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],+ mx_part_bytes(l, r), false);+ } else {+ rs.in = -1; // (failed: the slot goes back)+ }+ std::lock_guard<std::mutex> lk(sw_mu);+ ram_ready.push_back(rs);+ }+ }++ void cache_loop(SwapBufs& B) {+ for (;;) {+ Swap sw;+ {+ std::unique_lock<std::mutex> lk(sw_mu);+ sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });+ if (quit.load()) return;+ sw = sw_queue.front();+ sw_queue.pop_front();+ }+ Layer& l = L[(size_t) sw.layer];+ if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)+ swap_in_mx(l, sw, B);+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_ready.push_back(sw);+ continue;+ }+ parts_to_blob(l, l.hh[(size_t) sw.in], B.a);+ if (l.nm == 1) {+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t cs = B.st[(size_t) l.dev];+ cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);+ if (!file_experts)+ cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,+ cudaMemcpyDeviceToHost, cs);+ cudaStreamSynchronize(cs);+ } else { // a group: each member's part in, each member's part out+ for (int m = 0; m < l.nm; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = B.st[(size_t) x.dev];+ blob_to_half(l, B.a, m, B.h);+ cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);+ cudaStreamSynchronize(cs);+ if (file_experts) continue;+ cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,+ cudaMemcpyDeviceToHost, cs);+ cudaStreamSynchronize(cs);+ half_to_blob(l, B.h, m, B.b);+ }+ }+ if (!file_experts)+ for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);+ if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte+ std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);+ bool ok = true;+ for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)+ const int e = pass == 0 ? sw.in : sw.out;+ reg.resize(l.region_bytes[(size_t) e]);+ pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());+ if (pass == 0) {+ repack(l, reg.data(), blob.data());+ std::vector<uint8_t> dev(l.blob);+ if (l.nm == 1)+ cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);+ else+ for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)+ std::vector<uint8_t> h(member(l, m).hblob);+ cudaSetDevice(devs[(size_t) member(l, m).dev].id);+ cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,+ cudaMemcpyDeviceToHost);+ half_to_blob(l, h.data(), m, dev.data());+ }+ for (int r = 0; r < 3; ++r)+ ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&+ std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&+ std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;+ } else {+ for (int X = 0; X < NH; ++X) {+ repack_part(l, reg.data(), X, part.data());+ for (int r = 0; r < 3; ++r) {+ const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];+ ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&+ std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&+ std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;+ }+ }+ }+ }+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,+ ok ? "both match the model file" : "MISMATCH");+ }+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_ready.push_back(sw);+ }+ }++ // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);+ // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)+ // from the layer's spare RAM slot, which it is copied into from VRAM here+ void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {+ uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)+ const std::vector<cudaStream_t>& copy_st = B.st;+ bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till+ // this swap's switch)+ for (int m = 0; m < l.nm && ok; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);+ ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),+ cudaMemcpyHostToDevice, cs) == cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ }+ if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot+ bool got = ok;+ for (int m = 0; m < l.nm && got; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),+ unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);+ }+ if (got) blob_to_ram(l, sw_b, sw.ram);+ else sw.ram = -1; // (back to the file's pages then)+ }+ if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte+ std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);+ for (int m = 0; m < l.nm && ok; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),+ cudaMemcpyDeviceToHost);+ if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());+ }+ ok = ok && read_mx_file(l, sw.in, file.data());+ for (int r = 0; r < 3 && ok; ++r)+ ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;+ bool ok_out = true;+ if (sw.ram >= 0) {+ ok_out = read_mx_file(l, sw.out, file.data());+ ram_to_blob(l, sw.ram, dev.data());+ for (int r = 0; r < 3 && ok_out; ++r)+ ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;+ }+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,+ sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");+ } else if (!ok) {+ std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);+ }+ }++ void cache_tick(int T) {+ if (!o.expert_cache || cache_ths.empty()) return;+ std::vector<Swap> ready;+ std::vector<RamSwap> rready;+ {+ std::lock_guard<std::mutex> lk(sw_mu);+ ready.swap(sw_ready);+ rready.swap(ram_ready);+ }+ for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)+ --ram_inflight;+ Layer& l = L[(size_t) rs.layer];+ if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {+ ram_free.push_back(rs.slot);+ continue;+ }+ l.ram[(size_t) rs.in] = rs.slot;+ if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {+ ram_free.push_back(l.ram[(size_t) rs.out]);+ l.ram[(size_t) rs.out] = -1;+ }+ ++st.ram_moves;+ }+ for (const Swap& sw : ready) {+ Layer& l = L[(size_t) sw.layer];+ const int old_slot = l.slot[(size_t) sw.out];+ const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];+ if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)+ const int s_in = l.ram[(size_t) sw.in];+ l.ram[(size_t) sw.in] = -1;+ if (sw.ram >= 0) {+ l.ram[(size_t) sw.out] = sw.ram;+ copy_spare[(size_t) l.il] = s_in;+ } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)+ --ram_held;+ if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;+ }+ } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)+ ram_free.push_back(l.ram[(size_t) sw.in]);+ l.ram[(size_t) sw.in] = -1;+ }+ l.slot[(size_t) sw.in] = l.spare_vram;+ if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);+ l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};+ l.host[(size_t) sw.in] = nullptr;+ l.slot[(size_t) sw.out] = -1;+ if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)+ l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);+ } else if (file_experts) { // (back to its pages in the file)+ l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};+ l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];+ } else {+ l.hh[(size_t) sw.out] = l.spare_host;+ l.host[(size_t) sw.out] = l.spare_host[0];+ l.spare_host = old_parts;+ }+ l.spare_vram = old_slot;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ const Dev& dm = devs[(size_t) x.dev];+ l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));+ l.tab_pin[2 * m + 1] = 0ull;+ cudaSetDevice(dm.id); // (stream order: before the next window)+ cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);+ cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);+ }+ l.swap_pending = false;+ --sw_inflight;+ ++st.swaps;+ }+ decay_tokens += T;+ if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens+ decay_tokens = 0;+ for (Layer& l : L)+ for (float& h : l.heat) h *= c_decay;+ }+ bool queued = false, rqueued = false;+ for (Layer& l : L) {+ if (l.cpu_hits.empty()) continue;+ std::vector<int> hits;+ hits.swap(l.cpu_hits);+ int best = -1, vin = -1;+ for (int e : hits)+ if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;+ if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in+ int cold = -1;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;+ if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {+ l.swap_pending = true;+ ++sw_inflight;+ vin = best;+ Swap sw{l.il, best, cold};+ if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_queue.push_back(sw);+ queued = true;+ }+ }+ if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;+ int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)+ for (int e : hits)+ if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&+ (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))+ rb = e;+ if (rb < 0) continue;+ int rc = -1;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;+ const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)+ if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;+ RamSwap rs;+ rs.layer = l.il;+ rs.in = rb;+ rs.out = room ? -1 : rc;+ rs.slot = ram_free.back();+ ram_free.pop_back();+ ++ram_inflight;+ std::lock_guard<std::mutex> lk(sw_mu);+ ram_queue.push_back(rs);+ rqueued = true;+ }+ if (queued) sw_cv.notify_all();+ if (rqueued) ram_cv.notify_one();+ }++ // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes+ void prefetch_loop() {+ std::vector<std::pair<const void*, size_t>> b;+ for (;;) {+ {+ std::unique_lock<std::mutex> lk(pf_mu);+ pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });+ if (quit.load(std::memory_order_relaxed)) return;+ b.swap(pf_q);+ }+ os::prefetch_cold(b); // (the resident ones would only cost the page walk)+ b.clear();+ }+ }+ // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)+ static bool file_read(const Layer& l, int e) {+ return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);+ }+ // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4+ // experts read in place, some of them from the files now (none: nothing to read early - null)+ const Layer* ahead_layer(const Layer& l0, int a) const {+ const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;+ if (j >= end || j >= (int) L.size()) return nullptr;+ const Layer& n = L[(size_t) j];+ if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||+ (int) n.slot.size() < n.n_exp)+ return nullptr;+ for (int e = 0; e < n.n_exp; ++e)+ if (file_read(n, e)) return &n;+ return nullptr;+ }+ // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the+ // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts+ std::vector<std::pair<const void*, size_t>> pf_want;+ void ahead_ask(int li, const Req& r, unsigned seq, int T) {+ pf_want.clear();+ for (int a = 1; a <= 2; ++a) {+ const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;+ if (m <= 0 || j >= (int) L.size()) continue;+ const Layer& n = L[(size_t) j];+ uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;+ uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;+ if (ahead_for[(size_t) j] != seq + (unsigned) a) {+ std::memset(fl, 0, 1024);+ std::memset(rk, 0, 1024);+ ahead_for[(size_t) j] = seq + (unsigned) a;+ }+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < m; ++i) {+ const int e = r.ahead[t][(a - 1) * kAhead1 + i];+ if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;+ fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);+ if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);+ if ((fl[e] & 8) || !file_read(n, e)) continue;+ fl[e] |= 8;+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});+ ++st.ahead_pf;+ }+ }+ if (pf_want.empty()) return;+ {+ std::lock_guard<std::mutex> lk(pf_mu);+ pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());+ }+ pf_cv.notify_one();+ }++ // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's+ // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.+ void serve_lane() {+ unsigned next = 1;+ auto last = std::chrono::steady_clock::now();+ std::vector<Ent> ents;+ while (!quit.load(std::memory_order_relaxed)) {+ Req& r = reqs[next % kRing];+ if (*(volatile unsigned*) &r.seq != next) {+ cpu_relax();+ if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))+ std::this_thread::sleep_for(std::chrono::microseconds(20));+ continue;+ }+ std::atomic_thread_fence(std::memory_order_acquire);+ const int li = layer_of[next % kRing].load(std::memory_order_acquire);+ Resp& rp = resps[next % kRing];+ const int T = std::max(1, std::min(kWin, r.T));+ for (int t = 0; t < T; ++t) rp.cpu[t] = 0;+ if (li >= 0 && li < (int) L.size()) {+ Layer& l = L[(size_t) li];+ uint32_t* pr = prof.data() + (size_t) li * g.n_exp;+ ents.clear();+ // lookahead: the guesses made for this layer (when they are for this request), counted on decode+ const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next+ ? ahead_fl.data() + (size_t) li * 1024 : nullptr;+ // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row+ float wv[kWin * kPicks];+ int tok_of[kWin * kPicks];+ int nslots = 0;+ std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only+ const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;+ for (int t = 0; t < T; ++t) {+ float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)+ if (dsk)+ for (int i = 0; i < l.k_exp; ++i)+ if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];+ for (int i = 0; i < l.k_exp; ++i) {+ const int e = r.ids[t][i];+ if (e < 0 || e >= l.n_exp) continue;+ ++pr[e];+ l.heat[(size_t) e] += 1.0f;+ if (fl != nullptr) {+ ++st.ahead_picks;+ st.ahead_hit_k += fl[e] & 1;+ st.ahead_hit += (fl[e] & 6) != 0;+ st.ahead_hit2 += (fl[e] >> 2) & 1;+ }+ if (l.slot[(size_t) e] < 0) {+ l.cpu_hits.push_back(e);+ if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {+ ++st.skipped;+ continue;+ }+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;+ else if (fl != nullptr) {+ ++st.ahead_cold;+ st.ahead_cold_hit += (fl[e] >> 3) & 1;+ }+ picks.push_back({e, t * kPicks + i});+ }+ }+ }+ if (fl != nullptr) { // the guesses by rank: how many were picked+ const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;+ for (int e = 0; e < l.n_exp && e < 1024; ++e) {+ if (rk[e] == 0 || rk[e] > 16) continue;+ bool hit = false;+ for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;+ ++st.ahead_rank_n[rk[e] - 1];+ st.ahead_rank_hit[rk[e] - 1] += hit;+ }+ }+ if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);+ std::sort(picks.begin(), picks.end());+ for (size_t a = 0; a < picks.size();) {+ Ent en;+ en.e = picks[a].first;+ en.slot0 = nslots;+ en.n = 0;+ while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {+ const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;+ en.tok[en.n++] = t;+ tok_of[nslots] = t;+ wv[nslots] = r.w[t][i];+ ++nslots;+ ++a;+ }+ ents.push_back(en);+ }+ // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for+ // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)+ int ord[kWin * kPicks];+ const int* po = nullptr;+ if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&+ ents.size() <= (size_t) (kWin * kPicks)) {+ const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;+ int n = 0;+ for (int pass = 0; pass < 2; ++pass)+ for (size_t q = 0; q < ents.size(); ++q) {+ const int e = ents[q].e;+ const bool early = !file_read(l, e) || (fa[e] & 8) != 0;+ if (early == (pass == 0)) ord[n++] = (int) q;+ }+ po = ord;+ }+ if (!ents.empty()) {+ const double tc = now_ms();+ cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);+ const int D = g.dim;+ for (int t = 0; t < T; ++t) {+ bool any = false;+ for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;+ if (!any) continue;+ float* o = rp.part[t];+ for (int e = 0; e < D; ++e) o[e] = 0.0f;+ for (int sl = 0; sl < nslots; ++sl) {+ if (tok_of[sl] != t) continue;+ const float* y = c_yd.data() + (size_t) sl * D;+ for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];+ }+ rp.cpu[t] = 1;+ }+ st.cpu_ms += now_ms() - tc;+ st.cpu_experts += nslots;+ }+ }+ std::atomic_thread_fence(std::memory_order_release);+ *(volatile unsigned*) &rp.seq = next;+ ++next;+ last = std::chrono::steady_clock::now();+ }+ }++ // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's+ static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {+ bool g = false;+ for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;+ return g ? gg::mv(j, n, s) : x3::mv(j, n, s);+ }+ bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {+ x3::MvJob j[x3::kMaxMvJobs];+ int n = 0;+ for (const x3::MvJob& x : jobs) j[n++] = x;+ if (!mv_any(j, n, d.st)) {+ err = "exl3 mv failed";+ return false;+ }+ return true;+ }+ bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {+ for (int a = 0; a < n; a += x3::kMaxMvJobs)+ if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {+ err = "exl3 mv failed";+ return false;+ }+ return true;+ }+ // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs+ static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,+ float alpha = 1.0f) {+ x3::MvJob j;+ j.m = &m;+ j.xf = x;+ j.y = y;+ j.alpha = alpha;+ j.ncol = T;+ j.xf_cs = x_cs;+ j.y_cs = y_cs;+ return j;+ }++ // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:+ // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm+ // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x+ // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),+ // the trunk's head, the markov bias token by token (greedy) and the confidence head.+ bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {+ if (g.n_mtp == 0 || n <= 0) return true;+ if (n > kWin * kPicks) {+ err = "seeding the drafter with " + std::to_string(n) + " rows";+ return false;+ }+ Dev& d = devs[(size_t) d_dev];+ cudaSetDevice(d.id);+ const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;+ if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps+ for (Dev& e : devs) {+ cudaSetDevice(e.id);+ CK(cudaStreamSynchronize(e.st));+ }+ cudaSetDevice(d.id);+ const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};+ std::fwrite(hdr, 8, 4, dump_d);+ std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);+ std::fflush(dump_d);+ }+ for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)+ const int na = std::min(kWin, n - a);+ CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,+ cudaMemcpyHostToDevice, d.st));+ if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;+ }+ CK(cudaEventRecord(ev_seed, d.st));+ Finish fm;+ fm.w = d_main_norm;+ fm.eps = g.eps;+ finish(d.ed, n, D, D, fm, d.st);+ for (int i = 0; i < g.n_mtp; ++i) {+ Layer& l = L[(size_t) (g.nl + i)];+ if (l.dev != d_dev) {+ err = "the DSpark blocks are split across devices";+ return false;+ }+ for (int a = 0; a < n; a += kWin) {+ const int na = std::min(kWin, n - a);+ if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) (p0 + a);+ fk.freqs = d.freqs_win;+ fk.quant = 1;+ fk.dst = d.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(d.kvr, na, HD, HD, fk, d.st);+ ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);+ }+ }+ return true;+ }++ bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,+ int top_k, float min_p, uint64_t seed, uint64_t counter) {+ const double t0 = now_ms();+ drafts_q = temperature > 0.0f && sample_drafts;+ dq_temp = temperature;+ dq_top_p = top_p;+ dq_top_k = top_k;+ dq_min_p = min_p;+ dq_seed = seed;+ dq_counter = counter;+ const bool ok = draft_(y, out, conf, err);+ if (!ok) drafts_q = false;+ if (max_drafts > 0 && (int) out.size() > max_drafts) {+ out.resize((size_t) max_drafts);+ conf.resize((size_t) max_drafts);+ }+ st.draft_ms += now_ms() - t0;+ ++st.drafts_rounds;+ return ok;+ }+ bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {+ out.clear();+ conf.clear();+ const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;+ const int64_t p0 = pos;+ if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;+ int tk[kWin];+ tk[0] = y;+ for (int t = 1; t < T; ++t) tk[t] = g.d_noise;+ Dev& d = devs[(size_t) d_dev];+ cudaSetDevice(d.id);+ for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));+ embed(d.emb_rows, T, d.h, D, H, d.st);+ init_pre(d.pre, T, d.st);+ WinPlan dp; // (window attention alone: the mailbox seqs)+ dp.async = !sync_decode && T <= async_max_t;+ for (int i = 0; i < g.n_mtp && dp.async; ++i) {+ const int il = g.nl + i;+ dp.seq[il] = ++seq_issued;+ layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);+ }+ const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;+ if (dump_d != nullptr && h_dump == nullptr &&+ (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))+ return false;+ for (int i = 0; i < g.n_mtp; ++i) {+ Layer& l = L[(size_t) (g.nl + i)];+ if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)+ ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);+ CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));+ }+ if (!layer(l, 0, p0, T, dp, err)) return false;+ if (dump_d != nullptr) {+ cudaSetDevice(d.id);+ CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));+ }+ }+ cudaSetDevice(d.id);+ hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)+ Dev& hd = devs[(size_t) head_dev];+ // (through the host to the head's device, and to the head group's other members - the drafter's device can be+ // the head group's lead itself, a group of every GPU)+ if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {+ CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));+ CK(cudaEventRecord(d.ev_out, d.st));+ }+ if (d_dev != head_dev) {+ cudaSetDevice(hd.id);+ CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));+ CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));+ }+ // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)+ const std::vector<int>& hg = groups[(size_t) hd.grp].mem;+ const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)+ for (int m = (int) hg.size() - 1; m >= 0; --m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ if (m > 0) {+ CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));+ CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));+ } else+ CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));+ Finish fn;+ fn.w = e.d_norm;+ fn.eps = g.eps;+ finish(e.ao, T, D, D, fn, e.st);+ if (!head_share(e, m, e.ao, T, err, !mk)) return false;+ }+ h_idx[0] = y;+ if (mk) {+ // the markov steps over the group: each member its rows of the bias and its share's argmax, every member+ // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,+ // each step ~1/nm of their 66 MB and no gather of the shares+ const int nm = (int) hg.size();+ const unsigned seq = ++mk_seq;+ for (int m = 0; m < nm; ++m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));+ CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));+ }+ for (int i = 0; i < T; ++i)+ for (int m = 0; m < nm; ++m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;+ markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);+ argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);+ markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,+ e.st);+ }+ cudaSetDevice(hd.id);+ } else {+ CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));+ strata::kernels::SamplerParams sp;+ sp.greedy = true;+ sp.temperature = 0.0f;+ if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)+ sp.greedy = false;+ sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);+ sp.top_p = dq_top_p;+ sp.top_k = dq_top_k;+ sp.min_p = dq_min_p;+ sp.seed = dq_seed;+ }+ for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft+ markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);+ if (drafts_q) {+ sp.counter = dq_counter + (uint64_t) i;+ strata::kernels::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);+ } else+ strata::kernels::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);+ }+ }+ confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);+ int tok[kWin + 1];+ float cf[kWin];+ CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];+ for (int i = 0; i < T; ++i) cf[i] = h_w[i];+ if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden+ std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream+ CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));+ cudaSetDevice(d.id);+ CK(cudaStreamSynchronize(d.st));+ const int64_t hdr[4] = {'D', p0, y, T};+ std::fwrite(hdr, 8, 4, dump_d);+ std::fwrite(tok + 1, 4, (size_t) T, dump_d);+ std::fwrite(cf, 4, (size_t) T, dump_d);+ std::fwrite(xs.data(), 4, xs.size(), dump_d);+ for (int i = 0; i < g.n_mtp; ++i) {+ std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);+ std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);+ }+ std::fflush(dump_d);+ cudaSetDevice(hd.id);+ }+ for (int i = 0; i < T; ++i) {+ out.push_back(tok[i + 1]);+ conf.push_back(cf[i]);+ }+ return true;+ }++ int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,+ const float* bias) {+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ strata::kernels::SamplerParams sp;+ sp.temperature = temperature;+ sp.greedy = temperature <= 0.0f;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ sp.logit_bias = bias;+ strata::kernels::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);+ int t = -1;+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ return t;+ }++ // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.+ // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own+ // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution+ // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the+ // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {+ const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);+ const int T = n + 1, V = g.vocab;+ const int64_t p0 = pos;+ int tk[kWin];+ tk[0] = y;+ for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];+ spec_win = true;+ const double tv = now_ms();+ const bool ok = forward(tk, T, T, err, 0);+ st.verify_ms += now_ms() - tv;+ spec_win = false;+ if (!ok) return false;+ Dev& hd = devs[(size_t) head_dev];+ cudaSetDevice(hd.id);+ int m = 0;+ if (temperature <= 0.0f) {+ strata::kernels::SamplerParams sp;+ sp.greedy = true;+ sp.temperature = 0.0f;+ strata::kernels::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;+ next = h_idx[m];+ } else {+ // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)+ // (never d itself): each row a token distributed as the model's own draw at that position, p through the+ // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,+ // never kept); past the first rejection the rows do not count+ const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&+ dq_min_p == min_p && dq_seed == seed && dq_counter == counter;+ if (!own) { // the drafter's argmax: q a point at each draft+ for (int i = 0; i < n; ++i) {+ h_specq[(size_t) i].n = 1;+ h_specq[(size_t) i].ids[0] = drafts[(size_t) i];+ h_specq[(size_t) i].p[0] = 1.0;+ }+ CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::kernels::SpecList),+ cudaMemcpyHostToDevice, hd.st));+ }+ for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];+ CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));+ strata::kernels::SamplerParams sp;+ sp.greedy = false;+ sp.temperature = temperature;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ strata::kernels::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;+ next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);+ }+ drafts_q = false;+ rollback(p0, T, m + 1);+ if (!seed_drafter(p0, m + 1, 0, err)) return false;+ accepted = m;+ st.drafts_offered += n;+ st.drafts_accepted += m;+ for (int i = 0; i < n && i <= m && i < 8; ++i) {+ ++st.draft_reached[i];+ if (i < m) ++st.draft_kept[i];+ }+ return true;+ }++ // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone+ // (the compressed KV and the index keys past them are written again when those positions come)+ void rollback(int64_t p0, int T, int keep) {+ if (keep >= T) return;+ // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor+ // parallel, a millisecond of the host's before the next draft)+ std::vector<RingBatch> rb(devs.size());+ for (int il = 0; il < g.nl; ++il)+ for (int m = 0; m < L[(size_t) il].nm; ++m) {+ Layer& l = member(L[(size_t) il], m);+ RingBatch& b = rb[(size_t) l.dev];+ if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)+ cudaSetDevice(devs[(size_t) l.dev].id);+ ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);+ continue;+ }+ b.ring[b.n] = l.win;+ b.bak[b.n++] = l.ring_bak;+ }+ for (size_t di = 0; di < devs.size(); ++di)+ if (rb[di].n > 0) {+ cudaSetDevice(devs[di].id);+ ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);+ }+ for (int il = 0; il < g.nl; ++il)+ for (int m = 0; m < L[(size_t) il].nm; ++m) {+ Layer& l = member(L[(size_t) il], m);+ Dev& d = devs[(size_t) l.dev];+ if (l.c_snap != nullptr) {+ cudaSetDevice(d.id);+ const size_t ns = (size_t) l.ratio * g.hd * 4;+ cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);+ cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);+ compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,+ g.hd, l.c_norm, g.eps, d.latent, d.st);+ }+ }+ pos = p0 + keep;+ toks.resize((size_t) pos);+ ctoks.resize((size_t) pos);+ }++ // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows+ bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }+++ // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the+ // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in+ // place, the CPU lane's streamed in batches)+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {+ Layer& l = member(L0, mi);+ Dev& d = devs[(size_t) l.dev];+ Chunk& c = *chunks[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ prof_mark(d, 0);+ const int nm = L0.nm;+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;+ const ArGroup& ab = groups[(size_t) L0.grp].arb;+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;+ // Engram, in slabs of 256 tokens+ if (l.eg >= 0) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ const size_t per = (size_t) cols * (ehd + ehd / 32);+ const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)+ uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;+ CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.eg_ev[slot], s));+ for (int s0 = 0; s0 < T; s0 += 256) {+ const int ns = std::min(256, T - s0);+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);+ if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;+ engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);+ }+ }+ // attention+ if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;+ hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);+ hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))+ return false;+ if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;+ if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;+ Finish fq;+ fq.w = l.q_norm;+ fq.eps = g.eps;+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);+ // the window rows: the ring's (positions p0 - win ..) then the chunk's+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) p0;+ fk.freqs = freqs;+ fk.quant = 1;+ fk.dst = c.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(c.kvr, T, HD, HD, fk, s);+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it+ const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;+ if (snap) {+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);+ }+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);+ const uint8_t* ckv = nullptr;+ const long long ss = (long long) o.max_context + 64;+ if (l.ratio > 0) {+ Src& sr = srcs[(size_t) l.src];+ const int R = l.ratio;+ const int64_t g_first = p0 / R, g_end = (p0 + T) / R;+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);+ const float* latent = c.latent;+ if (l.kv_src) {+ if (R == 1) {+ Finish fc;+ fc.w = l.c_norm;+ fc.eps = g.eps;+ finish(c.ckr, T, HD, HD, fc, s);+ latent = c.ckr;+ } else {+ if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)+ const int64_t ps = p0 + wp.snap_t;+ for (int sl = 0; sl < R; ++sl) {+ const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;+ const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;+ const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;+ CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));+ }+ }+ compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);+ }+ }+ const int clen_max = (int) ((p0 + T) / R);+ if (l.ix_src) {+ Src& ks = srcs[(size_t) l.ik_src];+ if (l.owns_k && ng > 0) {+ x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);+ if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;+ Finish fi;+ fi.w = l.ik_norm;+ fi.eps = g.eps;+ fi.rd = g.rd;+ fi.pos = (int) (g_first * R);+ fi.pos_step = R;+ fi.freqs = freqs;+ fi.quant = 3;+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ fi.pdst_stride = (long long) ik_rb();+ finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);+ }+ if (clen_max > 0) {+ if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))+ return false;+ for (int a0 = 0; a0 < T; a0 += c.AI) {+ const int na = std::min(c.AI, T - a0);+ const int64_t pa = p0 + a0;+ if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,+ g.ix_heads * g.ix_hd, 0.0f, err))+ return false;+ Finish fi;+ fi.rd = g.rd;+ fi.pos = (int) pa;+ fi.vpp = g.ix_heads;+ fi.freqs = freqs;+ fi.quant = 3;+ finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);+ Rows lim;+ lim.gen = 1;+ lim.p0 = pa;+ lim.R = R;+ lim.kcap = g.ix_topk;+ lim.ccap = g.cand_topk;+ lim.blk = std::max(1, g.cand_blk);+ const int nlast = (int) ((pa + na) / R); // the slab's widest query+ int* tk = c.topk + (size_t) a0 * topk_stride();+ int* cd = c.cand + (size_t) a0 * cand_stride();+ const float* iw = c.iw + (size_t) a0 * 64;+ if (nlast <= 0) continue;+ if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {+ Rows cl = lim;+ cl.gen = 2;+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,+ cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);+ Rows tr = lim;+ tr.gen = 3;+ topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);+ } else {+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,+ lim, c.scores, ss, s);+ if (l.cand_src) {+ Rows cr = lim;+ cr.kcap = g.cand_topk;+ cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,+ (int) cand_stride(), s);+ }+ topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);+ }+ }+ }+ }+ if (l.kv_src && ng > 0) {+ Finish fc;+ fc.rd = g.rd;+ fc.pos = (int) (g_first * R);+ fc.pos_step = R;+ fc.freqs = freqs;+ fc.quant = 2;+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();+ fc.pdst_stride = (long long) ckv_rb();+ finish(const_cast<float*>(latent), ng, HD, HD, fc, s);+ if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once+ const int slot = wp.k % 2; // they took the chunk before last's)+ if (wp.k >= 2)+ for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);+ Xfer x;+ x.src[0] = fc.pdst;+ x.dst[0] = sr.stage(slot);+ x.bytes[0] = (size_t) ng * ckv_rb();+ x.n = 1;+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();+ x.bytes[1] = (size_t) ng * ik_rb();+ x.n = 2;+ }+ xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);+ }+ }+ ckv = sr.ckv[(size_t) l.dev];+ }+ // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc+ if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)+ for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {+ const int na = std::min(c.A, T - a0);+ if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;+ Finish fr;+ fr.rd = g.rd;+ fr.pos = (int) (p0 + a0);+ fr.vpp = NHL;+ fr.freqs = freqs;+ finish(c.q, na * NHL, HD, HD, fr, s);+ AttnRows ar;+ ar.gen = 1;+ ar.p0 = p0;+ ar.t0 = a0;+ ar.win = g.win;+ ar.R = l.ratio;+ ar.kcap = g.ix_topk;+ sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,+ na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)+ Finish fo;+ fo.rd = g.rd;+ fo.pos = (int) (p0 + a0);+ fo.vpp = NHL;+ fo.freqs = freqs;+ fo.inverse = 1;+ fo.quant = 1;+ finish(c.o, na * NHL, HD, HD, fo, s);+ x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);+ const int per = HQ / OG;+ for (int gi = 0; gi < OG; ++gi)+ if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,+ OG * g.o_lora, 0.0f, err))+ return false;+ x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);+ if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;+ }+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);+ prof_mark(d, 1);+ // the MoE+ if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;+ hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);+ hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)+ for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)+ const int nr = std::min(kVlRows, T - r0);+ VlBits vb;+ const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);+ route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,+ c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);+ }+ const int n_vs = L0.n_slots + (int) L0.host_list.size();+ CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)+ moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);+ CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));+ // the shared expert into acc while the host waits for the bounds+ if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))+ return false;+ swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);+ x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);+ if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;+ prof_mark(d, 2);+ CK(cudaStreamSynchronize(s));+ std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);+ if (dump_r != nullptr && mi == 0) {+ const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;+ std::vector<int> ri(np), rt(nk);+ std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);+ CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,+ cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};+ std::fwrite(hdr, 8, 8, dump_r);+ std::fwrite(ri.data(), 4, np, dump_r);+ std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);+ std::fwrite(b.data(), 4, b.size(), dump_r);+ std::fwrite(rt.data(), 4, nk, dump_r);+ std::fwrite(gl.data(), 4, gl.size(), dump_r);+ std::fwrite(gb.data(), 4, gb.size(), dump_r);+ std::fflush(dump_r);+ }+ if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) {+ const int v = L0.vs_pin[e];+ pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);+ }+ st.experts += (int64_t) T * l.k_exp;+ }+ if (c.flow) {+ if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 3);+ return true;+ }+ // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones+ // in shares between them (the device works while the copies run)+ const int di = l.dev, P = Chunk::kParts;+ int nb = 0;+ for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;+ int v_done = 0;+ for (int bi = 0; bi < nb; ++bi) {+ const Batch& bt = S.b[S.kp];+ const int half = (int) (S.kp % P);+ CK(cudaStreamWaitEvent(s, c.filled[half], 0));+ const int v0 = L0.n_slots + bt.j0;+ if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,+ b.data(), v0, v0 + bt.n, T, err))+ return false;+ CK(cudaEventRecord(c.freed[half], s));+ ++S.kp;+ if (!copy_ahead(d, c, S, di, err)) return false;+ const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;+ v_done = v_to;+ }+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 3);+ return true;+ }++ // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)+ void plan_streams() {+ for (Layer& l : L) {+ if (l.mtp || l.vs_pin == nullptr) continue;+ l.host_list.clear();+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);+ for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)+ Layer& x = member(l, m);+ int j = 0;+ for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;+ }+ }+ }++ // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows+ Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ void* dv[4] = {c.h, c.pre, c.topk, c.cand};+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};+ const size_t nb[4] = {nh, np, nk, nc};+ Xfer x;+ x.n = 4;+ for (int i = 0; i < 4; ++i) {+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];+ x.dst[i] = put ? (void*) hv[i] : dv[i];+ x.bytes[i] = nb[i];+ }+ return x;+ }++ // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the+ // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows+ // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across+ // layers and chunks+ struct Run {+ std::vector<int64_t> p0;+ std::vector<int> T;+ std::vector<unsigned> seq;+ int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)+ };+ // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):+ // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its+ // layers and back before the next - so a layer's experts are read from the files once for the whole run (the+ // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk+ bool run_dev_layers(int di, const Run& run, std::string& err) {+ Dev& d = devs[(size_t) di];+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();+ Streamer S; // (none: the flow streams)+ const size_t hb = chunk_hand_bytes();+ // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while+ // the one before runs (slot u % 2, once unit u - 2's copy of it is done)+ std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)+ for (int il : c.fl_layers)+ if (L[(size_t) il].eg >= 0)+ for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);+ auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {+ if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);+ const int li = egu[u].first, k = egu[u].second;+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);+ };+ size_t eg_u = 0;+ std::future<void> eg_next;+ if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);+ for (size_t li = 0; li < c.fl_layers.size(); ++li) {+ const int il = c.fl_layers[li];+ Layer& L0 = L[(size_t) il];+ const bool last_l = li + 1 == c.fl_layers.size();+ for (int k = 0; k < nk; ++k) {+ const int64_t p0 = run.p0[(size_t) k];+ const int T = run.T[(size_t) k], slot = k % 2;+ const bool last_k = k + 1 == nk;+ WinPlan wp;+ wp.par = true;+ wp.async = true;+ wp.win = run.seq[(size_t) k];+ wp.k = k;+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;+ if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way+ const double te = now_ms();+ eg_next.get();+ st.engram_ms += now_ms() - te;+ wp.eg_slot = (int) (eg_u % 2);+ if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);+ }+ if (li == 0) { // the chunk's input: its embedding rows+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));+ for (int t = 0; t < T; ++t)+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),+ (size_t) D * 2);+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.emb_ev[slot], s));+ embed(c.emb, T, c.h, D, H, s);+ init_pre(c.pre, T, s);+ } else { // or its state after the layer before+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);+ for (int i = 0; i < x.n; ++i)+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));+ }+ if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (it != g.d_targets.end()) {+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int nr = std::min(T, tap_cap);+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);+ CK(cudaStreamWaitEvent(s, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,+ (size_t) nr, cudaMemcpyDeviceToHost, s));+ }+ }+ if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;+ if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);+ if (!last_l) { // the chunk's state kept for the next layer+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);+ for (int i = 0; i < x.n; ++i)+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));+ continue;+ }+ // through every layer: the chunk's end, and the logits+ if ((size_t) k < chunk_done.size()) {+ CK(cudaEventRecord(chunk_done[(size_t) k], s));+ chunk_rec[(size_t) k].store(1, std::memory_order_release);+ }+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;+ for (int r0 = 0; r0 < T; r0 += kWin) {+ const int nr = std::min(kWin, T - r0);+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, nr, err)) return false;+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ }+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)+ const int t0 = T - n_logits;+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;+ }+ }+ lm_layers.store((int) li + 1, std::memory_order_release);+ }+ return true;+ }+ bool run_dev_chunks(int di, const Run& run, std::string& err) {+ Dev& d = devs[(size_t) di];+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();+ Streamer S;+ const int B = std::max(1, c.ring_n / Chunk::kParts);+ if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)+ CK(cudaStreamSynchronize(s));+ CK(cudaStreamSynchronize(c.cs));+ c.items.clear();+ c.base = c.n_read = c.n_dma = c.n_comp = 0;+ c.n_file = 0;+ std::fill(c.stage_item.begin(), c.stage_item.end(), -1);+ c.fl_layers.clear();+ for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)+ for (int m = 0; m < L[(size_t) il].nm; ++m)+ if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);+ c.fl_T = run.T;+ c.spec_k = c.spec_il = -1;+ c.wait_ms = 0;+ c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;+ c.lc_il = -1;+ c.lc_used = 0;+ c.lc_first = c.lc_chk = 0;+ c.io_d0 = c.io->direct_bytes();+ c.io_b0 = c.io->buffered_bytes();+ if (!c.fl_layers.empty()) {+ flow_spec(c, 0, c.fl_layers[0], run.T[0]);+ if (!flow_pump(c, err)) return false;+ }+ if (c.lm) return run_dev_layers(di, run, err);+ }+ for (int k = 0; k < nk && !c.flow; ++k)+ for (int il = 0; il < g.nl; ++il) {+ const Layer& L0 = L[(size_t) il];+ bool here = false;+ for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;+ if (!here) continue;+ for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)+ S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});+ }+ if (!S.b.empty() && c.ring == nullptr) {+ err = "no slots to stream experts into on CUDA" + std::to_string(d.id);+ return false;+ }+ std::atomic<bool> ra_stop{false};+ std::thread ra;+ if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });+ struct Join { // (every way out of the run: the read-ahead ends before S does)+ std::atomic<bool>& stop;+ std::thread& th;+ ~Join() {+ stop.store(true);+ if (th.joinable()) th.join();+ }+ } ra_join{ra_stop, ra};+ if (!copy_ahead(d, c, S, di, err)) return false;+ // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once+ // chunk k - 2's copy of it is done)+ std::vector<int> egs;+ for (const Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m)+ if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);+ auto read_eg = [this, &c, &run, &egs, eg_per](int k) {+ if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);+ for (int li : egs)+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);+ };+ std::future<void> eg_next;+ if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);+ for (int k = 0; k < nk; ++k) {+ const int64_t p0 = run.p0[(size_t) k];+ const int T = run.T[(size_t) k], slot = k % 2;+ const bool last_k = k + 1 == nk;+ if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way+ const double te = now_ms();+ eg_next.get();+ if (d.mi == 0) st.engram_ms += now_ms() - te;+ if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);+ }+ WinPlan wp;+ wp.par = true;+ wp.async = true;+ wp.win = run.seq[(size_t) k];+ wp.k = k;+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;+ bool started = false;+ std::vector<char> got(srcs.size(), 0);+ for (int il = 0; il < g.nl; ++il) {+ Layer& L0 = L[(size_t) il];+ int mi = -1;+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == di) mi = m;+ if (mi < 0) continue;+ if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off+ started = true;+ if (il == 0) {+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));+ for (int t = 0; t < T; ++t)+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),+ (size_t) D * 2);+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.emb_ev[slot], s));+ embed(c.emb, T, c.h, D, H, s);+ init_pre(c.pre, T, s);+ } else {+ Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];+ xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);+ set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);+ }+ }+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {+ if (sidx < 0 || got[(size_t) sidx]) continue;+ got[(size_t) sidx] = 1;+ Src& sr = srcs[(size_t) sidx];+ if (L[(size_t) sr.layer].grp == L0.grp) continue;+ const int64_t g_first = p0 / sr.ratio;+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);+ Xfer x;+ if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {+ x.src[x.n] = sr.stage(slot);+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();+ x.bytes[x.n++] = (size_t) ng * ckv_rb();+ }+ if (ng > 0 && sr.ik[(size_t) di] != nullptr) {+ x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();+ x.bytes[x.n++] = (size_t) ng * ik_rb();+ }+ if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);+ set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)+ }+ if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (it != g.d_targets.end()) {+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int nr = std::min(T, tap_cap);+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);+ CK(cudaStreamWaitEvent(s, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,+ (size_t) nr, cudaMemcpyDeviceToHost, s));+ }+ }+ if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;+ if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last+ const int nnext = L[(size_t) il + 1].nm;+ if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);+ xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);+ }+ }+ if (di == head_dev && (size_t) k < chunk_done.size()) {+ CK(cudaEventRecord(chunk_done[(size_t) k], s));+ chunk_rec[(size_t) k].store(1, std::memory_order_release);+ }+ if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;+ for (int r0 = 0; r0 < T; r0 += kWin) {+ const int nr = std::min(kWin, T - r0);+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, nr, err)) return false;+ if (di != head_dev) continue;+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ }+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)+ const int t0 = T - n_logits;+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;+ }+ }+ }+ return true;+ }++ // ---- a prompt's path: the flow or windows of 16+ // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a+ // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in+ // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the+ // flow, as before)+ void calibrate_paths() {+ if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;+ if (const char* v = std::getenv("DS4_PATH_PICK"))+ if (std::atoi(v) == 0) return;+ int di = -1;+ for (size_t i = 0; i < chunks.size() && di < 0; ++i)+ if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;+ if (di < 0) return;+ Chunk& c = *chunks[(size_t) di];+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ const double t0 = now_ms();+ // the DMA: staging -> the ring (once to warm up, once timed)+ const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});+ double ms = 0;+ for (int rep = 0; rep < 2; ++rep) {+ const double a = now_ms();+ if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||+ cudaStreamSynchronize(c.cs) != cudaSuccess) {+ cudaGetLastError();+ return;+ }+ ms = now_ms() - a;+ }+ pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);+ // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are+ auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {+ const double a = now_ms();+ left.store((int) jobs.size());+ c.io->push(jobs.data(), (int) jobs.size());+ while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ return now_ms() - a;+ };+ {+ const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;+ std::vector<uint8_t> src(part * 8, 1);+ std::vector<IoQueue::Job> jobs(8);+ std::atomic<int> left{0};+ std::atomic<bool> bad{false};+ for (int i = 0; i < 8; ++i) {+ jobs[(size_t) i].mem = src.data() + (size_t) i * part;+ jobs[(size_t) i].dst = c.stage + (size_t) i * part;+ jobs[(size_t) i].len = part;+ jobs[(size_t) i].left = &left;+ jobs[(size_t) i].bad = &bad;+ }+ run_jobs(jobs, left);+ const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);+ pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);+ }+ // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:+ // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane+ // three times (from the disk, then in RAM: the faster of two)+ int best = -1, most = 0;+ for (int il = 0; il < g.nl; ++il) {+ int n = 0;+ for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;+ if (n > most) most = n, best = il;+ }+ if (best < 0) return;+ const Layer& l = L[(size_t) best];+ std::vector<int> cold;+ const int n_read = std::min(8, c.S), n_cpu = 8;+ for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {+ if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;+ bool res = false;+ for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);+ if (!res) cold.push_back(e);+ }+ if ((int) cold.size() >= n_read + n_cpu) {+ std::vector<IoQueue::Job> jobs;+ std::atomic<int> left{0};+ std::atomic<bool> bad{false};+ for (int i = 0; i < n_read; ++i) {+ uint8_t* sb = c.stage + (size_t) i * c.stage_stride;+ for (int r = 0; r < 3; ++r) {+ IoQueue::Job j;+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];+ j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];+ j.len = mx_part_bytes(l, r);+ j.dst = sb;+ j.left = &left;+ j.bad = &bad;+ j.map = pk.at(l.mx_file[r], j.off);+ jobs.push_back(j);+ sb += flow_region(r);+ }+ }+ const double rd = run_jobs(jobs, left);+ if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);+ }+ { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM+ const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;+ std::vector<Ent> ents;+ for (int i = 0; i < n_cpu; ++i) {+ Ent en;+ en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;+ if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)+ for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)+ if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;+ if (en.e < 0) break;+ en.n = 1;+ en.slot0 = i;+ en.tok[0] = 0;+ ents.push_back(en);+ }+ if (ents.empty()) return;+ std::vector<float> x((size_t) g.dim);+ for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);+ const double bytes = (double) ents.size() * (double) l.blob;+ double a = now_ms();+ cpu_experts(l, x.data(), ents, (size_t) g.dim);+ const double t_first = now_ms() - a;+ double t_warm = 1e30;+ for (int rep = 0; rep < 2; ++rep) {+ a = now_ms();+ cpu_experts(l, x.data(), ents, (size_t) g.dim);+ t_warm = std::min(t_warm, now_ms() - a);+ }+ pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);+ if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);+ }+ pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;+ if (o.verbose) {+ char dk[32] = "not measured", cc[32] = "not measured";+ if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);+ if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);+ std::fprintf(stderr,+ "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "+ "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",+ pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);+ }+ }+ // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's+ // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)+ double pm_rate(const Layer& l, int& n_cold) const {+ const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0, cold = 0;+ n_cold = 0;+ for (int e = 0; e < l.n_exp; ++e) {+ sum += pr[e];+ if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;+ }+ if (n_cold == 0) return 0.0;+ const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;+ return pm.m * (double) l.k_exp * share / n_cold;+ }+ // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))+ double pm_picks(int n) const {+ double d = 0;+ for (int il = 0; il < g.nl; ++il) {+ int nc = 0;+ const double a = pm_rate(L[(size_t) il], nc);+ d += nc * (1.0 - std::exp(-a * n));+ }+ return d;+ }+ // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation+ // that (bisection over its log; a running mean of the log)+ void pm_learn(int n, double picked) {+ if (!pm.on || n < 2 || picked < 1) return;+ const double m0 = pm.m;+ double lo = std::log(1e-3), hi = std::log(1e3);+ for (int it = 0; it < 40; ++it) {+ pm.m = std::exp(0.5 * (lo + hi));+ (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);+ }+ const double obs = 0.5 * (lo + hi);+ pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);+ ++pm.n_m;+ }+ // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's+ // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -+ // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:+ // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at+ // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size+ // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted+ // faster (by half, with less than kPickRes in RAM); said with --verbose+ bool pick_path(int n) {+ pm.last = -1;+ if (!pm.on || n >= kPickMax) return true;+ double f_res = 0, f_cold = 0, w = 0;+ const int nw = n / kWin, rem = n % kWin;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ int nc = 0;+ const double a = pm_rate(l, nc);+ if (nc == 0) continue;+ int sampled = 0, res = 0, seen = 0;+ const int step = std::max(1, nc / 32);+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;+ bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;+ if (!r) {+ r = true;+ for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);+ }+ ++sampled;+ res += r;+ }+ const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;+ auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };+ f_res += share * u(n) * b;+ f_cold += (1.0 - share) * u(n) * b;+ w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;+ }+ const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;+ const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);+ const double t_flow = f_res / r_fr + f_cold / r_fc;+ const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;+ const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;+ // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the+ // fixed costs of every layer on every GPU again, which a pass of the flow pays once)+ const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;+ // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane+ // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would+ // cost the most; windows win where they are all in RAM)+ const double res_share = f_res / std::max(1.0, f_res + f_cold);+ const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));+ pm.last = flow ? 1 : 0;+ pm.last_cls = cls;+ pm.last_nw = n_win;+ pm.last_pred = flow ? t_flow : t_win;+ if (o.verbose)+ std::fprintf(stderr,+ "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "+ "experts' %.1f GB in RAM)\n",+ n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,+ 100.0 * res_share, (f_res + f_cold) / 1e9);+ return flow;+ }+ // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more+ static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }+ // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs+ // is mostly the fixed costs a window pays again and again)+ double pm_k(int c) const {+ for (int dc = 0; dc < 4; ++dc)+ for (int cc : {c - dc, c + dc})+ if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];+ return 1.0;+ }+ // n tokens through windows of 16, the last one's logits_rows rows of logits+ bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {+ logits_all = false;+ for (int a = 0; a < n; a += kWin) {+ const int T = std::min(kWin, n - a);+ if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;+ }+ return true;+ }+ // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken+ // / predicted), windows' time a window - running means+ void path_done(double ms) {+ if (pm.last < 0 || pm.last_pred <= 0) return;+ if (pm.last == 0) {+ const double w = ms / 1000.0 / std::max(1, pm.last_nw);+ pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;+ ++pm.win_n;+ } else {+ const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));+ double& k = pm.k[0][pm.last_cls];+ int64_t& n = pm.kn[0][pm.last_cls];+ k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;+ ++n;+ }+ pm.last = -1;+ }+ // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the+ // first would read from the files again more than the chunks' states moved to host memory and back at every layer+ // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,+ // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB+ // sets it) - at least one expert, or not layer-major+ bool lm_begin(const Run& run) {+ static const char* env = std::getenv("DS4_LAYER_MAJOR");+ if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;+ Chunk& c = *chunks[0];+ if (!c.flow) return false;+ size_t need = 0, n_exp = 0, blob_max = 0;+ double file_b = 0;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ size_t ne = 0;+ for (int e : l.host_list)+ if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;+ const size_t stride = (l.blob + 63) & ~(size_t) 63;+ need = std::max(need, ne * stride);+ n_exp = std::max(n_exp, (size_t) l.n_exp);+ blob_max = std::max(blob_max, stride);+ file_b += (double) ne * (double) l.blob;+ }+ int64_t rows = 0;+ for (int T : run.T) rows += T;+ const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;+ const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;+ if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;+ size_t lc_bytes = need;+ if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);+ else {+ const uint64_t avail = ram_avail();+ if (avail != ~uint64_t{0})+ lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);+ }+ if (lc_bytes < blob_max) return false;+ void* st_p = nullptr;+ c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;+ if (!c.lm_pinned) {+ cudaGetLastError();+ st_p = os::anon_alloc(state_b, -1, 1);+ if (st_p == nullptr) return false;+ }+ c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);+ if (c.lc == nullptr) {+ if (c.lm_pinned) cudaFreeHost(st_p);+ else os::anon_free(st_p, state_b);+ return false;+ }+ c.lm_state = (uint8_t*) st_p;+ c.lm_state_bytes = state_b;+ c.lc_bytes = lc_bytes;+ c.lc_of.assign(n_exp, -1);+ c.lc_writer.assign(n_exp, -1);+ c.lm = true;+ lm_layers.store(0);+ return true;+ }+ void lm_end() {+ for (size_t di = 0; di < chunks.size(); ++di) {+ Chunk& c = *chunks[di];+ if (!c.lm) continue;+ cudaSetDevice(devs[di].id);+ cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's+ cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)+ for (size_t i = c.base; i < c.n_read; ++i)+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ if (c.lm_pinned) cudaFreeHost(c.lm_state);+ else os::anon_free(c.lm_state, c.lm_state_bytes);+ os::anon_free(c.lc, c.lc_bytes);+ c.lm_state = c.lc = nullptr;+ c.lm_state_bytes = c.lc_bytes = 0;+ c.lm = c.lm_pinned = false;+ }+ }++ // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of+ // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)+ bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {+ if (n < 1) return true;+ if (pos + n > o.max_context) {+ err = "the context is full";+ return false;+ }+ for (int t = 0; t < n; ++t)+ if (tk[t] < 0 || tk[t] >= g.vocab) {+ err = "token " + std::to_string(tk[t]) + " out of range";+ return false;+ }+ const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;+ Run run;+ for (int a = 0; a < n; a += per) {+ run.p0.push_back(pos + a);+ run.T.push_back(std::min(per, n - a));+ run.seq.push_back(++win_seq);+ }+ logits_all = logits_rows > kWin;+ if (logits_all) chunk_logits.resize((size_t) n * g.vocab);+ for (int t = 0; t < n; ++t) {+ toks.push_back(tk[t]);+ ctoks.push_back(ctok(pos + t, tk[t]));+ }+ plan_streams();+ if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way+ run.snap_pos = pos + n - 1;+ for (int il = 0; il < g.nl; ++il) {+ Layer& l = L[(size_t) il];+ Dev& d = devs[(size_t) l.dev];+ if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;+ if (l.c_kv != nullptr && l.snap_kv == nullptr &&+ (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))+ return false;+ }+ }+ const int T_last = run.T.back();+ n_logits = logits_all ? 0 : std::min(logits_rows, T_last);+ const double tl = now_ms();+ { // each chunk's end on the head device, for the progress reports+ cudaSetDevice(devs[(size_t) head_dev].id);+ while (chunk_done.size() < run.T.size()) {+ cudaEvent_t e = nullptr;+ CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));+ chunk_done.push_back(e);+ }+ chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);+ for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);+ }+ prof_chunk = prof_on;+ const bool lm = lm_begin(run);+ struct LmEnd { // (every way out: its host memory freed once the device is done with it)+ Impl* p;+ bool on;+ ~LmEnd() {+ if (on) p->lm_end();+ }+ } lm_end_{this, lm};+ std::vector<std::string> errs(devs.size());+ for (size_t di = 0; di < devs.size(); ++di)+ workers[di]->submit([this, di, &run, &errs] {+ std::string e;+ if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;+ });+ size_t reported = 0;+ for (;;) {+ bool all = true;+ for (auto& w : workers) all = all && w->idle();+ if (progress && lm) { // (layer-major: the share of the layers every chunk went through)+ const int nd = lm_layers.load(std::memory_order_acquire);+ if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);+ } else if (progress) { // chunks through every layer (the last layer's device past their event)+ while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&+ cudaEventQuery(chunk_done[reported]) == cudaSuccess)+ ++reported;+ if (reported > 0 && reported < run.T.size())+ progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);+ }+ if (all) break;+ std::this_thread::sleep_for(std::chrono::milliseconds(50));+ }+ for (const std::string& e : errs)+ if (!e.empty()) {+ err = e;+ return false;+ }+ st.layer_ms += now_ms() - tl;+ eg_stats();+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ const double th = now_ms();+ CK(cudaStreamSynchronize(d.st));+ st.head_ms += now_ms() - th;+ if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed+ for (Dev& dv : devs) {+ cudaSetDevice(dv.id);+ CK(cudaStreamSynchronize(dv.st));+ }+ cudaSetDevice(d.id);+ prof_collect();+ std::fprintf(stderr,+ "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "+ "(with the waits for streamed ones), between layers %.0f ms\n",+ n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);+ for (double& v : cprof_ms) v = 0;+ prof_chunk = false;+ }+ if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {+ const double ms = now_ms() - tl;+ for (size_t di = 0; di < chunks.size(); ++di) {+ const Chunk& c = *chunks[di];+ if (!c.flow) continue;+ const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;+ const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;+ const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)+ char lcs[160] = "";+ if (c.lm)+ std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "+ "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);+ std::fprintf(stderr,+ "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "+ "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "+ "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "+ "%.0f ms for reads\n",+ devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),+ (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,+ (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,+ (unsigned long long) c.it_spec_unused, c.wait_ms);+ }+ }+ if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)+ bool tp = false;+ for (const Layer& l : L) tp = tp || l.nm > 1;+ double picked = 0;+ for (const auto& cp : chunks)+ if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;+ if (!tp) pm_learn(n, picked);+ }+ if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)+ pos += n;+ st.tokens += n;+ cache_tick(n);+ const int nr = std::min(T_last, tap_cap);+ if (!seed_drafter(pos - nr, nr, 0, err)) return false;+ if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -+ // the next feed seeds them again from its own rows, past this position)+ Check& ckp = ck[snap_slot & 1];+ ckp.pos = run.snap_pos;+ ckp.win.resize(L.size());+ ckp.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ Dev& dv = devs[(size_t) l.dev];+ cudaSetDevice(dv.id);+ const bool kept = !l.mtp && l.snap_win != nullptr;+ ckp.win[i].resize((size_t) g.win * win_rb());+ CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),+ cudaMemcpyDeviceToHost, dv.st));+ if (l.c_kv) {+ ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);+ ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);+ CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,+ ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));+ CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,+ ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));+ }+ }+ for (Dev& dv : devs) {+ cudaSetDevice(dv.id);+ CK(cudaStreamSynchronize(dv.st));+ }+ }+ return true;+ }++ // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)+ void plan_window(int64_t p0, int T, WinPlan& wp) {+ wp.par = par;+ wp.async = par || (!sync_decode && T <= async_max_t);+ wp.vl = 0;+ for (int t = 0; t < T && !imgs.empty(); ++t)+ if (in_image(p0 + t)) wp.vl |= 1u << t;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ if (l.ratio > 0) {+ const int R = l.ratio;+ int n[kWin], k[kWin], clen_max = 0;+ for (int t = 0; t < T; ++t) {+ n[t] = (int) ((p0 + t + 1) / R);+ k[t] = std::min(g.ix_topk, n[t]);+ clen_max = std::max(clen_max, n[t]);+ }+ if (l.ix_src) {+ for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];+ if (clen_max > 0 && l.cand_src)+ for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);+ for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];+ }+ for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];+ }+ if (wp.async) {+ wp.seq[il] = ++seq_issued;+ layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);+ }+ }+ }++ int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }+ // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's+ // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows+ void head_cols(int nm, int mi, int& v0, int& vn) const {+ const Tensor* tw = pk.get("head.weight");+ if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {+ vn = g.vocab / nm;+ v0 = mi * vn;+ } else+ split128(g.vocab, nm, mi, v0, vn);+ }+ // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's+ // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two+ // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather+ // false: each member keeps its share (the drafter's markov steps over the group, mk_split)+ bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {+ const int V = g.vocab, D = g.dim, nm = head_nm();+ int v0 = 0, vn = 0, Vmax = 0, z = 0;+ head_cols(nm, mi, v0, vn);+ head_cols(nm, 0, z, Vmax);+ Group& G = groups[(size_t) d.grp];+ float* out = mi == 0 ? d.logits : d.logits_part;+ const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;+ cudaSetDevice(d.id);+ // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4+ // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)+ const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head));+ for (int c = 0; c < rows; c += per)+ if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))+ return false;+ if (nm == 1 || !gather) return true;+ const unsigned seq = ++d.head_seq;+ const int slot = (int) (seq % 2);+ auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };+ auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };+ if (mi > 0) {+ if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);+ Xfer x;+ x.n = 1;+ x.src[0] = d.logits_part;+ x.dst[0] = stage(mi);+ x.bytes[0] = (size_t) rows * vn * 4;+ xfer_put(x, flag(mi), seq, d.xcnt, d.st);+ return true;+ }+ for (int m = 1; m < nm; ++m) {+ int m0 = 0, mn = 0;+ head_cols(nm, m, m0, mn);+ Xfer x;+ x.n = 1;+ x.src[0] = stage(m);+ x.dst[0] = d.logits_part;+ x.bytes[0] = (size_t) rows * mn * 4;+ xfer_get(x, flag(m), seq, d.st);+ CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,+ (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));+ }+ set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);+ return true;+ }++ // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)+ bool head_rows_dev(int di, int T, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ const int D = g.dim, H = g.hc;+ if (n_logits <= 0) return true;+ const int t0 = T - n_logits;+ hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);+ return head_share(d, d.mi, d.x, n_logits, err);+ }++ // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read+ // h_taps)+ bool tap(Dev& d, int il, int T, std::string& err) {+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (g.n_mtp == 0 || it == g.d_targets.end()) return true;+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int D = g.dim;+ cudaSetDevice(d.id);+ hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);+ CK(cudaStreamWaitEvent(d.st, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,+ cudaMemcpyDeviceToHost, d.st));+ return true;+ }++ bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {+ if (T < 1 || T > kWin) {+ err = "a window of " + std::to_string(T) + " tokens";+ return false;+ }+ if (pos + T > o.max_context) {+ err = "the context is full";+ return false;+ }+ for (int t = 0; t < T; ++t)+ if (tk[t] < 0 || tk[t] >= g.vocab) {+ err = "token " + std::to_string(tk[t]) + " out of range";+ return false;+ }+ const int64_t p0 = pos;+ for (int t = 0; t < T; ++t) {+ toks.push_back(tk[t]);+ ctoks.push_back(ctok(pos + t, tk[t]));+ }+ // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is+ // left of their reads by then is the window's Engram time+ const double te = now_ms();+ const int64_t eg_w0 = eg_wait_ns.load();+ for (int li = 0; li < pk.eg_layers; ++li) {+ EgBatch& b = *eg_win[(size_t) li];+ if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)+ eg_issue(p0, T, li, nullptr, b);+ }+ st.engram_ms += now_ms() - te;+ const int D = g.dim;+ for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)+ std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);+ plan_window(p0, T, wplan);+ prof_window = prof_on && T == 1;+ n_logits = std::min(logits_rows, T);+ const double tl = now_ms();+ if (par) {+ if (!forward_par(p0, T, err)) return false;+ } else if (!forward_serial(p0, T, err)) return false;+ st.layer_ms += now_ms() - tl;+ for (auto& b : eg_win)+ if (b->live) eg_finish(*b);+ st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;+ eg_stats();+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ const double th = now_ms();+ CK(cudaStreamSynchronize(d.st));+ st.head_ms += now_ms() - th;+ prof_collect();+ pos += T;+ st.tokens += T;+ cache_tick(T);+ if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;+ return true;+ }++ // one thread: the layers device after device, the hand-offs through pinned memory and events+ bool forward_serial(int64_t p0, int T, std::string& err) {+ const int D = g.dim, H = g.hc;+ int cur = L[0].dev;+ {+ Dev& d = devs[(size_t) cur];+ cudaSetDevice(d.id);+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));+ embed(d.emb_rows, T, d.h, D, H, d.st);+ init_pre(d.pre, T, d.st);+ }+ std::vector<float> dump;+ if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);+ for (int il = 0; il < g.nl; ++il) {+ Layer& l = L[(size_t) il];+ if (l.dev != cur) {+ Dev& a = devs[(size_t) cur];+ Dev& b = devs[(size_t) l.dev];+ cudaSetDevice(a.id);+ const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ uint8_t* bb = a.bounce;+ CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));+ CK(cudaEventRecord(a.ev_out, a.st));+ cudaSetDevice(b.id);+ CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));+ CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));+ cur = l.dev;+ }+ if (!tap(devs[(size_t) l.dev], il, T, err)) return false;+ if (!layer(l, 0, p0, T, wplan, err)) return false;+ if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers+ Dev& dd = devs[(size_t) l.dev];+ std::vector<float> hv((size_t) T * H * D);+ cudaSetDevice(dd.id);+ CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));+ CK(cudaStreamSynchronize(dd.st));+ for (int t = 0; t < T; ++t)+ std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,+ (size_t) H * D * 4);+ }+ }+ if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);+ if (cur != head_dev) {+ err = "the head is not on the last layer's GPU";+ return false;+ }+ return head_rows_dev(head_dev, T, err);+ }++ // threads: each device's thread enqueues its layers (a group's members each their share) at once, the+ // devices waiting on each other through host-mapped flags (the window's seq)+ bool forward_par(int64_t p0, int T, std::string& err) {+ wplan.win = ++win_seq;+ std::vector<std::string> errs(devs.size());+ for (size_t di = 0; di < devs.size(); ++di)+ workers[di]->submit([this, di, p0, T, &errs] {+ std::string e;+ if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;+ });+ for (auto& w : workers) w->wait();+ for (const std::string& e : errs)+ if (!e.empty()) {+ err = e;+ return false;+ }+ return true;+ }++ // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer+ Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ void* dv[4] = {d.h, d.pre, d.topk, d.cand};+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};+ const size_t nb[4] = {nh, np, nk, nc};+ Xfer x;+ x.n = 4;+ for (int i = 0; i < 4; ++i) {+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];+ x.dst[i] = put ? (void*) hv[i] : dv[i];+ x.bytes[i] = nb[i];+ }+ return x;+ }++ bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc;+ bool started = false;+ std::vector<char> got(srcs.size(), 0);+ for (int il = 0; il < g.nl; ++il) {+ Layer& L0 = L[(size_t) il];+ int mi = -1;+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == di) mi = m;+ if (mi < 0) continue;+ if (!started) { // the window's input: the embedding rows, or the previous group's hand-off+ started = true;+ if (il == 0) {+ CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, s));+ embed(d.emb_rows, T, d.h, D, H, s);+ init_pre(d.pre, T, s);+ } else {+ const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];+ xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);+ }+ }+ // the compressed KV / index keys another group's source layer wrote this window+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {+ if (sidx < 0 || got[(size_t) sidx]) continue;+ got[(size_t) sidx] = 1;+ Src& sr = srcs[(size_t) sidx];+ if (L[(size_t) sr.layer].grp == L0.grp) continue;+ const int64_t g_first = p0 / sr.ratio;+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);+ if (ng == 0) continue;+ Xfer x;+ if (sr.ckv[(size_t) di] != nullptr) {+ x.src[x.n] = sr.x_stage;+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();+ x.bytes[x.n++] = (size_t) ng * ckv_rb();+ }+ if (sr.ik[(size_t) di] != nullptr) {+ x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();+ x.bytes[x.n++] = (size_t) ng * ik_rb();+ }+ if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);+ }+ if (mi == 0 && !tap(d, il, T, err)) return false;+ if (!layer(L0, mi, p0, T, wp, err)) return false;+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;+ if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);+ }+ if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);+ return true;+ }++ // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,+ // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it+ bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,+ size_t bytes, int from, std::string& err) {+ bool any = false;+ for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);+ if (!any) return true;+ Dev& d = devs[(size_t) from];+ CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));+ CK(cudaEventRecord(ev, d.st));+ for (size_t od = 0; od < devs.size(); ++od) {+ if (copies[od] == nullptr || (int) od == from) continue;+ Dev& o2 = devs[od];+ cudaSetDevice(o2.id);+ CK(cudaStreamWaitEvent(o2.st, ev, 0));+ CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));+ }+ cudaSetDevice(d.id);+ return true;+ }++ // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;+ // the partial sums all-reduced across the group)+ bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {+ Layer& l = member(L0, mi);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int nm = L0.nm;+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;+ const ArGroup& ag = groups[(size_t) L0.grp].ar;+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;+ prof_mark(d, 0);+ // Engram+ if (l.eg >= 0) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ const size_t per = (size_t) cols * (ehd + ehd / 32);+ eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)+ uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;+ for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]+ CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,+ cudaMemcpyHostToDevice, s));+ uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;+ for (int t = 0; t < T; ++t)+ CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,+ (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, eg_q6k[(size_t) l.eg] != 0);+ if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;+ engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);+ }+ // attention+ hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);+ hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);+ hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);+ {+ x3::MvJob j[4];+ int nj = 0;+ j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);+ j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);+ if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);+ if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);+ if (!mv_n(d, j, nj, err)) return false;+ }+ Finish fq;+ fq.w = l.q_norm;+ fq.eps = g.eps;+ finish(d.qr, T, g.q_lora, g.q_lora, fq, s);+ if (NHL > 0) {+ if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;+ Finish fr;+ fr.rd = g.rd;+ fr.pos = (int) p0;+ fr.vpp = NHL;+ fr.freqs = freqs;+ finish(d.q, T * NHL, HD, HD, fr, s);+ }+ // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark+ // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of+ // that ring and to the whole block, and leaves the ring as it was+ ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) p0;+ fk.freqs = freqs;+ fk.quant = 1;+ fk.dst = d.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(d.kvr, T, HD, HD, fk, s);+ if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);+ AttnRows ar;+ for (int t = 0; t < T; ++t) {+ if (l.mtp) {+ ar.win0[t] = 0;+ ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);+ ar.nsel[t] = T;+ continue;+ }+ const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)+ ar.win0[t] = first;+ ar.nwin[t] = g.win + t - first + 1;+ ar.nsel[t] = 0;+ }+ const void* ckv = nullptr;+ int ckv_fmt = kKvFp4;+ if (l.ratio > 0) {+ Src& sr = srcs[(size_t) l.src];+ const int R = l.ratio;+ // the groups this window completes: g_first .. g_first + ng - 1+ const int64_t g_first = p0 / R; // the first group whose last position is >= p0+ const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);+ if (l.kv_src) {+ if (R == 1) {+ Finish fc;+ fc.w = l.c_norm;+ fc.eps = g.eps;+ finish(d.ckr, T, HD, HD, fc, s);+ CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));+ } else {+ if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected+ const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;+ CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));+ }+ compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);+ }+ }+ Rows lim;+ int clen_max = 0;+ for (int t = 0; t < T; ++t) {+ lim.n[t] = (int) ((p0 + t + 1) / R);+ lim.k[t] = std::min(g.ix_topk, lim.n[t]);+ clen_max = std::max(clen_max, lim.n[t]);+ }+ if (l.ix_src) {+ Src& ks = srcs[(size_t) l.ik_src];+ if (l.owns_k && ng > 0) {+ if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;+ Finish fi;+ fi.w = l.ik_norm;+ fi.eps = g.eps;+ fi.rd = g.rd;+ fi.pos = (int) (g_first * R);+ fi.pos_step = R;+ fi.freqs = freqs;+ fi.quant = 3;+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ fi.pdst_stride = (long long) ik_rb();+ finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);+ if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),+ (size_t) ng * ik_rb(), l.dev, err))+ return false;+ }+ if (clen_max > 0) {+ if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;+ Finish fi;+ fi.rd = g.rd;+ fi.pos = (int) p0;+ fi.vpp = g.ix_heads;+ fi.freqs = freqs;+ fi.quant = 3;+ finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);+ // (iw rows are 64 apart)+ gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);+ const long long ss = (long long) o.max_context + 64;+ bool restrict_ = false;+ for (int t = 0; t < T; ++t)+ restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;+ if (l.uses_cand && restrict_) {+ Rows cl; // the candidate positions of each query: its blocks x blk+ int nmax = 0;+ for (int t = 0; t < T; ++t) {+ cl.n[t] = lim.n[t];+ cl.k[t] = wp.cand[l.il][t];+ nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);+ }+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,+ (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);+ Rows tr;+ for (int t = 0; t < T; ++t) {+ tr.n[t] = wp.cand[l.il][t] * g.cand_blk;+ tr.k[t] = lim.k[t];+ }+ topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);+ } else {+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,+ lim, d.scores, ss, s);+ if (l.cand_src) {+ Rows cr;+ for (int t = 0; t < T; ++t) {+ cr.n[t] = lim.n[t];+ cr.k[t] = g.cand_topk;+ }+ cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,+ (int) cand_stride(), s);+ }+ topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);+ }+ }+ }+ if (l.kv_src && ng > 0) {+ Finish fc;+ fc.rd = g.rd;+ fc.pos = (int) (g_first * R);+ fc.pos_step = R;+ fc.freqs = freqs;+ fc.quant = 2;+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();+ fc.pdst_stride = (long long) ckv_rb();+ finish(d.latent, ng, HD, HD, fc, s);+ if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),+ (size_t) ng * ckv_rb(), l.dev, err))+ return false;+ if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers+ Xfer x;+ x.src[0] = fc.pdst;+ x.dst[0] = sr.x_stage;+ x.bytes[0] = (size_t) ng * ckv_rb();+ x.n = 1;+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ x.dst[1] = sr.x_stage + x_cap() * ckv_rb();+ x.bytes[1] = (size_t) ng * ik_rb();+ x.n = 2;+ }+ xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);+ }+ }+ for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;+ ckv = sr.ckv[(size_t) l.dev];+ }+ if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection+ ckv = d.ext + (size_t) g.win * HD;+ ckv_fmt = kKvBf16;+ iota_rows(d.topk, T, T, (int) topk_stride(), s);+ }+ if (NHL > 0) {+ sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,+ 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);+ Finish fo;+ fo.rd = g.rd;+ fo.pos = (int) p0;+ fo.vpp = NHL;+ fo.freqs = freqs;+ fo.inverse = 1;+ fo.quant = 1;+ finish(d.o, T * NHL, HD, HD, fo, s);+ x3::MvJob j[16];+ const int per = HQ / OG;+ for (int gi = 0; gi < OG; ++gi)+ j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);+ if (!mv_n(d, j, OG, err)) return false;+ if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;+ } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)+ if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);+ hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)+ prof_mark(d, 1);+ // the MoE+ hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);+ hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);+ hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);+ if (wp.async) {+ // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members+ // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the+ // router made), and waits for the answer only before the experts' sum+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);+ const unsigned sq = wp.seq[l.il];+ const int slot = (int) (sq % kRing);+ const bool lead = mi == 0;+ int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below+ for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {+ const Layer* nx = ahead_layer(L0, a);+ if (nx == nullptr) break;+ const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);+ float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;+ gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN+ // weights instead: the same hit rate)+ route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);+ na[a - 1] = m;+ }+ route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,+ d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],+ na[1]);+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;+ swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;+ prof_mark(d, 2);+ const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;+ if (T == 1 && moe_token_path) {+ x3::MoeArgs ma;+ ma.plan_ptr = d.plan_ptr;+ ma.plan_w = d.rw;+ ma.cpu_part = lead ? resps[slot].part[0] : nullptr;+ ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;+ ma.lay = lay;+ ma.k = l.k_exp;+ ma.n_embd = D;+ ma.n_ff = FF;+ ma.limit = g.lim;+ const bool mx = lay.fmt == gg::kFmtMXFP4;+ if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);+ else x3::moe_gate_up(ma, d.x, d.xg, s);+ prof_mark(d, 3);+ if (lead) wait_seq(&resps[slot].seq, sq, s);+ prof_mark(d, 4);+ if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);+ else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);+ } else { // each distinct expert read once for the window's tokens+ gf::MoeWin mw;+ mw.plan_ptr = d.plan_ptr;+ mw.T = T;+ mw.k = l.k_exp;+ mw.grp_ptr = d.grp_ptr;+ mw.grp_start = d.grp_start;+ mw.grp_ent = d.grp_ent;+ mw.n_groups = d.n_groups;+ gf::moe_win_group(mw, s);+ x3::WinArgs wa;+ wa.grp_ptr = d.grp_ptr;+ wa.grp_start = d.grp_start;+ wa.grp_ent = d.grp_ent;+ wa.n_groups = d.n_groups;+ wa.plan_ptr = d.plan_ptr;+ wa.plan_w = d.rw;+ wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;+ wa.cpu_stride = kMaxDim;+ wa.cpu_flag = lead ? resps[slot].cpu : nullptr;+ wa.wait_ptr = lead ? &resps[slot].seq : nullptr;+ wa.wait_val = sq;+ wa.x = d.x;+ wa.x_stride = (size_t) D;+ wa.h = d.xg;+ wa.part = d.ed;+ wa.sh_out = d.sh_out;+ wa.out = d.moe_out;+ wa.out_stride = (size_t) D;+ wa.lay = lay;+ wa.T = T;+ wa.k = l.k_exp;+ wa.n_embd = D;+ wa.n_ff = FF;+ wa.limit = g.lim;+ if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {+ err = "exl3 win_experts failed";+ return false;+ }+ }+ if (lead) st.experts += (int64_t) T * l.k_exp;+ if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 5);+ return true;+ }+ if (nm > 1) {+ err = "a tensor-parallel layer on the host-routed path";+ return false;+ }+ // (glog rows are 1024 apart: route reads them so)+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);+ VlBits vb = {};+ vb.w[0] = l.mtp ? 0u : wp.vl;+ route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,+ vb.w[0] != 0 ? &vb : nullptr);+ CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaEventRecord(d.ev, s));+ // the shared expert while the host looks at the picks+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;+ swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;+ const double tw = now_ms();+ CK(cudaEventSynchronize(d.ev));+ st.gpu_wait_ms += now_ms() - tw;+ // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the+ // CPU lane's entries+ Combine cm;+ Gather ga;+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];+ int nvex = 0, nrows = 0;+ std::vector<Ent> ents;+ std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)+ std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)+ for (int t = 0; t < T; ++t) {+ cm.n[t] = 0;+ cm.cpu[t] = 0;+ for (int i = 0; i < l.k_exp; ++i) {+ const int e = h_idx[t * kPicks + i];+ if (e < 0 || e >= l.n_exp) {+ err = "router picked expert " + std::to_string(e);+ return false;+ }+ ++pr[e];+ l.heat[(size_t) e] += 1.0f;+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);+ per[(size_t) e].push_back({t, i});+ }+ }+ int nslots = 0;+ for (int e = 0; e < l.n_exp; ++e) {+ const auto& pe = per[(size_t) e];+ if (pe.empty()) continue;+ if (l.slot[(size_t) e] >= 0) {+ vex[nvex] = e;+ vrow0[nvex] = nrows;+ vn[nvex] = (int) pe.size();+ for (const auto& tp : pe) {+ ga.src[nrows] = tp.first;+ const int t = tp.first;+ cm.row[t][cm.n[t]] = nrows;+ cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];+ ++cm.n[t];+ ++nrows;+ }+ ++nvex;+ } else {+ for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {+ Ent en;+ en.e = e;+ en.slot0 = nslots;+ en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);+ for (int i = 0; i < en.n; ++i) {+ en.tok[i] = pe[a + (size_t) i].first;+ cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});+ cm.cpu[en.tok[i]] = 1;+ }+ nslots += en.n;+ ents.push_back(en);+ }+ }+ }+ st.experts += (int64_t) T * l.k_exp;+ st.cpu_experts += nslots;+ if (nrows > 0) {+ gather_rows(d.x, ga, nrows, D, d.xe, s);+ x3::MvJob j[2 * kWin * kPicks];+ int nj = 0;+ for (int i = 0; i < nvex; ++i) {+ const auto& vm = l.vm[(size_t) vex[i]];+ j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);+ j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);+ }+ if (!mv_n(d, j, nj, err)) return false;+ swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);+ nj = 0;+ for (int i = 0; i < nvex; ++i)+ j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,+ vn[i], g.ff, D);+ if (!mv_n(d, j, nj, err)) return false;+ }+ if (!ents.empty()) {+ const double tc = now_ms();+ cpu_experts(l, h_x, ents);+ std::fill(h_part, h_part + (size_t) T * D, 0.0f);+ for (int sl = 0; sl < nslots; ++sl) {+ float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;+ const float wv = cpu_rows[(size_t) sl].second;+ const float* y = c_yd.data() + (size_t) sl * D;+ for (int e = 0; e < D; ++e) o[e] += wv * y[e];+ }+ st.cpu_ms += now_ms() - tc;+ CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));+ }+ combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)+ return true;+ }++ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ strata::kernels::SamplerParams sp;+ sp.temperature = temperature;+ sp.greedy = temperature <= 0.0f;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ strata::kernels::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,+ sp, d.tok, d.st);+ int t = -1;+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ return t;+ }++ bool logits(std::vector<float>& out) {+ if (logits_all) {+ out = chunk_logits;+ return true;+ }+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ out.resize((size_t) std::max(1, n_logits) * g.vocab);+ return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;+ }++ void save_profile() const {+ if (o.profile.empty()) return;+ const std::string tmp = o.profile + ".tmp";+ std::FILE* f = std::fopen(tmp.c_str(), "wb");+ if (f == nullptr) return;+ const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();+ std::fclose(f);+ // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile+ // never moved on and only .tmp files were left)+ if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());+ }++ std::vector<std::string> info_lines() const {+ std::vector<std::string> v;+ int nh = 0;+ for (const Layer& l : L) nh += l.n_exp - l.n_vram;+ char b[512];+ // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)+ std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "+ "cpu_experts=%d ram_experts=%d engine=ds4-0.1",+ (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);+ v.push_back(b);+ if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)+ std::snprintf(b, sizeof b,+ "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "+ "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "+ "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);+ v.push_back(b);+ std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),+ pagefile_hint);+ v.push_back(b);+ }+ if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor+ std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,+ pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);+ v.push_back(b);+ }+ if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)+ std::snprintf(b, sizeof b,+ "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "+ "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",+ vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,+ (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),+ (long long) (vt.device_bytes() >> 20));+ v.push_back(b);+ } else if (!vision_err.empty()) { // (the server offers no pictures and says why)+ std::string why = vision_err;+ for (char& ch : why)+ if (ch == '\n' || ch == '\r') ch = ' ';+ v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");+ v.push_back("INFO vision=off");+ }+ return v;+ }+};++Model::Model() : p_(new Impl) {}+Model::~Model() = default;+bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }+bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }+bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {+ Impl& p = *p_;+ p.pm.last = -1;+ if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here+ const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)+ const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)+ : p.feed_windows(tokens, n, logits_rows, err);+ if (ok) p.path_done(now_ms() - t0);+ return ok;+ }+ return p.feed_windows(tokens, n, logits_rows, err);+}+bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {+ Impl& p = *p_;+ p.pm.last = -1;+ const double t0 = now_ms();+ if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers+ const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)+ if (ok) p.path_done(now_ms() - t0);+ return ok;+ }+ if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;+ p.checkpoint(slot);+ const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);+ if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)+ return ok;+}+int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }+void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }+int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {+ return p_->sample(t, tp, tk, mp, seed, counter);+}+bool Model::logits(std::vector<float>& out) { return p_->logits(out); }+int Model::image_token() const { return p_->img_tok; }+int Model::dim() const { return p_->g.dim; }+bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {+ return p_->encode_image(rgb, width, height, span, err);+}+void Model::set_images(std::vector<Image> images) {+ std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });+ p_->imgs = std::move(images);+}+int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }+bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,+ float mp, uint64_t seed, uint64_t counter) {+ return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);+}+bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,+ int& accepted, int& next, std::string& err) {+ return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);+}+int64_t Model::pos() const { return p_->pos; }+int64_t Model::max_context() const { return p_->o.max_context; }+int Model::vocab() const { return p_->g.vocab; }+const std::vector<int32_t>& Model::tokens() const { return p_->toks; }+void Model::reset() { p_->reset(); }+void Model::checkpoint(int slot) { p_->checkpoint(slot); }+bool Model::restore(int64_t limit) { return p_->restore(limit); }+int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }+void Model::save_profile() const { p_->save_profile(); }+const Stats& Model::stats() const { return p_->st; }+std::string Model::describe() const { return p_->describe(); }+std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }++} // namespace strata::ds4diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppnew file mode 100644index 000000000..c6b255a5c--- /dev/null+++ b/src/dsv41/ds4_model.hpp@@ -0,0 +1,153 @@+// src/ds4/ds4_model.hpp - DeepSeek-V4.1-Flash on Maya: an EXL3 model (tools/ds4/pack.py's pack beside its files) over+// the machine's GPUs, layer-split, with the routed experts that do not fit in VRAM on the CPU lane.+//+// Windows of up to 16 tokens through every layer (a prompt in windows, a decode step a window of one): each query of a+// window sees exactly what the reference's decode path gives it (DeepSeek's inference/model.py at start_pos > 0) -+// its own 128-token window, the compressed groups completed by its position. The state a token leaves is the+// 128-token window ring of every layer, the compressors' partial groups, the compressed KV and index keys of the KV+// source layers (append-only), and the Engram n-gram history; checkpoint() / restore() keep the first two (the rest is+// truncated by position) so a conversation can go back to the end of its last prompt.+#pragma once++#include <cstddef>+#include <cstdint>+#include <functional>+#include <memory>+#include <string>+#include <vector>++namespace strata::ds4 {++struct Options {+ std::string pack; // <model>/maya-ds4+ int64_t max_context = 32768;+ std::vector<int> gpus; // empty: every visible GPU+ int64_t reserve_mib = 600; // VRAM left free on each GPU+ double vram_experts = 1.0; // the share of the expert budget to use (tests: less, so the CPU lane works)+ int cpu_threads = 0; // 0: the physical cores+ std::string profile; // expert hit counts (read to place the hottest experts in VRAM, written back)+ bool expert_cache = true; // experts move between VRAM and the CPU lane as they are used+ // the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of+ // copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold them+ int mmap_experts = -1;+ bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding+ float dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this+ // tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the+ // heads, of the output projection and of every expert's FFN, summed through host-mapped memory (a GPU unlike the+ // next stays alone); tp > 1 runs with threads (--tp 1 --threads off: one host thread, layer split only)+ int tp = 2;+ bool threads = false; // a host thread per GPU enqueues its layers (the hand-offs host-mapped)+ // prompts in chunks of this many tokens through each layer at once (threads): the GEMMs on the tensor cores, the+ // CPU lane's experts streamed to the GPUs a chunk at a time instead of computed on the host; 0: windows of 16;+ // -1: 2048, or 8192 with the experts read in place (every chunk streams them all from the files)+ int prefill_chunk = -1;+ bool vision = true; // the vision tower (the checkpoint's vision.* / aligner.*), when it has one+ bool vision_only = false; // (tests) load the vision tower alone: encode_image() and nothing else+ bool verbose = false;+};++/// A picture in the prompt: its span's rows (every position of the span carries the image token), BF16 [span][dim]+struct Image {+ int64_t start = 0;+ std::shared_ptr<const std::vector<uint16_t>> rows;+};++struct Stats {+ int64_t tokens = 0;+ int64_t experts = 0, cpu_experts = 0; // routed experts run, and how many of them on the CPU lane+ double cpu_ms = 0.0, gpu_wait_ms = 0.0; // the CPU lane's time; the host's waits for the routing+ double layer_ms = 0.0, head_ms = 0.0, engram_ms = 0.0; // the layers' host time in all; the head's wait; Engram reads+ int64_t eg_rows = 0, eg_hits = 0, eg_reads = 0; // Engram rows asked for, served without a read, read from the files+ double eg_io_ms = 0.0; // the windows' Engram reads, issue to last done (mostly hidden)+ int64_t eg_cached = 0, eg_cache_cap = 0; // the row cache's rows held, its capacity+ int64_t swaps = 0; // experts the cache moved into VRAM+ int64_t skipped = 0; // routed experts STRATA_DS4_DISK_SKIP_W left out (light, not in RAM)+ int64_t ram_experts = 0, ram_moves = 0; // CPU-lane experts read from the RAM tier; experts moved into it+ double prof_ms[6] = {}; // DS4_PROF: the GPU's time in decode by stage (between layers, attention, routing ++ int64_t prof_tokens = 0; // shared expert, experts' gate / up, the wait for the CPU lane, down + post)+ // lookahead (decode windows of one): the picks of layers guessed a layer before (the next layer's gate on this+ // layer's FFN input), of them in the guess's top k / in all its guesses / in the guess two layers before; the picks+ // read from the files, of them prefetched early; experts prefetched early+ int64_t ahead_picks = 0, ahead_hit_k = 0, ahead_hit = 0, ahead_hit2 = 0;+ int64_t ahead_cold = 0, ahead_cold_hit = 0, ahead_pf = 0;+ int64_t ahead_rank_n[16] = {}, ahead_rank_hit[16] = {}; // the next-layer guesses by rank, and how many were picked+ int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;+ // per draft position (the 1st .. 8th of a round): verified with all the drafts before it accepted, and accepted+ int64_t draft_reached[8] = {}, draft_kept[8] = {};+ double draft_ms = 0.0, verify_ms = 0.0; // the drafter's time, the verification windows' time+};++class Model {+public:+ Model();+ ~Model();+ bool load(const Options& o, std::string& err);++ /// Feeds `token` at position pos(); with logits, the next token's logits are left on the head's device.+ bool step(int token, bool logits, std::string& err);+ /// Feeds n tokens (windows of up to 16, or a long run in chunks of prefill_chunk); the last `logits_rows` of them get+ /// logits (logits_rows > 16: only a run of one chunk or window).+ bool feed(const int* tokens, int n, int logits_rows, std::string& err);+ /// feed() of a prompt's end that also keeps the state before its last token in checkpoint `slot` (a prompt's+ /// last token then rides along in the run of chunks instead of a window of its own); the same as+ /// feed(n - 1), checkpoint(slot), feed(1)+ bool feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err);+ /// The next token from the last feed's last logit row (greedy when temperature <= 0).+ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter);+ /// The last feed's logit rows on the host (rows x vocab floats).+ bool logits(std::vector<float>& out);++ /// The DSpark drafter (speculative decoding): its block size (0: not loaded), drafts for the positions after `y`+ /// (y: the token at pos(), not fed yet) with their confidences, and the verification of [y, drafts]: how many of the+ /// drafts stand and the token after them (the state then holds y and the accepted drafts). The drafts are the+ /// drafter's argmax; a sampled request passes its chain (the same temperature / top_p / top_k / min_p / seed /+ /// counter as its verify()) so that with DS4_DSPARK_SAMPLE=1 they are drawn from the drafter's distribution, as+ /// DeepSeek's reference does. verify() keeps a draft d with probability min(1, p(d) / q(d)) over the request's+ /// chain - the speculative-sampling rule, so every token is still distributed as the model's own draw.+ int draft_block() const;+ bool draft(int y, std::vector<int>& drafts, std::vector<float>& conf, std::string& err, float temperature = 0.0f,+ float top_p = 1.0f, int top_k = 0, float min_p = 0.0f, uint64_t seed = 0, uint64_t counter = 0);+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err);++ /// Tokens a feed call takes at a time to use the chunked prompt path (all of a prompt: its chunks run as a+ /// pipeline; 16 without it).+ int feed_unit() const;+ /// Called from feed() while a run of chunks goes through the layers, with the tokens done so far.+ void set_progress(std::function<void(int64_t)> f);+ int64_t pos() const;+ int64_t max_context() const;+ int vocab() const;+ int dim() const;+ const std::vector<int32_t>& tokens() const; // what the state has consumed, in order++ void reset();+ /// Keep the state at the current position in checkpoint slot 0 or 1 (the end of a prompt, the position before it);+ /// restore(limit) goes back to the latest one at or before limit (-1: none).+ void checkpoint(int slot = 0);+ bool restore(int64_t limit);+ int64_t restorable(int64_t limit) const;++ /// The vision tower: the token every position of an image's span carries (-1: no tower).+ int image_token() const;+ /// An image's pixels (RGB rows, width x height, both multiples of the patch size: the server's resize) -> the+ /// rows of its span, BF16 [n_h * (n_w + 1) + 2][dim]: image_start, n_h rows of (n_w aligner rows, image_newline),+ /// image_end.+ bool encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err);+ /// The pictures of the prompt about to be fed ({}: none): positions inside a span read its rows (and route with+ /// the image bias, and take no part in the Engram n-grams); the rest the embedding table's.+ void set_images(std::vector<Image> images);++ void save_profile() const;+ const Stats& stats() const;+ std::string describe() const;+ /// INFO / VRAM lines for the server's Monitor.+ std::vector<std::string> info_lines() const;++ struct Impl;++private:+ std::unique_ptr<Impl> p_;+};++} // namespace strata::ds4diff --git a/src/dsv41/ds4_mx_kern.inc b/src/dsv41/ds4_mx_kern.incnew file mode 100644index 000000000..24bd08213--- /dev/null+++ b/src/dsv41/ds4_mx_kern.inc@@ -0,0 +1,249 @@+// src/ds4/ds4_mx_kern.inc - the CPU lane's int8 MXFP4 rows (ds4_gguf_cpu.hpp: mx_rows), one body for the x86 tiers.+// ds4_gguf_cpu.cpp includes it once a tier, inside its unnamed namespace, after <immintrin.h>, with+// MXK_NS the tier's namespace+// MXK_ATTR the tier's instruction set as a function attribute (GCC / Clang: target(...); MSVC: nothing - it takes+// any intrinsic) on every function here, so the file is built for the baseline and each tier's code runs+// only where ds4_gguf_cpu.cpp's CPU check says it can+// MXK_256 or MXK_512, and MXK_VNNI (vpdpbusd; else vpmaddubsw + vpmaddwd)+// and gets MXK_NS::rows(...) (and with MXK_256 and no MXK_VNNI, MXK_NS::quant(...)); the macros are undefined at the end.+//+// The arithmetic is the device's (ds4_gguf.cu, mx_dot) - a block's 32 codes times the int8 activations summed exactly+// in integers, times (weight scale x activation scale) into F32 sums - in ONE order for every tier, the scalar one+// included (ds4_gguf_cpu.cpp's rows_q_scalar), so every tier gives the same bits and the choice of tier never changes+// an answer:+// * the blocks go in groups of 8; block j of a group adds into accumulator L(j), L = {0, 4, 1, 5, 2, 6, 3, 7} (the+// order a 256-bit hadd tree leaves the 8 blocks' totals in: lanes = blocks 0, 2, 4, 6, 1, 3, 5, 7):+// acc[L(j)] = fma(float(total_j), dw_j * dx_j, acc[L(j)]), the groups in turn;+// * the row is ((acc0 + acc4) + (acc2 + acc6)) + ((acc1 + acc5) + (acc3 + acc7)) (hsum256's tree).+// The codes come out of a shuffle table as w + 12 (0..24, unsigned bytes), so the products go through vpmaddubsw /+// vpdpbusd as u8 x s8; each block's total then gets the activation's -12 x (sum of its 32 values) back, exactly.+//+// The activation of a call (mx_act_bytes(k): k % 256 == 0 here; quant() / ds4_gguf_cpu.cpp's quant_scalar):+// [0, k) int8 values in quads of 4 blocks, quad g at 128 g: the 4 blocks' values 0..15 (block j at+// +16 j), then their 16..31 - one 256-bit step takes a pair of blocks, one 512-bit step a quad+// [k, k + k/8) F32 scale of each block, group G (8 blocks) at 8 G in lane order (blocks 0, 2, 4, 6, 1, 3, 5, 7)+// [k + k/8, + k/8) int32 -12 x (sum of the block's 32 values), the same order+namespace MXK_NS {++constexpr int kBlockBytes = 17; // E8M0 scale + 16 code bytes++MXK_ATTR inline float hsum256(__m256 v) {+ const __m128 a = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps(v, 1));+ const __m128 b = _mm_add_ps(a, _mm_movehl_ps(a, a));+ return _mm_cvtss_f32(_mm_add_ss(b, _mm_shuffle_ps(b, b, 1)));+}+// the 8 weight scales of a group (its first block at p), in lane order+MXK_ATTR inline __m256 group_dw(const uint8_t* p, const float* e8) {+ return _mm256_setr_ps(e8[p[0]], e8[p[2 * kBlockBytes]], e8[p[4 * kBlockBytes]], e8[p[6 * kBlockBytes]],+ e8[p[kBlockBytes]], e8[p[3 * kBlockBytes]], e8[p[5 * kBlockBytes]], e8[p[7 * kBlockBytes]]);+}++#if defined(MXK_256)+// ---------------------------------------------------------------- 256-bit: a pair of blocks a step+// a pair's codes (a1: values 0..15 of both blocks, a2: 16..31) times x1 / x2: 8 int32 lanes, block A's 4 then B's 4+MXK_ATTR inline __m256i pair_dot(__m256i a1, __m256i a2, __m256i x1, __m256i x2) {+#if defined(MXK_VNNI)+ return _mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), a1, x1), a2, x2);+#else+ const __m256i p = _mm256_add_epi16(_mm256_maddubs_epi16(a1, x1), _mm256_maddubs_epi16(a2, x2));+ return _mm256_madd_epi16(p, _mm256_set1_epi16(1));+#endif+}++template <int N>+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,+ const float* e8, int pf) {+ const __m256i lut = _mm256_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0, //+ 12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0);+ const __m256i m4 = _mm256_set1_epi8(0x0f);+ const int ng = k / 256;+ const int8_t* xq[N];+ const float* xd[N];+ const int32_t* xc[N];+ for (int i = 0; i < N; ++i) {+ const uint8_t* a = (const uint8_t*) act[i];+ xq[i] = (const int8_t*) a;+ xd[i] = (const float*) (a + k);+ xc[i] = (const int32_t*) (a + k + k / 8);+ }+ for (int r = r0; r < r1; ++r) {+ const uint8_t* p = w + (size_t) r * rb;+ __m256 acc[N];+ for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();+ for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {+ if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);+ const __m256 dw = group_dw(p, e8);+ // the group's 4 pairs of blocks (2 q, 2 q + 1) decoded once; then each activation in turn: its 4 pair dots,+ // the hadd tree to the 8 blocks' totals (lanes: blocks 0 2 4 6 | 1 3 5 7), the FMA - only the accumulators+ // stay live from one activation to the next (16 YMM registers on AVX2)+ __m256i a1[4], a2[4];+ for (int q = 0; q < 4; ++q) {+ const uint8_t* pb = p + 2 * q * kBlockBytes;+ const __m256i c = _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_loadu_si128((const __m128i*) (pb + 1))),+ _mm_loadu_si128((const __m128i*) (pb + 18)), 1);+ a1[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(c, m4));+ a2[q] = _mm256_shuffle_epi8(lut, _mm256_and_si256(_mm256_srli_epi16(c, 4), m4));+ }+ const int o = G * 256; // (quads 2 G and 2 G + 1: pair q at (q / 2) * 128 + (q % 2) * 32)+ for (int i = 0; i < N; ++i) {+ const int8_t* x = xq[i] + o;+ const __m256i s0 = pair_dot(a1[0], a2[0], _mm256_loadu_si256((const __m256i*) x),+ _mm256_loadu_si256((const __m256i*) (x + 64)));+ const __m256i s1 = pair_dot(a1[1], a2[1], _mm256_loadu_si256((const __m256i*) (x + 32)),+ _mm256_loadu_si256((const __m256i*) (x + 96)));+ const __m256i s2 = pair_dot(a1[2], a2[2], _mm256_loadu_si256((const __m256i*) (x + 128)),+ _mm256_loadu_si256((const __m256i*) (x + 192)));+ const __m256i s3 = pair_dot(a1[3], a2[3], _mm256_loadu_si256((const __m256i*) (x + 160)),+ _mm256_loadu_si256((const __m256i*) (x + 224)));+ const __m256i u = _mm256_hadd_epi32(_mm256_hadd_epi32(s0, s1), _mm256_hadd_epi32(s2, s3));+ const __m256i tot = _mm256_add_epi32(u, _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));+ acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);+ }+ }+ for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);+ }+}++#if !defined(MXK_VNNI)+// blocks [b0, b1) of x (x at block b0's first value) into act, the bytes of quant_scalar (ds4_gguf_cpu.cpp): the same+// IEEE division and round-half-even conversion, the same max+MXK_ATTR inline void quant(const float* x, int b0, int b1, uint8_t* act, int k) {+ int8_t* q = (int8_t*) act;+ float* dx = (float*) (act + k);+ int32_t* corr = (int32_t*) (act + k + k / 8);+ const __m256 sign = _mm256_set1_ps(-0.0f);+ const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);+ for (int b = b0; b < b1; ++b, x += 32) {+ const __m256 v0 = _mm256_loadu_ps(x), v1 = _mm256_loadu_ps(x + 8);+ const __m256 v2 = _mm256_loadu_ps(x + 16), v3 = _mm256_loadu_ps(x + 24);+ const __m256 m = _mm256_max_ps(_mm256_max_ps(_mm256_andnot_ps(sign, v0), _mm256_andnot_ps(sign, v1)),+ _mm256_max_ps(_mm256_andnot_ps(sign, v2), _mm256_andnot_ps(sign, v3)));+ __m128 m2 = _mm_max_ps(_mm256_castps256_ps128(m), _mm256_extractf128_ps(m, 1));+ m2 = _mm_max_ps(m2, _mm_movehl_ps(m2, m2));+ m2 = _mm_max_ss(m2, _mm_shuffle_ps(m2, m2, 1));+ const float am = _mm_cvtss_f32(m2);+ const float sc = am / 127.0f;+ int8_t* lo = q + (b >> 2) * 128 + 16 * (b & 3);+ int8_t* hi = lo + 64;+ int s = 0;+ if (am > 0.0f) {+ const __m256 vs = _mm256_set1_ps(sc);+ const __m256i q0 = _mm256_cvtps_epi32(_mm256_div_ps(v0, vs)), q1 = _mm256_cvtps_epi32(_mm256_div_ps(v1, vs));+ const __m256i q2 = _mm256_cvtps_epi32(_mm256_div_ps(v2, vs)), q3 = _mm256_cvtps_epi32(_mm256_div_ps(v3, vs));+ const __m256i pk = _mm256_permutevar8x32_epi32(+ _mm256_packs_epi16(_mm256_packs_epi32(q0, q1), _mm256_packs_epi32(q2, q3)), order);+ _mm_storeu_si128((__m128i*) lo, _mm256_castsi256_si128(pk));+ _mm_storeu_si128((__m128i*) hi, _mm256_extracti128_si256(pk, 1));+ __m256i t = _mm256_add_epi32(_mm256_add_epi32(q0, q1), _mm256_add_epi32(q2, q3));+ __m128i t2 = _mm_add_epi32(_mm256_castsi256_si128(t), _mm256_extracti128_si256(t, 1));+ t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0x4E));+ t2 = _mm_add_epi32(t2, _mm_shuffle_epi32(t2, 0xB1));+ s = _mm_cvtsi128_si32(t2);+ } else {+ _mm_storeu_si128((__m128i*) lo, _mm_setzero_si128());+ _mm_storeu_si128((__m128i*) hi, _mm_setzero_si128());+ }+ const int j = b & 7, lane = 8 * (b >> 3) + (j & 1) * 4 + (j >> 1);+ dx[lane] = sc;+ corr[lane] = -12 * s;+ }+}+#endif+#endif // MXK_256++#if defined(MXK_512)+// ---------------------------------------------------------------- 512-bit: a quad of blocks a step+MXK_ATTR inline __m512i quad_dot(__m512i a1, __m512i a2, __m512i x1, __m512i x2) {+#if defined(MXK_VNNI)+ return _mm512_dpbusd_epi32(_mm512_dpbusd_epi32(_mm512_setzero_si512(), a1, x1), a2, x2);+#else+ const __m512i p = _mm512_add_epi16(_mm512_maddubs_epi16(a1, x1), _mm512_maddubs_epi16(a2, x2));+ return _mm512_madd_epi16(p, _mm512_set1_epi16(1));+#endif+}++template <int N>+MXK_ATTR inline void rows_n(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, float* const* y,+ const float* e8, int pf) {+ const __m512i lut = _mm512_broadcast_i32x4(_mm_setr_epi8(12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0));+ const __m512i m4 = _mm512_set1_epi8(0x0f);+ // a quad's 16 lanes after adjacent-pair sums: block j's halves in lanes 4 j and 4 j + 2; the group's two quads ->+ // the 8 blocks in lane order (0 2 4 6 1 3 5 7), first halves (pa) and second halves (pb)+ const __m512i pa = _mm512_setr_epi32(0, 8, 16, 24, 4, 12, 20, 28, 0, 0, 0, 0, 0, 0, 0, 0);+ const __m512i pb = _mm512_setr_epi32(2, 10, 18, 26, 6, 14, 22, 30, 0, 0, 0, 0, 0, 0, 0, 0);+ const int ng = k / 256;+ const int8_t* xq[N];+ const float* xd[N];+ const int32_t* xc[N];+ for (int i = 0; i < N; ++i) {+ const uint8_t* a = (const uint8_t*) act[i];+ xq[i] = (const int8_t*) a;+ xd[i] = (const float*) (a + k);+ xc[i] = (const int32_t*) (a + k + k / 8);+ }+ for (int r = r0; r < r1; ++r) {+ const uint8_t* p = w + (size_t) r * rb;+ __m256 acc[N];+ for (int i = 0; i < N; ++i) acc[i] = _mm256_setzero_ps();+ for (int G = 0; G < ng; ++G, p += 8 * kBlockBytes) {+ if (pf > 0) _mm_prefetch((const char*) p + pf, _MM_HINT_T0);+ const __m256 dw = group_dw(p, e8);+ __m512i h0[N];+ for (int qd = 0; qd < 2; ++qd) { // the group's quads: blocks 4 qd .. 4 qd + 3+ const uint8_t* pq = p + 4 * qd * kBlockBytes;+ __m512i c = _mm512_castsi128_si512(_mm_loadu_si128((const __m128i*) (pq + 1)));+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 18)), 1);+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 35)), 2);+ c = _mm512_inserti32x4(c, _mm_loadu_si128((const __m128i*) (pq + 52)), 3);+ const __m512i a1 = _mm512_shuffle_epi8(lut, _mm512_and_si512(c, m4));+ const __m512i a2 = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(c, 4), m4));+ const int o = (2 * G + qd) * 128;+ for (int i = 0; i < N; ++i) {+ const __m512i s = quad_dot(a1, a2, _mm512_loadu_si512((const void*) (xq[i] + o)),+ _mm512_loadu_si512((const void*) (xq[i] + o + 64)));+ const __m512i h = _mm512_add_epi32(s, _mm512_shuffle_epi32(s, (_MM_PERM_ENUM) 0xB1));+ if (qd == 0) {+ h0[i] = h;+ continue;+ }+ const __m512i ta = _mm512_permutex2var_epi32(h0[i], pa, h), tb = _mm512_permutex2var_epi32(h0[i], pb, h);+ const __m256i tot = _mm256_add_epi32(+ _mm256_add_epi32(_mm512_castsi512_si256(ta), _mm512_castsi512_si256(tb)),+ _mm256_loadu_si256((const __m256i*) (xc[i] + 8 * G)));+ acc[i] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(tot), _mm256_mul_ps(dw, _mm256_loadu_ps(xd[i] + 8 * G)), acc[i]);+ }+ }+ }+ for (int i = 0; i < N; ++i) y[i][r - r0] = hsum256(acc[i]);+ }+}+#endif // MXK_512++// y[i][r - r0] for every i < n, in calls of up to 8 activations; e8 the 256 E8M0 scales as F32 (2^(e - 128)), pf the+// bytes to prefetch ahead in a row (0: none)+MXK_ATTR inline void rows(const uint8_t* w, size_t rb, int k, int r0, int r1, const void* const* act, int n,+ float* const* y, const float* e8, int pf) {+ for (int i0 = 0; i0 < n; i0 += 8) {+ float* yy[8];+ const int c = n - i0 < 8 ? n - i0 : 8;+ for (int i = 0; i < c; ++i) yy[i] = y[i0 + i];+ switch (c) {+ case 1: rows_n<1>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 2: rows_n<2>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 3: rows_n<3>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 4: rows_n<4>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 5: rows_n<5>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 6: rows_n<6>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ case 7: rows_n<7>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ default: rows_n<8>(w, rb, k, r0, r1, act + i0, yy, e8, pf); break;+ }+ }+}++} // namespace MXK_NS++#undef MXK_NS+#undef MXK_ATTR+#undef MXK_256+#undef MXK_512+#undef MXK_VNNIdiff --git a/src/dsv41/ds4_numerics.cuh b/src/dsv41/ds4_numerics.cuhnew file mode 100644index 000000000..36b1f6849--- /dev/null+++ b/src/dsv41/ds4_numerics.cuh@@ -0,0 +1,63 @@+// src/ds4/ds4_numerics.cuh - the BF16 and FP8 (E4M3) conversions of maya-ds4's kernels, on CUDA through its+// intrinsics (cuda_bf16.h / cuda_fp8.h) and on HIP in plain arithmetic that gives the same bits: BF16 round to+// nearest even; E4M3 (OCP "fn": no infinities, max 448) decoded exactly, rounded to nearest even and saturated.+#pragma once++#include <cstdint>+#include <cuda_runtime.h>+#if !defined(STRATA_USE_HIP)+#include <cuda_bf16.h>+#include <cuda_fp16.h>+#include <cuda_fp8.h>+#endif++namespace strata::ds4 {++/// F32 -> BF16 bits, round to nearest even (a NaN stays a NaN).+__device__ __forceinline__ uint16_t bf16_rn(float f) {+#if !defined(STRATA_USE_HIP)+ return __bfloat16_as_ushort(__float2bfloat16_rn(f));+#else+ const uint32_t u = __float_as_uint(f);+ if ((u & 0x7fffffffu) > 0x7f800000u) return (uint16_t) ((u >> 16) | 0x40u);+ return (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);+#endif+}++/// An E4M3 code's value (exact in F32).+__device__ __forceinline__ float e4m3_value(uint8_t q) {+#if !defined(STRATA_USE_HIP)+ const __half_raw h = __nv_cvt_fp8_to_halfraw((__nv_fp8_storage_t) q, __NV_E4M3);+ return __half2float(__half(h));+#else+ const uint32_t e = (q >> 3) & 15u, m = q & 7u;+ float v;+ if (e == 15u && m == 7u) v = __uint_as_float(0x7fc00000u); // NaN+ else if (e == 0u) v = (float) m * 0.001953125f; // m 2^-9+ else v = __uint_as_float(((e + 120u) << 23) | (m << 20)); // (1 + m / 8) 2^(e - 7)+ return (q & 0x80u) ? -v : v;+#endif+}++/// v rounded to the nearest E4M3 value (ties to even), saturated at +-448.+__device__ __forceinline__ float e4m3_round(float v) {+#if !defined(STRATA_USE_HIP)+ const __nv_fp8_storage_t q = __nv_cvt_float_to_fp8(v, __NV_SATFINITE, __NV_E4M3);+ const __half_raw h = __nv_cvt_fp8_to_halfraw(q, __NV_E4M3);+ return __half2float(__half(h));+#else+ const float a = fabsf(v);+ if (!(a == a)) return v;+ float r;+ if (a >= 448.0f) r = 448.0f;+ else {+ const int e = (int) ((__float_as_uint(a) >> 23) & 0xffu) - 127; // a's binade (a > 0)+ const float step = e < -6 ? 0.001953125f : __uint_as_float((uint32_t) (e - 3 + 127) << 23);+ r = a == 0.0f ? 0.0f : rintf(a / step) * step; // (a power of two: exact)+ r = fminf(r, 448.0f);+ }+ return v < 0.0f ? -r : r;+#endif+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppnew file mode 100644index 000000000..2b61860c0--- /dev/null+++ b/src/dsv41/ds4_os.hpp@@ -0,0 +1,436 @@+// src/ds4/ds4_os.hpp - the few OS services maya-ds4 needs, on Linux and on Windows: files read at an offset and+// mapped read-only (the experts read in place), page-cache hints, anonymous host memory on a NUMA node, thread pinning,+// each NUMA node's physical cores, the RAM this process can still get. (On Windows as src/core/expert_source.cpp+// does it for GLM: no FILE_FLAG_RANDOM_ACCESS - the page cache must keep what it read.)+#pragma once++#include <algorithm>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <sstream>+#include <string>+#include <thread>+#include <vector>++#if defined(_WIN32)+#define WIN32_LEAN_AND_MEAN+#ifndef NOMINMAX+#define NOMINMAX+#endif+#include <windows.h>+#include <psapi.h> // (K32QueryWorkingSetEx: kernel32's, no psapi.lib)+#else+#include <fcntl.h>+#include <pthread.h>+#include <sched.h>+#include <sys/mman.h>+#include <sys/stat.h>+#include <sys/syscall.h>+#include <unistd.h>+#endif++namespace strata::ds4::os {++using fd_t = intptr_t;+constexpr fd_t kBadFd = -1;++#if defined(_WIN32)+inline std::wstring wide(const std::string& s) { // UTF-8 -> UTF-16 (a path with non-ASCII characters)+ const int n = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, nullptr, 0);+ std::wstring w((size_t) (n > 0 ? n : 1), L'\0');+ if (n > 0) MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, &w[0], n);+ return w;+}+#endif++/// A file opened for reading (kBadFd: it could not be).+inline fd_t open_ro(const std::string& path) {+#if defined(_WIN32)+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,+ FILE_ATTRIBUTE_NORMAL, nullptr);+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;+#else+ const int fd = ::open(path.c_str(), O_RDONLY);+ return fd < 0 ? kBadFd : (fd_t) fd;+#endif+}+/// The same file opened for unbuffered reads (Linux: O_DIRECT - no page cache, the disk straight into the buffer;+/// kBadFd where the filesystem refuses it, or on Windows, whose unbuffered reads of a mapped file take a slow path)+inline fd_t open_direct(const std::string& path) {+#if defined(_WIN32)+ (void) path;+ return kBadFd;+#else+ const int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);+ return fd < 0 ? kBadFd : (fd_t) fd;+#endif+}+/// [off, off + n) through an unbuffered descriptor into dst, a kDirectAlign-aligned buffer of at least+/// direct_span(off, n) bytes: the file read from off rounded down, so the bytes land at dst + (off % kDirectAlign).+/// False if a read fails (the caller reads buffered instead).+constexpr uint64_t kDirectAlign = 4096;+inline uint64_t direct_span(uint64_t off, uint64_t n) {+ const uint64_t a0 = off & ~(kDirectAlign - 1), a1 = (off + n + kDirectAlign - 1) & ~(kDirectAlign - 1);+ return a1 - a0;+}+inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {+#if defined(_WIN32)+ (void) f, (void) dst, (void) n, (void) off;+ return false;+#else+ const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;+ uint8_t* p = (uint8_t*) dst;+ uint64_t got = 0;+ while (got < need) {+ const ssize_t r = ::pread((int) f, p + got, (size_t) std::min<uint64_t>(need - got, 1ull << 30), (off_t) (a0 + got));+ if (r < 0) return false;+ if (r == 0) break; // (the file's end: the last aligned read is short)+ got += (uint64_t) r;+ if ((uint64_t) r % kDirectAlign != 0) break;+ }+ return got >= want;+#endif+}+/// The same file opened for reads from many threads at once (Windows runs the reads of a synchronous handle one at a+/// time, so an overlapped one; Linux: a plain descriptor, pread is concurrent anyway)+inline fd_t open_par(const std::string& path) {+#if defined(_WIN32)+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,+ FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, nullptr);+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;+#else+ const int fd = ::open(path.c_str(), O_RDONLY);+ return fd < 0 ? kBadFd : (fd_t) fd;+#endif+}+/// n bytes at off through open_par's handle (any thread, concurrently)+inline bool pread_par(fd_t f, void* dst, uint64_t n, uint64_t off) {+#if defined(_WIN32)+ struct Ev { // (each thread its own, closed when it ends)+ HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);+ ~Ev() {+ if (h != nullptr) CloseHandle(h);+ }+ };+ static thread_local Ev ev;+ if (ev.h == nullptr) return false;+ uint8_t* p = (uint8_t*) dst;+ while (n > 0) {+ const DWORD want = (DWORD) std::min<uint64_t>(n, 1ull << 30);+ OVERLAPPED ov{};+ ov.Offset = (DWORD) (off & 0xffffffffu);+ ov.OffsetHigh = (DWORD) (off >> 32);+ ov.hEvent = ev.h;+ ResetEvent(ev.h);+ DWORD got = 0;+ if (!ReadFile((HANDLE) f, p, want, nullptr, &ov) && GetLastError() != ERROR_IO_PENDING) return false;+ if (!GetOverlappedResult((HANDLE) f, &ov, &got, TRUE) || got == 0) return false;+ p += got, off += got, n -= got;+ }+ return true;+#else+ uint8_t* p = (uint8_t*) dst;+ while (n > 0) {+ const ssize_t r = ::pread((int) f, p, (size_t) std::min<uint64_t>(n, 1ull << 30), (off_t) off);+ if (r <= 0) return false;+ p += r, off += (uint64_t) r, n -= (uint64_t) r;+ }+ return true;+#endif+}+/// `from` renamed to `to`, replacing `to` if it exists (rename(2) does; Windows' rename fails on an existing file, so+/// MoveFileExW with MOVEFILE_REPLACE_EXISTING)+inline bool replace_file(const std::string& from, const std::string& to) {+#if defined(_WIN32)+ return MoveFileExW(wide(from).c_str(), wide(to).c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0;+#else+ return std::rename(from.c_str(), to.c_str()) == 0;+#endif+}+inline void close_fd(fd_t f) {+ if (f == kBadFd) return;+#if defined(_WIN32)+ CloseHandle((HANDLE) f);+#else+ ::close((int) f);+#endif+}+inline uint64_t file_size(fd_t f) {+#if defined(_WIN32)+ LARGE_INTEGER sz{};+ return GetFileSizeEx((HANDLE) f, &sz) ? (uint64_t) sz.QuadPart : 0;+#else+ struct stat sb;+ return ::fstat((int) f, &sb) == 0 && sb.st_size > 0 ? (uint64_t) sb.st_size : 0;+#endif+}+/// n bytes at offset off into dst (any thread, concurrently); false if the file ends first or a read fails.+inline bool pread_full(fd_t f, void* dst, uint64_t n, uint64_t off) {+ uint8_t* p = (uint8_t*) dst;+ while (n > 0) {+ const uint64_t want = std::min<uint64_t>(n, 1ull << 30);+#if defined(_WIN32)+ OVERLAPPED ov{};+ ov.Offset = (DWORD) (off & 0xffffffffu);+ ov.OffsetHigh = (DWORD) (off >> 32);+ DWORD got = 0;+ if (!ReadFile((HANDLE) f, p, (DWORD) want, &got, &ov) || got == 0) return false;+ const uint64_t r = got;+#else+ const ssize_t rr = ::pread((int) f, p, (size_t) want, (off_t) off);+ if (rr <= 0) return false;+ const uint64_t r = (uint64_t) rr;+#endif+ p += r, off += r, n -= r;+ }+ return true;+}+/// The whole file mapped read-only (nullptr: it could not be).+inline void* map_ro(fd_t f, uint64_t bytes) {+ if (bytes == 0) return nullptr;+#if defined(_WIN32)+ HANDLE m = CreateFileMappingW((HANDLE) f, nullptr, PAGE_READONLY, 0, 0, nullptr);+ if (m == nullptr) return nullptr;+ void* v = MapViewOfFile(m, FILE_MAP_READ, 0, 0, 0);+ CloseHandle(m); // (the view keeps the mapping)+ return v;+#else+ void* p = ::mmap(nullptr, (size_t) bytes, PROT_READ, MAP_SHARED, (int) f, 0);+ return p == MAP_FAILED ? nullptr : p;+#endif+}+inline void unmap(void* p, uint64_t bytes) {+ if (p == nullptr) return;+#if defined(_WIN32)+ (void) bytes;+ UnmapViewOfFile(p);+#else+ ::munmap(p, (size_t) bytes);+#endif+}+/// A hint that [off, off + n) of the file is needed soon (need) or not again soon (!need); nothing where the OS has+/// no such hint for a file (Windows' page cache reads ahead on its own).+inline void advise(fd_t f, uint64_t off, uint64_t n, bool need) {+#if defined(_WIN32)+ (void) f, (void) off, (void) n, (void) need;+#else+ posix_fadvise((int) f, (off_t) off, (off_t) n, need ? POSIX_FADV_WILLNEED : POSIX_FADV_DONTNEED);+#endif+}++/// Ranges of a file's mapping about to be read: the OS reads them in now, with large requests side by side, instead of+/// a page fault at a time later (Windows: PrefetchVirtualMemory - cold expert-sized pieces came in 5x faster on an NVMe;+/// Linux: madvise WILLNEED, the kernel's read-ahead of the whole range). Best effort: a failure changes nothing.+inline void prefetch(const std::vector<std::pair<const void*, size_t>>& ranges) {+ if (ranges.empty()) return;+#if defined(_WIN32)+ std::vector<WIN32_MEMORY_RANGE_ENTRY> r(ranges.size());+ for (size_t i = 0; i < ranges.size(); ++i) {+ r[i].VirtualAddress = const_cast<void*>(ranges[i].first);+ r[i].NumberOfBytes = ranges[i].second;+ }+ PrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) r.size(), r.data(), 0);+#else+ const uintptr_t pg = 4096;+ for (const auto& x : ranges) {+ const uintptr_t a = (uintptr_t) x.first, b = a & ~(pg - 1);+ madvise((void*) b, x.second + (a - b), MADV_WILLNEED);+ }+#endif+}++/// Whether every page of [p, p + n) of a file's mapping is in the page cache (Linux mincore; elsewhere false: the+/// caller reads buffered anyway, which the cache serves)+inline bool resident(const void* p, size_t n) {+#if defined(_WIN32)+ (void) p, (void) n;+ return false;+#else+ const uintptr_t pg = 4096, a = (uintptr_t) p, b = a & ~(pg - 1);+ const size_t len = n + (a - b), np = (len + pg - 1) / pg;+ thread_local std::vector<unsigned char> v;+ if (v.size() < np) v.resize(np);+ if (mincore((void*) b, len, v.data()) != 0) return false;+ for (size_t i = 0; i < np; ++i)+ if ((v[i] & 1) == 0) return false;+ return true;+#endif+}++/// Whether a range of a file's mapping looks resident, so that prefetch() would only cost CPU time (it walks every page:+/// ~50 ns a 4 KiB page on Windows - 0.34 ms for a 6.3 MB expert matrix on a Strix Halo): `samples` pages spread over+/// it all resident (Linux: mincore, the page cache; Windows: QueryWorkingSetEx, the process's working set - a page on+/// the standby list counts as not resident, and prefetch() is what brings those back cheaply). An expert's pages are+/// read together, so they come and go together: a wrong "yes" costs page faults, a wrong "no" a prefetch.+inline bool resident_sampled(const void* p, size_t bytes, int samples = 8) {+ if (bytes == 0) return true;+ const uintptr_t pg = 4096, a = (uintptr_t) p, span = bytes - 1;+ samples = std::max(1, std::min(samples, 16));+#if defined(_WIN32)+ PSAPI_WORKING_SET_EX_INFORMATION w[16] = {};+ for (int i = 0; i < samples; ++i)+ w[i].VirtualAddress = (PVOID) ((a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1));+ if (!K32QueryWorkingSetEx(GetCurrentProcess(), w, (DWORD) (samples * sizeof w[0]))) return false;+ for (int i = 0; i < samples; ++i)+ if (!w[i].VirtualAttributes.Valid) return false;+ return true;+#else+ for (int i = 0; i < samples; ++i) {+ unsigned char v = 0;+ const uintptr_t at = (a + (samples > 1 ? span / (samples - 1) * i : 0)) & ~(pg - 1);+ if (mincore((void*) at, pg, &v) != 0 || !(v & 1)) return false;+ }+ return true;+#endif+}+/// prefetch() of the ranges that do not look resident (resident_sampled) - on Windows, where PrefetchVirtualMemory walks+/// every page of a resident range (0.34 ms a 6.3 MB expert matrix on a Strix Halo); Linux's madvise(WILLNEED) on cached+/// pages costs ~1 us a matrix (a Core i5-12600T), less than the residency check (2.5 us): every range there+inline void prefetch_cold(const std::vector<std::pair<const void*, size_t>>& ranges) {+#if defined(_WIN32)+ static const bool all = [] { // (DS4_PF_ALL=1: every range, as before)+ const char* v = std::getenv("DS4_PF_ALL");+ return v != nullptr && std::atoi(v) != 0;+ }();+ if (all) return prefetch(ranges);+ std::vector<std::pair<const void*, size_t>> cold;+ for (const auto& x : ranges)+ if (!resident_sampled(x.first, x.second)) cold.push_back(x);+ prefetch(cold);+#else+ prefetch(ranges);+#endif+}++/// Whether [p, p + n) of a file's mapping is in RAM now: every page (Linux, one mincore), sampled pages of the working+/// set (Windows; a page on the standby list counts as not in RAM)+inline bool in_ram(const void* p, size_t n) {+#if defined(_WIN32)+ return resident_sampled(p, n, 8);+#else+ return resident(p, n);+#endif+}++/// Anonymous memory on NUMA node `node` of `nodes` (node < 0 or nodes < 2: wherever the first touch puts it), on huge+/// pages where the OS gives them; nullptr if it cannot be had.+inline void* anon_alloc(size_t bytes, int node, int nodes) {+#if defined(_WIN32)+ if (node >= 0 && nodes > 1)+ return VirtualAllocExNuma(GetCurrentProcess(), nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE,+ (DWORD) node);+ return VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);+#else+ void* m = ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);+ if (m == MAP_FAILED) return nullptr;+ madvise(m, bytes, MADV_HUGEPAGE);+ if (node >= 0 && nodes > 1) {+ unsigned long mask[4] = {0, 0, 0, 0};+ mask[node / 64] = 1ul << (node % 64);+ syscall(SYS_mbind, m, bytes, 2 /* MPOL_BIND */, mask, 256, 0);+ }+ return m;+#endif+}+inline void anon_free(void* p, size_t bytes) {+ if (p == nullptr) return;+#if defined(_WIN32)+ (void) bytes;+ VirtualFree(p, 0, MEM_RELEASE);+#else+ ::munmap(p, bytes);+#endif+}++/// The calling thread, or `t`, on logical CPU `cpu` (Windows: the first 64 CPUs, processor group 0).+inline void pin_self(int cpu) {+#if defined(_WIN32)+ if (cpu >= 0 && cpu < 64) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR) 1 << cpu);+#else+ cpu_set_t set;+ CPU_ZERO(&set);+ CPU_SET(cpu, &set);+ sched_setaffinity(0, sizeof set, &set);+#endif+}+inline void pin_thread(std::thread& t, int cpu) {+#if defined(_WIN32)+ if (cpu >= 0 && cpu < 64) SetThreadAffinityMask((HANDLE) t.native_handle(), (DWORD_PTR) 1 << cpu);+#else+ cpu_set_t set;+ CPU_ZERO(&set);+ CPU_SET(cpu, &set);+ pthread_setaffinity_np(t.native_handle(), sizeof(set), &set);+#endif+}++#if defined(_WIN32)+/// Each NUMA node's physical cores (the first logical CPU of each), processor group 0 - Windows' answer to+/// /sys/devices/system/node; empty if the OS does not say.+inline std::vector<std::vector<int>> node_cores() {+ std::vector<std::vector<int>> nodes;+ auto query = [](LOGICAL_PROCESSOR_RELATIONSHIP rel, std::vector<uint8_t>& buf) {+ DWORD len = 0;+ GetLogicalProcessorInformationEx(rel, nullptr, &len);+ buf.resize(len);+ return len > 0 && GetLogicalProcessorInformationEx(rel, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len);+ };+ std::vector<uint8_t> nb, cb;+ if (!query(RelationNumaNode, nb) || !query(RelationProcessorCore, cb)) return nodes;+ std::vector<KAFFINITY> masks;+ for (size_t o = 0; o < nb.size();) {+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (nb.data() + o);+ if (e->NumaNode.GroupMask.Group == 0) masks.push_back(e->NumaNode.GroupMask.Mask);+ o += e->Size;+ }+ nodes.resize(masks.size());+ for (size_t o = 0; o < cb.size();) {+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (cb.data() + o);+ if (e->Processor.GroupCount > 0 && e->Processor.GroupMask[0].Group == 0) {+ const KAFFINITY m = e->Processor.GroupMask[0].Mask;+ int first = -1;+ for (int b = 0; b < 64; ++b)+ if ((m >> b) & 1) {+ first = b;+ break;+ }+ for (size_t n = 0; n < masks.size() && first >= 0; ++n)+ if ((masks[n] >> first) & 1) nodes[n].push_back(first);+ }+ o += e->Size;+ }+ nodes.erase(std::remove_if(nodes.begin(), nodes.end(), [](const std::vector<int>& v) { return v.empty(); }),+ nodes.end());+ for (auto& v : nodes) std::sort(v.begin(), v.end());+ return nodes;+}+/// The RAM this process can still get (~0: not known).+inline uint64_t ram_avail() {+ MEMORYSTATUSEX s{};+ s.dwLength = sizeof(s);+ return GlobalMemoryStatusEx(&s) ? (uint64_t) s.ullAvailPhys : ~uint64_t{0};+}+#endif++/// Windows' commit (RAM + page file, what every private allocation is charged to - on an integrated GPU the GPU's+/// own allocations too): the limit, what this process can still commit, the RAM; all ~0 where there is no such limit+/// (Linux overcommits; its RAM is ram_avail's business).+struct Commit {+ uint64_t limit = ~uint64_t{0}, avail = ~uint64_t{0}, ram = ~uint64_t{0};+};+inline Commit commit() {+ Commit c;+#if defined(_WIN32)+ MEMORYSTATUSEX s{};+ s.dwLength = sizeof(s);+ if (GlobalMemoryStatusEx(&s)) {+ c.limit = (uint64_t) s.ullTotalPageFile;+ c.avail = (uint64_t) s.ullAvailPageFile;+ c.ram = (uint64_t) s.ullTotalPhys;+ }+#endif+ return c;+}++} // namespace strata::ds4::osdiff --git a/src/dsv41/ds4_pool.hpp b/src/dsv41/ds4_pool.hppnew file mode 100644index 000000000..9888f5e3b--- /dev/null+++ b/src/dsv41/ds4_pool.hpp@@ -0,0 +1,222 @@+// src/ds4/ds4_pool.hpp - the DeepSeek engine's CPU workers: run(n, fn) runs fn(0..n-1) on the workers and the caller and+// returns when every job is done. The design is glmfast::Workers' (src/core/glm_fast_state.hpp): one 64-bit ticket+// (generation << 32 | next index) claimed by CAS, so a worker that wakes late for an older batch can never take a job+// of the current one; idle workers spin for a while, then sleep on a condition variable.+#pragma once++#include <atomic>+#include <chrono>+#include <condition_variable>+#include <cstdint>+#include <cstring>+#include <deque>+#include <functional>+#include <mutex>+#include <thread>+#include <vector>++#include "ds4_os.hpp"+#if defined(_MSC_VER)+#include <intrin.h>+#endif++namespace strata::ds4 {++inline void cpu_relax() {+#if defined(_MSC_VER)+ _mm_pause();+#elif defined(__x86_64__) || defined(_M_X64)+ __builtin_ia32_pause();+#endif+}++class Pool {+public:+ explicit Pool(int n, const std::vector<int>& cpus = {}, int spin_us = 200) : spin_us_(spin_us) {+ for (int i = 0; i < n; ++i) {+ th_.emplace_back([this] { loop(); });+ if (!cpus.empty()) os::pin_thread(th_.back(), cpus[(size_t) i % cpus.size()]);+ }+ }+ ~Pool() {+ quit_.store(true);+ {+ std::lock_guard<std::mutex> lk(mu_);+ }+ cv_.notify_all();+ for (auto& t : th_) t.join();+ }+ int size() const { return (int) th_.size() + 1; }++ void run(int n, const std::function<void(int)>& fn) {+ start(n, fn);+ help();+ wait();+ }+ /// The batch goes to the workers; the caller may help() and must wait() before fn goes out of scope. (Two pools+ /// - one a NUMA node - run at once this way: start both, help one, wait for both.)+ void start(int n, const std::function<void(int)>& fn) {+ n_ = n;+ if (n <= 0) return;+ const uint64_t g = (gen_.load(std::memory_order_relaxed) + 1) & 0xffffffffu;+ ticket_.store((g << 32) | 0x7fffffffu, std::memory_order_seq_cst);+ fn_ = &fn;+ total_.store(n, std::memory_order_release);+ done_.store(0, std::memory_order_relaxed);+ ticket_.store(g << 32, std::memory_order_release);+ gen_.store(g, std::memory_order_release);+ if (sleeping_.load(std::memory_order_acquire) > 0) {+ std::lock_guard<std::mutex> lk(mu_);+ cv_.notify_all();+ }+ }+ void help() {+ if (n_ > 0) work(gen_.load(std::memory_order_acquire));+ }+ void wait() {+ if (n_ <= 0) return;+ while (done_.load(std::memory_order_acquire) < n_) cpu_relax();+ n_ = 0;+ }++private:+ void work(uint64_t g) {+ for (;;) {+ uint64_t t = ticket_.load(std::memory_order_acquire);+ if ((t >> 32) != g) return;+ const int n = total_.load(std::memory_order_acquire);+ if ((int) (t & 0xffffffffu) >= n) return;+ if (!ticket_.compare_exchange_weak(t, t + 1, std::memory_order_acq_rel)) continue;+ (*fn_)((int) (t & 0xffffffffu));+ done_.fetch_add(1, std::memory_order_acq_rel);+ }+ }+ void loop() {+ uint64_t seen = gen_.load();+ for (;;) {+ const auto t0 = std::chrono::steady_clock::now();+ uint64_t g;+ for (;;) {+ g = gen_.load(std::memory_order_acquire);+ if (g != seen || quit_.load(std::memory_order_relaxed)) break;+ cpu_relax();+ if (std::chrono::steady_clock::now() - t0 > std::chrono::microseconds(spin_us_)) {+ std::unique_lock<std::mutex> lk(mu_);+ sleeping_.fetch_add(1);+ cv_.wait(lk, [&] { return gen_.load() != seen || quit_.load(); });+ sleeping_.fetch_sub(1);+ }+ }+ if (quit_.load()) return;+ seen = g;+ work(g);+ }+ }++ std::vector<std::thread> th_;+ int spin_us_;+ int n_ = 0; // the current batch's size (the caller's side)+ const std::function<void(int)>* fn_ = nullptr;+ alignas(64) std::atomic<uint64_t> ticket_{0};+ alignas(64) std::atomic<uint64_t> gen_{0};+ alignas(64) std::atomic<int> total_{0};+ alignas(64) std::atomic<int> done_{0};+ std::atomic<int> sleeping_{0};+ std::atomic<bool> quit_{false};+ std::mutex mu_;+ std::condition_variable cv_;+};++/// File reads on threads of their own, in the order they were pushed, the pusher never waiting: the prompt path's+/// streamed experts (each part of an expert a job), unbuffered where the job has a direct descriptor and its range is+/// not all in the page cache already (the disk straight into dst, the cache neither used nor disturbed), buffered+/// otherwise or when the unbuffered read fails - the bytes land at dst + off % os::kDirectAlign either way. A job's `left` is+/// counted down when it is done (`bad` set if it failed); wait_any() sleeps until some job finished or the timeout.+class IoQueue {+public:+ struct Job {+ os::fd_t fd = os::kBadFd, dfd = os::kBadFd;+ uint64_t off = 0, len = 0;+ uint8_t* dst = nullptr;+ std::atomic<int>* left = nullptr;+ std::atomic<bool>* bad = nullptr;+ const uint8_t* mem = nullptr; // non-null: a copy of len bytes from memory to dst instead of a read+ const void* map = nullptr; // the range in the file's mapping: all in the page cache, it is read buffered+ uint8_t* keep = nullptr; // non-null (a read): the len bytes read copied here as well+ };+ explicit IoQueue(int n) {+ for (int i = 0; i < n; ++i) th_.emplace_back([this] { loop(); });+ }+ ~IoQueue() {+ {+ std::lock_guard<std::mutex> lk(mu_);+ quit_ = true;+ }+ cv_.notify_all();+ for (auto& t : th_) t.join();+ }+ void push(const Job* jobs, int n) {+ {+ std::lock_guard<std::mutex> lk(mu_);+ for (int i = 0; i < n; ++i) q_.push_back(jobs[i]);+ }+ if (n == 1) cv_.notify_one();+ else cv_.notify_all();+ }+ void wait_any(int us) {+ std::unique_lock<std::mutex> lk(done_mu_);+ const uint64_t seen = done_;+ done_cv_.wait_for(lk, std::chrono::microseconds(us), [&] { return done_ != seen; });+ }+ double busy_ms() const { return busy_us_.load(std::memory_order_relaxed) / 1000.0; }+ uint64_t direct_bytes() const { return direct_.load(std::memory_order_relaxed); }+ uint64_t buffered_bytes() const { return buffered_.load(std::memory_order_relaxed); }++private:+ void loop() {+ for (;;) {+ Job j;+ {+ std::unique_lock<std::mutex> lk(mu_);+ cv_.wait(lk, [&] { return quit_ || !q_.empty(); });+ if (quit_ && q_.empty()) return;+ j = q_.front();+ q_.pop_front();+ }+ const auto t0 = std::chrono::steady_clock::now();+ const uint64_t lead = j.off % os::kDirectAlign;+ bool ok = false;+ if (j.mem != nullptr) {+ std::memcpy(j.dst, j.mem, (size_t) j.len);+ ok = true;+ } else if (j.dfd != os::kBadFd && !(j.map != nullptr && os::resident(j.map, (size_t) j.len)) &&+ os::pread_direct(j.dfd, j.dst, j.len, j.off)) { // (the page cache's ones: buffered, below)+ ok = true;+ direct_.fetch_add(j.len, std::memory_order_relaxed);+ } else if (os::pread_par(j.fd, j.dst + lead, j.len, j.off)) {+ ok = true;+ buffered_.fetch_add(j.len, std::memory_order_relaxed);+ }+ if (ok && j.keep != nullptr && j.mem == nullptr) std::memcpy(j.keep, j.dst + lead, (size_t) j.len);+ busy_us_.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::steady_clock::now() - t0).count(),+ std::memory_order_relaxed);+ if (!ok && j.bad != nullptr) j.bad->store(true, std::memory_order_release);+ if (j.left != nullptr) j.left->fetch_sub(1, std::memory_order_acq_rel);+ {+ std::lock_guard<std::mutex> lk(done_mu_);+ ++done_;+ }+ done_cv_.notify_all();+ }+ }+ std::vector<std::thread> th_;+ std::mutex mu_, done_mu_;+ std::condition_variable cv_, done_cv_;+ std::deque<Job> q_;+ uint64_t done_ = 0;+ bool quit_ = false;+ std::atomic<uint64_t> busy_us_{0}, direct_{0}, buffered_{0};+};++} // namespace strata::ds4diff --git a/src/dsv41/ds4_q6k.cuh b/src/dsv41/ds4_q6k.cuhnew file mode 100644index 000000000..2eb517eb0--- /dev/null+++ b/src/dsv41/ds4_q6k.cuh@@ -0,0 +1,28 @@+// src/ds4/ds4_q6k.cuh - one value of a ggml Q6_K superblock (256 values, 210 bytes: ql[128] the low 4 bits, qh[64] the+// high 2, scales[16] int8 a 16 values, d F16), as ggml's dequantize_row_q6_K computes it: d * scale * (q - 32), the two+// products in that order in F32 - the same bits (the Engram tables a GGUF may hold in Q6_K).+#pragma once++#include <cuda_fp16.h>+#include <cstdint>++namespace strata::ds4 {++constexpr int kQ6kBytes = 210;++__device__ __forceinline__ float q6k_value(const uint8_t* __restrict__ b, int i) {+ const uint8_t* ql = b;+ const uint8_t* qh = b + 128;+ const int8_t* sc = (const int8_t*) (b + 192);+ // (the F16 scale's bits as a half: HIP's headers take no brace-initialized __half_raw from them)+ const float d = __half2float(__ushort_as_half((unsigned short) (b[208] | (b[209] << 8))));+ const int h = i >> 7, w = i & 127, quarter = w >> 5, l = w & 31; // (ggml's loop: half h, 32-value quarters)+ ql += h * 64;+ qh += h * 32;+ sc += h * 8;+ const int lo = quarter & 1 ? ql[l + 32] : ql[l];+ const int q = ((quarter & 2 ? lo >> 4 : lo & 0xF) | (((qh[l] >> (2 * quarter)) & 3) << 4)) - 32;+ return __fmul_rn(__fmul_rn(d, (float) sc[(l >> 4) + 2 * quarter]), (float) q);+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_vision.cu b/src/dsv41/ds4_vision.cunew file mode 100644index 000000000..b19b307d8--- /dev/null+++ b/src/dsv41/ds4_vision.cu@@ -0,0 +1,517 @@+// src/ds4/ds4_vision.cu - DeepSeek-V4.1's vision tower (see ds4_vision.cuh).+//+// One encode, N patches on a gh x gw grid:+// patchify (the reference's (x / 255 - 0.5) / 0.5, BF16) -> x = patch_embed(patches) FP32 [N][1024]+// per block, its weights in slot (b % 2) while block b + 1's copy runs:+// xn = norm1(x); qkv = rope(wqkv(xn) + bias) (rows in pieces of rb_: the FP32 GEMM output)+// per query piece of qb_ rows, per head: S = Q K^T / 8 (FP32), P = softmax(S) (BF16), O[:, head] = P V;+// x[piece] += wo(O) + bias+// xn = norm2(x); per row piece: x += w2(silu(gate) * up), gate | up = w1(xn)+// xn = norm(x); the 3 x 3 unfold; GELU(w1 + b1); w2 + b2 -> BF16 rows (the aligner's weights stream in pieces too)+#include "ds4_vision.cuh"++#include <cublas_v2.h>+#include "ds4_numerics.cuh"+#include <cuda_runtime.h>++#include <algorithm>+#include <cfloat>+#include <cmath>+#include <cstring>++namespace strata::ds4 {+namespace {++#define VK(call) \+ do { \+ cudaError_t e_ = (call); \+ if (e_ != cudaSuccess) { \+ err = std::string("vision: ") + #call + ": " + cudaGetErrorString(e_); \+ return false; \+ } \+ } while (0)+#define VB(call) \+ do { \+ cublasStatus_t e_ = (call); \+ if (e_ != CUBLAS_STATUS_SUCCESS) { \+ err = std::string("vision: ") + #call + ": cuBLAS status " + std::to_string((int) e_); \+ return false; \+ } \+ } while (0)++constexpr int kT = 256; // threads of the row kernels+constexpr int kSmT = 512, kSmPer = 20; // softmax: a row of up to 10240 keys in registers+constexpr size_t kWs = 16u << 20; // cuBLAS workspace++__device__ __forceinline__ float bf(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }+__device__ __forceinline__ uint16_t bfr(float f) { return bf16_rn(f); }++__device__ float block_reduce(float v, bool is_max) {+ __shared__ float red[32];+ __shared__ float tot;+ for (int o = 16; o > 0; o >>= 1) {+ const float u = __shfl_xor_sync(0xffffffffu, v, o);+ v = is_max ? fmaxf(v, u) : v + u;+ }+ const int lane = threadIdx.x & 31, w = threadIdx.x >> 5, nw = (blockDim.x + 31) >> 5;+ __syncthreads(); // (an earlier call's readers of red / tot are done)+ if (lane == 0) red[w] = v;+ __syncthreads();+ if (w == 0) {+ v = lane < nw ? red[lane] : (is_max ? -FLT_MAX : 0.0f);+ for (int o = 16; o > 0; o >>= 1) {+ const float u = __shfl_xor_sync(0xffffffffu, v, o);+ v = is_max ? fmaxf(v, u) : v + u;+ }+ if (lane == 0) tot = v;+ }+ __syncthreads();+ return tot;+}++// patch i of the gh x gw grid: its 3 x p x p values, channel-major (the reference's reshape / permute)+__global__ void patchify_kernel(const uint8_t* __restrict__ rgb, int width, int gw, int p, uint16_t* __restrict__ out) {+ const int i = blockIdx.x, e = threadIdx.x, pp = p * p;+ if (e >= 3 * pp) return;+ const int c = e / pp, r = e % pp, y = (i / gw) * p + r / p, x = (i % gw) * p + r % p;+ const float u = (float) rgb[((size_t) y * width + x) * 3 + c];+ out[(size_t) i * 3 * pp + e] = bfr(__fdiv_rn(__fsub_rn(__fdiv_rn(u, 255.0f), 0.5f), 0.5f));+}++// x[r] += y[r] (y null: nothing) + bias+__global__ void add_rows_kernel(float* __restrict__ x, const float* __restrict__ y, int ldy,+ const uint16_t* __restrict__ bias, int dim) {+ const int r = blockIdx.x;+ for (int e = threadIdx.x; e < dim; e += blockDim.x) {+ float v = x[(size_t) r * dim + e];+ if (y != nullptr) v += y[(size_t) r * ldy + e];+ if (bias != nullptr) v += bf(bias[e]);+ x[(size_t) r * dim + e] = v;+ }+}++__global__ void rmsnorm_kernel(const float* __restrict__ x, const uint16_t* __restrict__ w, int dim, float eps,+ uint16_t* __restrict__ out) {+ const int r = blockIdx.x;+ const float* xr = x + (size_t) r * dim;+ float ss = 0.0f;+ for (int e = threadIdx.x; e < dim; e += blockDim.x) ss += xr[e] * xr[e];+ ss = block_reduce(ss, false);+ const float rs = rsqrtf(ss / (float) dim + eps);+ for (int e = threadIdx.x; e < dim; e += blockDim.x) out[(size_t) r * dim + e] = bfr(bf(w[e]) * (xr[e] * rs));+}++// a piece's rows of wqkv's output (+ bias, rounded to BF16 as the reference's linear) into qkv, q and k rotated: the+// head's first half pairs with its second; frequency j < hd / 4 turns with the patch's row, the rest with its column+__global__ void qkv_rope_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int r0, int gw,+ const float* __restrict__ inv_freq, int heads, int hd, uint16_t* __restrict__ qkv) {+ const int r = blockIdx.x, gr = r0 + r, dim = heads * hd, half = hd / 2, nf = hd / 4;+ const int row = gr / gw, col = gr % gw;+ const float* yr = y + (size_t) r * 3 * dim;+ uint16_t* o = qkv + (size_t) gr * 3 * dim;+ for (int i = threadIdx.x; i < 2 * heads * half; i += blockDim.x) {+ const int base = (i / half) * hd, d = i % half;+ const float x1 = bf(bfr(yr[base + d] + bf(bias[base + d])));+ const float x2 = bf(bfr(yr[base + d + half] + bf(bias[base + d + half])));+ float sn, cs;+ sincosf((float) (d < nf ? row : col) * inv_freq[d % nf], &sn, &cs);+ o[base + d] = bfr(x1 * cs - x2 * sn);+ o[base + d + half] = bfr(x2 * cs + x1 * sn);+ }+ for (int e = 2 * dim + threadIdx.x; e < 3 * dim; e += blockDim.x) o[e] = bfr(yr[e] + bf(bias[e]));+}++__global__ void __launch_bounds__(kSmT) softmax_kernel(const float* __restrict__ s, int n, uint16_t* __restrict__ p) {+ const float* sr = s + (size_t) blockIdx.x * n;+ uint16_t* pr = p + (size_t) blockIdx.x * n;+ float v[kSmPer];+ float m = -FLT_MAX;+#pragma unroll+ for (int k = 0; k < kSmPer; ++k) {+ const int e = threadIdx.x + k * kSmT;+ v[k] = e < n ? sr[e] : -FLT_MAX;+ m = fmaxf(m, v[k]);+ }+ m = block_reduce(m, true);+ float sum = 0.0f;+#pragma unroll+ for (int k = 0; k < kSmPer; ++k) {+ const int e = threadIdx.x + k * kSmT;+ v[k] = e < n ? expf(v[k] - m) : 0.0f;+ sum += v[k];+ }+ const float inv = 1.0f / block_reduce(sum, false);+#pragma unroll+ for (int k = 0; k < kSmPer; ++k) {+ const int e = threadIdx.x + k * kSmT;+ if (e < n) pr[e] = bfr(v[k] * inv);+ }+}++// mid = silu(gate) * up, gate | up the halves of w1's row (BF16 steps, as the reference's)+__global__ void silu_mul_kernel(const float* __restrict__ y, int inter, uint16_t* __restrict__ mid) {+ const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;+ if (e >= inter) return;+ const float gt = bf(bfr(y[(size_t) r * 2 * inter + e])), up = bf(bfr(y[(size_t) r * 2 * inter + inter + e]));+ mid[(size_t) r * inter + e] = bfr(bf(bfr(gt / (1.0f + expf(-gt)))) * up);+}++// the aligner's input: block (bi, bj) of the m x m patches as one row, value c * m * m + kh * m + kw (F.unfold's+// order), zero past the grid+__global__ void unfold_kernel(const uint16_t* __restrict__ xn, int gh, int gw, int m, int dim, int lw,+ uint16_t* __restrict__ un) {+ const int l = blockIdx.x, bi = l / lw, bj = l % lw, mm = m * m;+ for (int f = threadIdx.x; f < dim * mm; f += blockDim.x) {+ const int c = f / mm, k = f % mm, y = bi * m + k / m, x = bj * m + k % m;+ un[(size_t) l * dim * mm + f] = y < gh && x < gw ? xn[((size_t) y * gw + x) * dim + c] : (uint16_t) 0;+ }+}++// out = act(bf16(y + bias)): GELU (erf) or none+__global__ void bias_act_kernel(const float* __restrict__ y, const uint16_t* __restrict__ bias, int n, int gelu,+ uint16_t* __restrict__ out) {+ const int r = blockIdx.y, e = blockIdx.x * blockDim.x + threadIdx.x;+ if (e >= n) return;+ float v = bf(bfr(y[(size_t) r * n + e] + bf(bias[e])));+ if (gelu) v = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f));+ out[(size_t) r * n + e] = bfr(v);+}++int cdiv(int a, int b) { return (a + b - 1) / b; }++// y[r][o] = alpha * sum_k x[r][k] W[o][k] (row-major W [out][in] with row stride ldw, x [rows][ldx], y [rows][ldy])+bool gemm(cublasHandle_t h, const uint16_t* W, int ldw, const uint16_t* x, int ldx, void* y, cudaDataType yt, int ldy,+ int rows, int in, int out, float alpha, std::string& err) {+ const float beta = 0.0f;+ VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, out, rows, in, &alpha, W, CUDA_R_16BF, ldw, x, CUDA_R_16BF, ldx, &beta,+ y, yt, ldy, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));+ return true;+}++} // namespace++template <class T>+bool VisionTower::dalloc(T*& p, size_t n, std::string& err) {+ void* v = nullptr;+ const size_t b = (n * sizeof(T) + 255) / 256 * 256;+ if (cudaMalloc(&v, b) != cudaSuccess) {+ err = "vision: cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(at_) + " failed";+ return false;+ }+ dallocs_.push_back(v);+ dev_bytes_ += b;+ p = (T*) v;+ return true;+}++VisionTower::~VisionTower() { release(); }++void VisionTower::release() {+ if (at_ < 0) return;+ int cur = 0;+ cudaGetDevice(&cur);+ cudaSetDevice(at_);+ for (int i = 0; i < 2; ++i) {+ if (copied_[i]) cudaEventDestroy((cudaEvent_t) copied_[i]);+ if (used_[i]) cudaEventDestroy((cudaEvent_t) used_[i]);+ copied_[i] = used_[i] = nullptr;+ }+ if (blas_) cublasDestroy((cublasHandle_t) blas_);+ if (st_) cudaStreamDestroy((cudaStream_t) st_);+ if (cs_) cudaStreamDestroy((cudaStream_t) cs_);+ for (void* p : dallocs_) cudaFree(p);+ if (host_) cudaFreeHost(host_);+ cudaSetDevice(cur);+ blas_ = st_ = cs_ = nullptr;+ dallocs_.clear();+ host_ = nullptr;+ dev_ = at_ = -1;+ dev_bytes_ = 0;+}++bool VisionTower::load(const VisionGeo& g, int device, const Reader& read, std::string& err) {+ g_ = g;+ const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = g.heads > 0 ? g.dim / g.heads : 0;+ if (g.layers <= 0 || hd * g.heads != D || hd % 4 != 0 || P <= 0 || M <= 0 || g.max_tokens < 5) {+ err = "vision: geometry not covered";+ return false;+ }+ // the largest aligner grid within max_tokens (n_h * (n_w + 1) + 2): the work buffers' size+ int cells = 0;+ for (int h = 1; h <= g.max_tokens - 2; ++h) {+ const int w = (g.max_tokens - 2) / h - 1;+ if (w >= 1) cells = std::max(cells, h * w);+ }+ max_n_ = cells * M * M;+ if (max_n_ > kSmT * kSmPer) {+ err = "vision: " + std::to_string(max_n_) + " patches are more than the attention covers";+ return false;+ }+ cudaSetDevice(device);+ at_ = device;+ dev_bytes_ = 0;+ // the checkpoint's tensors, checked before anything is read+ struct Want {+ std::string name;+ std::vector<int64_t> shape;+ };+ std::vector<Want> want = {{"vision.patch_embed.proj.weight", {D, 3 * P * P}},+ {"vision.patch_embed.proj.bias", {D}},+ {"vision.norm.weight", {D}},+ {"aligner.w1.weight", {g.out, (int64_t) D * M * M}},+ {"aligner.w1.bias", {g.out}},+ {"aligner.w2.weight", {g.out, g.out}},+ {"aligner.w2.bias", {g.out}}};+ for (int l = 0; l < g.layers; ++l) {+ const std::string B = "vision.blocks." + std::to_string(l) + ".";+ want.push_back({B + "norm1.weight", {D}});+ want.push_back({B + "norm2.weight", {D}});+ want.push_back({B + "attn.wqkv.weight", {3 * D, D}});+ want.push_back({B + "attn.wqkv.bias", {3 * D}});+ want.push_back({B + "attn.wo.weight", {D, D}});+ want.push_back({B + "attn.wo.bias", {D}});+ want.push_back({B + "mlp.w1.weight", {2 * I, D}});+ want.push_back({B + "mlp.w2.weight", {D, I}});+ }+ for (const Want& w : want) {+ std::vector<int64_t> s;+ if (!read(w.name, s, nullptr)) {+ err = "vision: " + w.name + " is not in the checkpoint";+ return false;+ }+ if (s != w.shape) {+ err = "vision: " + w.name + " has another shape than the config's";+ return false;+ }+ }+ // the streamed weights, pinned+ block_n_ = (size_t) 3 * D * D + (size_t) D * D + (size_t) 2 * I * D + (size_t) D * I;+ al1_off_ = block_n_ * g.layers;+ al2_off_ = al1_off_ + (size_t) g.out * D * M * M;+ const size_t host_n = al2_off_ + (size_t) g.out * g.out;+ if (cudaHostAlloc((void**) &host_, host_n * 2, cudaHostAllocDefault) != cudaSuccess) {+ host_ = nullptr;+ err = "vision: pinning " + std::to_string((host_n * 2) >> 20) + " MiB of host memory failed";+ return false;+ }+ std::vector<int64_t> s;+ for (int l = 0; l < g.layers; ++l) {+ const std::string B = "vision.blocks." + std::to_string(l) + ".";+ uint16_t* b = host_ + (size_t) l * block_n_;+ if (!read(B + "attn.wqkv.weight", s, b) || !read(B + "attn.wo.weight", s, b + (size_t) 3 * D * D) ||+ !read(B + "mlp.w1.weight", s, b + (size_t) 4 * D * D) ||+ !read(B + "mlp.w2.weight", s, b + (size_t) 4 * D * D + (size_t) 2 * I * D)) {+ err = "vision: block " + std::to_string(l) + ": read failed";+ return false;+ }+ }+ if (!read("aligner.w1.weight", s, host_ + al1_off_) || !read("aligner.w2.weight", s, host_ + al2_off_)) {+ err = "vision: the aligner: read failed";+ return false;+ }+ // the small ones on the device+ auto up = [&](const std::string& name, size_t n, uint16_t*& d) -> bool {+ std::vector<uint16_t> h(n);+ std::vector<int64_t> sh;+ if (!read(name, sh, h.data())) {+ err = "vision: " + name + ": read failed";+ return false;+ }+ if (!dalloc(d, n, err)) return false;+ VK(cudaMemcpy(d, h.data(), n * 2, cudaMemcpyHostToDevice));+ return true;+ };+ bool ok = up("vision.patch_embed.proj.weight", (size_t) D * 3 * P * P, pe_w_) &&+ up("vision.patch_embed.proj.bias", D, pe_b_) && up("vision.norm.weight", D, norm_) &&+ up("aligner.w1.bias", g.out, al1_b_) && up("aligner.w2.bias", g.out, al2_b_);+ n1_.assign(g.layers, nullptr);+ n2_.assign(g.layers, nullptr);+ bqkv_.assign(g.layers, nullptr);+ bo_.assign(g.layers, nullptr);+ for (int l = 0; l < g.layers && ok; ++l) {+ const std::string B = "vision.blocks." + std::to_string(l) + ".";+ ok = up(B + "norm1.weight", D, n1_[l]) && up(B + "norm2.weight", D, n2_[l]) &&+ up(B + "attn.wqkv.bias", 3 * D, bqkv_[l]) && up(B + "attn.wo.bias", D, bo_[l]);+ }+ if (!ok) return false;+ { // the rotary's frequencies: theta ^ -(2i / (hd / 2)), hd / 4 of them+ const int nf = hd / 4;+ std::vector<float> f(nf);+ for (int i = 0; i < nf; ++i) f[i] = 1.0f / std::pow(g.theta, (float) (2 * i) / (float) (hd / 2));+ if (!dalloc(inv_freq_, nf, err)) return false;+ VK(cudaMemcpy(inv_freq_, f.data(), nf * 4, cudaMemcpyHostToDevice));+ }+ // the work buffers: the patch grid's rows whole (the residual, its norm, q / k / v), the GEMM outputs a piece of+ // rows at a time, the attention's scores a piece of queries at a time (~48 MB)+ const int L = cells;+ rb_ = 2048;+ qb_ = std::max(16, (int) ((48u << 20) / ((size_t) max_n_ * 4)) / 16 * 16);+ slot_n_ = std::max(block_n_, (size_t) 1 << 22);+ const size_t y_n = (size_t) rb_ * std::max({3 * D, 2 * I, D});+ const size_t qkv_n = std::max({(size_t) max_n_ * 3 * D,+ (size_t) max_n_ * 3 * P * P + ((size_t) max_n_ * 3 * P * P + 1) / 2, // patches, pixels+ (size_t) L * D * M * M + (size_t) 2 * L * g.out}); // the aligner's+ if ((size_t) L * g.out > y_n || (size_t) max_n_ * D > y_n) {+ err = "vision: work buffer sizes";+ return false;+ }+ ok = dalloc(x_, (size_t) max_n_ * D, err) && dalloc(xn_, (size_t) max_n_ * D, err) && dalloc(qkv_, qkv_n, err) &&+ dalloc(y_, y_n, err) && dalloc(o_, (size_t) max_n_ * D, err) && dalloc(mid_, (size_t) rb_ * I, err) &&+ dalloc(s_, (size_t) qb_ * max_n_, err) && dalloc(p_, (size_t) qb_ * max_n_, err) &&+ dalloc(slot_[0], slot_n_, err) && dalloc(slot_[1], slot_n_, err) && dalloc(ws_, kWs, err);+ if (!ok) return false;+ patch_ = qkv_; // (before block 0's qkv)+ rgb_ = (uint8_t*) (qkv_ + (size_t) max_n_ * 3 * P * P);+ un_ = qkv_; // (after the last block's attention)+ a16_ = un_ + (size_t) L * D * M * M;+ outb_ = a16_ + (size_t) L * g.out;+ cublasHandle_t h = nullptr;+ VB(cublasCreate(&h));+ blas_ = h;+ cudaStream_t st = nullptr, cs = nullptr;+ VK(cudaStreamCreateWithFlags(&st, cudaStreamNonBlocking));+ VK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));+ st_ = st;+ cs_ = cs;+ VB(cublasSetStream(h, st));+ VB(cublasSetWorkspace(h, ws_, kWs));+ for (int i = 0; i < 2; ++i) {+ cudaEvent_t a = nullptr, b = nullptr;+ VK(cudaEventCreateWithFlags(&a, cudaEventDisableTiming));+ VK(cudaEventCreateWithFlags(&b, cudaEventDisableTiming));+ copied_[i] = a;+ used_[i] = b;+ }+ dev_ = device;+ return true;+}++bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,+ std::string& err) {+ if (dev_ < 0) {+ err = "vision: not loaded";+ return false;+ }+ const VisionGeo& g = g_;+ const int D = g.dim, I = g.inter, P = g.patch, M = g.merge, hd = D / g.heads, H3 = 3 * D;+ if (width <= 0 || height <= 0 || width % P != 0 || height % P != 0) {+ err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) ++ " pixels (both sides must be multiples of " + std::to_string(P) + ")";+ return false;+ }+ const int gh = height / P, gw = width / P, N = gh * gw;+ n_h = cdiv(gh, M);+ n_w = cdiv(gw, M);+ const int L = n_h * n_w;+ if (N > max_n_ || n_h * (n_w + 1) + 2 > g.max_tokens) {+ err = "vision: an image of " + std::to_string(width) + " x " + std::to_string(height) + " pixels is more than " ++ std::to_string(g.max_tokens) + " image tokens";+ return false;+ }+ cudaSetDevice(dev_);+ cublasHandle_t h = (cublasHandle_t) blas_;+ cudaStream_t s = (cudaStream_t) st_, cs = (cudaStream_t) cs_;+ // the weights in the order the encode takes them: the blocks, then the aligner's in pieces of a slot+ std::vector<Piece> pcs;+ for (int l = 0; l < g.layers; ++l) pcs.push_back({host_ + (size_t) l * block_n_, block_n_});+ const int in1 = D * M * M, r1 = (int) std::min<size_t>(g.out, slot_n_ / in1), r2 = (int) std::min<size_t>(g.out, slot_n_ / g.out);+ for (int o0 = 0; o0 < g.out; o0 += r1)+ pcs.push_back({host_ + al1_off_ + (size_t) o0 * in1, (size_t) std::min(r1, g.out - o0) * in1});+ for (int o0 = 0; o0 < g.out; o0 += r2)+ pcs.push_back({host_ + al2_off_ + (size_t) o0 * g.out, (size_t) std::min(r2, g.out - o0) * g.out});+ auto fetch = [&](size_t i) -> bool {+ const int k = (int) (i % 2);+ VK(cudaStreamWaitEvent(cs, (cudaEvent_t) used_[k], 0));+ VK(cudaMemcpyAsync(slot_[k], pcs[i].host, pcs[i].n * 2, cudaMemcpyHostToDevice, cs));+ VK(cudaEventRecord((cudaEvent_t) copied_[k], cs));+ return true;+ };+ size_t pi = 0; // the piece being used+ auto take = [&]() -> const uint16_t* { // the next piece's slot, once its copy is in (the one after it started)+ const int k = (int) (pi % 2);+ if (pi + 1 < pcs.size() && !fetch(pi + 1)) return nullptr;+ if (cudaStreamWaitEvent(s, (cudaEvent_t) copied_[k], 0) != cudaSuccess) {+ err = "vision: waiting for a weight copy failed";+ return nullptr;+ }+ return slot_[k];+ };+ auto done = [&]() { // the piece's work is queued: its slot is free once that ran+ cudaEventRecord((cudaEvent_t) used_[pi % 2], s);+ ++pi;+ };+ if (!fetch(0)) return false;++ // the patches+ VK(cudaMemcpyAsync(rgb_, rgb, (size_t) N * 3 * P * P, cudaMemcpyHostToDevice, s));+ patchify_kernel<<<N, 3 * P * P, 0, s>>>(rgb_, width, gw, P, patch_);+ if (!gemm(h, pe_w_, 3 * P * P, patch_, 3 * P * P, x_, CUDA_R_32F, D, N, 3 * P * P, D, 1.0f, err)) return false;+ add_rows_kernel<<<N, kT, 0, s>>>(x_, nullptr, 0, pe_b_, D);++ const float scale = 1.0f / std::sqrt((float) hd);+ const int qb = (int) std::max<size_t>(1, std::min<size_t>((size_t) N, (size_t) qb_ * max_n_ / (size_t) N));+ for (int l = 0; l < g.layers; ++l) {+ const uint16_t* wqkv = take();+ if (wqkv == nullptr) return false;+ const uint16_t *wo = wqkv + (size_t) H3 * D, *w1 = wo + (size_t) D * D, *w2 = w1 + (size_t) 2 * I * D;+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n1_[l], D, g.eps, xn_);+ for (int r0 = 0; r0 < N; r0 += rb_) {+ const int nr = std::min(rb_, N - r0);+ if (!gemm(h, wqkv, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, H3, nr, D, H3, 1.0f, err)) return false;+ qkv_rope_kernel<<<nr, kT, 0, s>>>(y_, bqkv_[l], r0, gw, inv_freq_, g.heads, hd, qkv_);+ }+ for (int q0 = 0; q0 < N; q0 += qb) {+ const int nq = std::min(qb, N - q0);+ const float one = 1.0f, zero = 0.0f;+ for (int hh = 0; hh < g.heads; ++hh) {+ // S^T (keys x queries, column-major) = K Q^T: row-major S [query][key]+ VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, N, nq, hd, &scale, qkv_ + D + hh * hd, CUDA_R_16BF, H3,+ qkv_ + (size_t) q0 * H3 + hh * hd, CUDA_R_16BF, H3, &zero, s_, CUDA_R_32F, N,+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));+ softmax_kernel<<<nq, kSmT, 0, s>>>(s_, N, p_);+ // O^T (hd x queries) = V^T P^T: row-major O [query][head's hd] at column hh * hd+ VB(cublasGemmEx(h, CUBLAS_OP_N, CUBLAS_OP_N, hd, nq, N, &one, qkv_ + 2 * D + hh * hd, CUDA_R_16BF, H3,+ p_, CUDA_R_16BF, N, &zero, o_ + hh * hd, CUDA_R_16BF, D, CUBLAS_COMPUTE_32F,+ CUBLAS_GEMM_DEFAULT));+ }+ if (!gemm(h, wo, D, o_, D, y_, CUDA_R_32F, D, nq, D, D, 1.0f, err)) return false;+ add_rows_kernel<<<nq, kT, 0, s>>>(x_ + (size_t) q0 * D, y_, D, bo_[l], D);+ }+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n2_[l], D, g.eps, xn_);+ for (int r0 = 0; r0 < N; r0 += rb_) {+ const int nr = std::min(rb_, N - r0);+ if (!gemm(h, w1, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, 2 * I, nr, D, 2 * I, 1.0f, err)) return false;+ silu_mul_kernel<<<dim3(cdiv(I, kT), nr), kT, 0, s>>>(y_, I, mid_);+ if (!gemm(h, w2, I, mid_, I, y_, CUDA_R_32F, D, nr, I, D, 1.0f, err)) return false;+ add_rows_kernel<<<nr, kT, 0, s>>>(x_ + (size_t) r0 * D, y_, D, nullptr, D);+ }+ done();+ }+ rmsnorm_kernel<<<N, kT, 0, s>>>(x_, norm_, D, g.eps, xn_);++ // the aligner+ unfold_kernel<<<L, kT, 0, s>>>(xn_, gh, gw, M, D, n_w, un_);+ for (int o0 = 0; o0 < g.out; o0 += r1) {+ const uint16_t* w = take();+ if (w == nullptr) return false;+ if (!gemm(h, w, in1, un_, in1, y_ + o0, CUDA_R_32F, g.out, L, in1, std::min(r1, g.out - o0), 1.0f, err))+ return false;+ done();+ }+ bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al1_b_, g.out, 1, a16_);+ for (int o0 = 0; o0 < g.out; o0 += r2) {+ const uint16_t* w = take();+ if (w == nullptr) return false;+ if (!gemm(h, w, g.out, a16_, g.out, y_ + o0, CUDA_R_32F, g.out, L, g.out, std::min(r2, g.out - o0), 1.0f, err))+ return false;+ done();+ }+ bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al2_b_, g.out, 0, outb_);+ rows.resize((size_t) L * g.out);+ VK(cudaMemcpyAsync(rows.data(), outb_, rows.size() * 2, cudaMemcpyDeviceToHost, s));+ VK(cudaStreamSynchronize(s));+ VK(cudaGetLastError());+ return true;+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_vision.cuh b/src/dsv41/ds4_vision.cuhnew file mode 100644index 000000000..e3bcb45e2--- /dev/null+++ b/src/dsv41/ds4_vision.cuh@@ -0,0 +1,77 @@+// src/ds4/ds4_vision.cuh - DeepSeek-V4.1's vision tower on one GPU (DeepSeek's inference/vision.py):+// patches (14 x 14 RGB, scaled to [-1, 1]) -> Linear(588 -> 1024)+// 32 blocks: x += attn(rmsnorm(x)) (16 heads of 64, full bidirectional attention over the image, a 2D rotary on q+// and k: the first half of each head's frequencies the patch row, the second its column);+// x += w2(silu(gate) * up) with gate, up = w1(rmsnorm(x))+// rmsnorm, then the aligner: each 3 x 3 block of patches (zero-padded at the right and bottom) as one 9216 row,+// Linear(9216 -> 5120), GELU, Linear(5120 -> 5120): one LLM embedding row per block, in reading order.+// The BF16 weights live in pinned host memory and stream to the GPU a block at a time (two slots, the next block's+// copy under this one's work), so the tower costs the model ~350 MB of the GPU's memory, not ~1 GB; the GEMMs are+// cuBLAS (BF16 in, FP32 accumulation), the residual stream FP32.+#pragma once++#include <cstddef>+#include <cstdint>+#include <functional>+#include <string>+#include <vector>++namespace strata::ds4 {++struct VisionGeo {+ int layers = 0, dim = 1024, heads = 16, inter = 2816, patch = 14, merge = 3, out = 5120;+ float theta = 10000.0f, eps = 1e-6f;+ int max_tokens = 1024; // an image's span in the prompt at most: n_h * (n_w + 1) + 2 tokens+};++class VisionTower {+public:+ /// read(name, shape, dst): the tensor's shape, and (dst not null) its BF16 data into dst; false: not there+ using Reader = std::function<bool(const std::string& name, std::vector<int64_t>& shape, void* dst)>;+ VisionTower() = default;+ VisionTower(const VisionTower&) = delete;+ VisionTower& operator=(const VisionTower&) = delete;+ ~VisionTower();+ bool load(const VisionGeo& g, int device, const Reader& read, std::string& err);+ /// Frees everything load took (a load that failed half way too); loaded() is false after it.+ void release();+ bool loaded() const { return dev_ >= 0; }+ int device() const { return dev_; }+ size_t device_bytes() const { return dev_bytes_; }+ /// The patches an image may have (its aligner grid within max_tokens).+ int max_patches() const { return max_n_; }+ /// rgb: height x width x 3 bytes, both sides multiples of the patch size -> the aligner's rows, BF16+ /// [n_h * n_w][out] in reading order (n_h x n_w: the patch grid over 3, rounded up).+ bool encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,+ std::string& err);++private:+ struct Piece { // a weight the encode streams in: host bits, its size+ const uint16_t* host = nullptr;+ size_t n = 0;+ };+ VisionGeo g_;+ int dev_ = -1, at_ = -1, max_n_ = 0, qb_ = 0, rb_ = 0; // (at_: the device its memory is on)+ size_t dev_bytes_ = 0, slot_n_ = 0;+ std::vector<void*> dallocs_;+ uint16_t* host_ = nullptr; // pinned: every block's [wqkv, wo, w1, w2], then aligner w1, w2+ size_t block_n_ = 0; // a block's weights (BF16 values)+ size_t al1_off_ = 0, al2_off_ = 0; // the aligner's weights in host_+ // resident (small): the patch embedding, the norms and biases+ uint16_t *pe_w_ = nullptr, *pe_b_ = nullptr, *norm_ = nullptr, *al1_b_ = nullptr, *al2_b_ = nullptr;+ std::vector<uint16_t*> n1_, n2_, bqkv_, bo_;+ // work buffers (max_n_ patches)+ uint8_t* rgb_ = nullptr;+ uint16_t *patch_ = nullptr, *xn_ = nullptr, *qkv_ = nullptr, *o_ = nullptr, *mid_ = nullptr, *p_ = nullptr,+ *un_ = nullptr, *a16_ = nullptr, *outb_ = nullptr;+ float *x_ = nullptr, *y_ = nullptr, *s_ = nullptr, *inv_freq_ = nullptr;+ uint16_t* slot_[2] = {nullptr, nullptr};+ void* blas_ = nullptr; // cublasHandle_t+ uint8_t* ws_ = nullptr;+ void *st_ = nullptr, *cs_ = nullptr; // cudaStream_t: the work, the weight copies+ void *copied_[2] = {nullptr, nullptr}, *used_[2] = {nullptr, nullptr}; // cudaEvent_t per slot+ template <class T>+ bool dalloc(T*& p, size_t n, std::string& err);+};++} // namespace strata::ds4diff --git a/src/dsv41/vendor/exl3.cu b/src/dsv41/vendor/exl3.cunew file mode 100644index 000000000..9307e4185--- /dev/null+++ b/src/dsv41/vendor/exl3.cu@@ -0,0 +1,1743 @@+// src/kernels/cuda/exl3.cu - the EXL3 device kernels (the format: include/strata/kernels/exl3.hpp).+//+// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/+// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh), and so does decoding a tile straight into tensor-core fragments+// (its small-m GEMV and GEMM); the kernels around them are shaped for this engine's decode:+//+// * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;+// warp w takes tile column w through every 16-row slice, decoding 8 weights per lane (one fragment: 2 columns x+// 4 rows) - on sm_80 and later straight into one mma.m16n8k16 a tile with up to 8 activations as its columns,+// F32 accumulation; before sm_80 (and on HIP) F32 FMAs on the CUDA cores;+// * the input rotation runs per block, 128 inputs at a time (F32 or q8_1 in, FP16 in shared memory), so the+// engine's activation buffers are used as they are; the next 128 inputs' tile words load while these decode;+// * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over+// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in+// split order (deterministic) and runs the output rotation.+//+// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy+// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The+// prompt path below is this file's on every GPU.+#include "strata/kernels/exl3.hpp"++#include <cuda_fp16.h>+#include <cuda_runtime.h>++#include <algorithm>+#include <atomic>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <map>+#include <mutex>+#include <utility>++#if !defined(STRATA_USE_HIP)+namespace strata::kernels::exl3::legacy { // exl3_legacy.cu: v1.1.8's decode kernels+int launch_errors();+bool prepare(cudaStream_t s);+bool several_ok();+bool mv(const MvJob* jobs, int n, cudaStream_t s);+void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s);+void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s);+bool win_experts(const WinArgs& a, cudaStream_t s);+} // namespace strata::kernels::exl3::legacy+#define EXL3_LEGACY(call) \+ if (use_legacy()) return legacy::call+#else+#define EXL3_LEGACY(call) (void) 0+#endif++namespace strata::kernels::exl3 {+namespace {++constexpr int kThreads = 256, kWarps = 8;+constexpr int kCols = 128; // outputs per block: 8 tiles, one rotation block+constexpr int kMaxKR = 4096; // inputs per block at most (mv's split)+// a routed expert's widths at most (moe_* / win_*: a block streams its inputs through block_gemv's two 128-value chunk+// buffers, so the shared memory does not grow with them; DeepSeek-V4.1's n_embd is 5120)+constexpr int kMaxMoeK = 8192;+constexpr float kHad = 0.088388347648f; // 1 / sqrt(128)++struct Q81 { // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8+ __half2 ds;+ int8_t qs[32];+};++std::atomic<int> g_launch_errors{0};+thread_local const int* g_skip = nullptr; // set_skip: the calling thread's flag (nullptr: none)+void launch_check(const char* what) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ }+}++// ---------------------------------------------------------------- the codebooks+__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {+#if defined(STRATA_USE_HIP)+ return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);+#else+ return __dp4a(x, 0x01010101u, acc);+#endif+}++// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)+template <int cb>+__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {+ if constexpr (cb == 2) {+ const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);+ const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);+ const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));+ return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));+ } else {+ uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;+ uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;+ x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;+ x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;+ const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),+ __ushort_as_half((unsigned short) (x1 & 0xffffu)));+ const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),+ __ushort_as_half((unsigned short) (x1 >> 16)));+ return __hadd2(lo, hi);+ }+}+template <int cb>+__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {+ return __half22float2(decode2h<cb>(s0, s1));+}++__device__ __forceinline__ uint32_t h2u(__half2 h) { return *reinterpret_cast<uint32_t*>(&h); }+__device__ __forceinline__ __half2 u2h(uint32_t u) { return *reinterpret_cast<__half2*>(&u); }++// (hi : lo) >> s, the low 32 bits+__device__ __forceinline__ uint32_t fsh(uint32_t lo, uint32_t hi, int s) {+ return (uint32_t) ((((uint64_t) hi << 32) | (uint64_t) lo) >> s);+}++// Step p's state is the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),+// wrapping around the tile. A lane decodes its 8 consecutive steps lane * 8 .. + 7 together, the windows of a group+// of steps cut from one or two words with funnel shifts - exllamav3's grouping per K (exl3_dq.cuh: dq8_aligned_*bits,+// dq8, dq4, dq2x2), which the steps' fixed offsets keep within two words. f[i]: steps 2i, 2i + 1 as FP16 values.+// Step lane * 8 + j sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column (lane >> 2) + 8 (j >> 2).+template <int cb>+__device__ __forceinline__ void decode_states(const uint32_t (&s)[8], uint32_t (&f)[4]) {+#pragma unroll+ for (int i = 0; i < 4; ++i) f[i] = h2u(decode2h<cb>(s[2 * i] & 0xffffu, s[2 * i + 1] & 0xffffu));+}+// K = 4: the lane's steps end in its own word b (word lane), the first three reach into a (word lane - 1)+__device__ __forceinline__ void states4(uint32_t a, uint32_t b, uint32_t (&s)[8]) {+ const uint32_t x = fsh(b, a, 20);+ s[0] = x >> 8;+ s[1] = x >> 4;+ s[2] = x;+ s[3] = b >> 16;+ s[4] = b >> 12;+ s[5] = b >> 8;+ s[6] = b >> 4;+ s[7] = b;+}+// step p .. p + G - 1's windows (G = 4 or 2) from the two words that hold them+template <int K, int G>+__device__ __forceinline__ void states_group(const uint32_t* w, int p, uint32_t* s) {+ constexpr int NW = 8 * K;+ const int b0 = (p + 257) * K - 16, b2 = b0 + (G - 1) * K + 16;+ const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;+ const uint32_t a = w[i0 % NW], b = w[i2 % NW];+#pragma unroll+ for (int j = 0; j < G; ++j) s[G - 1 - j] = fsh(b, a, s2 + j * K);+}+template <int K, int cb>+__device__ __forceinline__ void frag8(const uint32_t* w, int lane, uint32_t (&f)[4]) {+ const int p = lane * 8;+ uint32_t s[8];+ if constexpr (K == 4) {+ states4(w[(lane + 31) & 31], w[lane], s);+ } else if constexpr (K == 1 || K == 2) {+ // the 8 windows overlap in one 32-bit window: steps p + j end 8 - 1 - j K bits apart+ constexpr int NW = 8 * K, per = 32 / (8 * K); // lanes per word+ const int i1 = lane / per, i0 = (i1 + NW - 1) % NW;+ const uint32_t b = fsh(w[i1], w[i0], (per - 1 - lane % per) * 8 * K);+#pragma unroll+ for (int j = 0; j < 8; ++j) s[j] = b >> ((7 - j) * K);+ } else if constexpr (K == 3) {+ // all 8 windows within two words: two funnel shifts, the rest by shifting+ constexpr int NW = 24;+ const int b1 = (p + 257) * 3, b0 = b1 - 16, b2 = b1 + 21;+ const int i0 = b0 >> 5, i2 = (b2 - 1) >> 5, s2 = ((i2 + 1) << 5) - b2;+ const uint32_t a = w[i0 % NW], b = w[i2 % NW];+ const uint32_t hi = fsh(b, a, s2), lo = fsh(b, a, s2 + 12);+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ s[7 - j] = hi >> (3 * j);+ s[3 - j] = lo >> (3 * j);+ }+ } else if constexpr (K == 7) {+#pragma unroll+ for (int g = 0; g < 4; ++g) states_group<K, 2>(w, p + 2 * g, s + 2 * g);+ } else { // 5, 6, 8+ states_group<K, 4>(w, p, s);+ states_group<K, 4>(w, p + 4, s + 4);+ }+ decode_states<cb>(s, f);+}++// the lane's 8 steps as F32: v[j] = step lane * 8 + j+template <int K, int cb>+__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {+ uint32_t f[4];+ frag8<K, cb>(w, lane, f);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 d = __half22float2(u2h(f[i]));+ v[2 * i] = d.x;+ v[2 * i + 1] = d.y;+ }+}++// ---------------------------------------------------------------- the rotation+// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)+__device__ __forceinline__ void had128(float* v, int lane) {+ const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];+ v[0] = s0 + s1;+ v[1] = d0 + d1;+ v[2] = s0 - s1;+ v[3] = d0 - d1;+#pragma unroll+ for (int m = 1; m < 32; m <<= 1) {+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float o = __shfl_xor_sync(0xffffffffu, v[i], m);+ v[i] = (lane & m) ? o - v[i] : v[i] + o;+ }+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] *= kHad;+}++__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }++// the activation a warp rotates into its block's FP16 chunks: F32 (f) or the engine's q8_1 blocks (q)+struct Xin {+ const float* f = nullptr;+ const Q81* q = nullptr;+};++// the 128-input chunk of x from k0 on: the lane's 4 values and their suh ...+__device__ __forceinline__ void chunk_load(const Xin& x, const uint16_t* __restrict__ suh, int k0, int lane,+ float (&v)[4], float (&s)[4]) {+ const int base = k0 + lane * 4;+ if (x.f != nullptr) {+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = x.f[base + i];+ } else {+ const Q81* b = x.q + base / 32;+ const float d = __low2float(b->ds);+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) s[i] = h2f(suh[base + i]);+}+// ... and H(x * suh) of the chunk, FP16, into dst[0..127]+__device__ __forceinline__ void chunk_store(float (&v)[4], const float (&s)[4], int lane, __half* dst) {+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] *= s[i];+ had128(v, lane);+ __half2* d = reinterpret_cast<__half2*>(dst + lane * 4);+ d[0] = __floats2half2_rn(v[0], v[1]);+ d[1] = __floats2half2_rn(v[2], v[3]);+}++// ---------------------------------------------------------------- the block GEMV+// A block owns 128 outputs - 8 tiles, warp w the tile column tc0 + w through all of the block's inputs [k0, k1) - for+// nc <= kTok activations at once. Per 128-input chunk, warp c (< nc) scales activation c's chunk by suh and rotates+// it into FP16 (xb, two buffers), then each warp decodes its tile of each of the chunk's 8 slices and multiplies it+// with every activation: on sm_80 and later as one mma.m16n8k16 a tile (the decoded tile is the A operand - a lane's 8+// steps are its A fragment - and the activations are the B operand's columns, as exllamav3's small-m GEMV and its+// tensor-core GEMM decode tiles straight into fragments), F32 accumulation; elsewhere on the CUDA cores, in F32. An+// activation's sums do not depend on the other columns, so a window's token gets the token path's bits. The next+// chunk's tile words are loaded while this one is decoded. out[c * 128 + i]: activation c's sum for output i. Ends+// with a __syncthreads (out complete).+constexpr int kTok = 8; // activations a block takes at once (the MMA's n)++#if !defined(STRATA_USE_HIP) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800+#define EXL3_TC 1+#endif++#if !defined(EXL3_TC)+// a lane's 4 weights of one column times the activation's 4 inputs, added to a (the CUDA-core product)+__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {+ return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));+}+#else+__device__ __forceinline__ void mma_f32(float (&c)[4], uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,+ uint32_t b0, uint32_t b1) {+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "+ "{%0,%1,%2,%3};\n"+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])+ : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));+}+#endif++// a tile's words, and a lane's share of them (word lane + 32 r)+template <int K>+struct Words {+ static constexpr int W = 8 * K, R = (W + 31) / 32;+};++// the warp's tile of each of a chunk's 8 slices (slice s at tile0 + s sstride)+template <int K>+__device__ __forceinline__ void load_tiles(const uint32_t* __restrict__ tile0, size_t sstride, int lane,+ uint32_t (&w)[8][Words<K>::R]) {+ constexpr int W = Words<K>::W, R = Words<K>::R;+#pragma unroll+ for (int s = 0; s < 8; ++s)+#pragma unroll+ for (int r = 0; r < R; ++r) {+ const int i = lane + 32 * r;+ w[s][r] = (W % 32 == 0 || i < W) ? __ldg(tile0 + (size_t) s * sstride + i) : 0u;+ }+}++// one slice's fragment from the lane's words: K = 4 straight from the registers (the lane's steps end in its own word,+// the first three reach into the previous lane's), other K through the warp's stage (8 K words)+template <int K, int cb>+__device__ __forceinline__ void slice_frags(const uint32_t (&w)[Words<K>::R], uint32_t* st, int lane, uint32_t (&f)[4]) {+ if constexpr (K == 4) {+ uint32_t s[8];+ states4(__shfl_sync(0xffffffffu, w[0], (lane + 31) & 31), w[0], s);+ decode_states<cb>(s, f);+ } else {+ constexpr int W = Words<K>::W, R = Words<K>::R;+ __syncwarp();+#pragma unroll+ for (int r = 0; r < R; ++r)+ if (W % 32 == 0 || lane + 32 * r < W) st[lane + 32 * r] = w[r];+ __syncwarp();+ frag8<K, cb>(st, lane, f);+ }+}++// the blocks an SM keeps of the block_gemv kernels (their registers), as measured on an RTX 3090 and an RTX 5060 Ti:+// the routed experts four at K <= 4 (a token's 8 experts, 256 blocks, are one wave on 82 SMs), two at wider codes;+// mv two, three at K > 4 on sm_80 .. sm_90 (sm_120 spills at the 80 registers that takes). Before sm_80 (the CUDA+// cores' product, more registers) two; HIP reads the second bound differently and keeps its default.+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800+template <int K>+constexpr int kMinBlocksX = K <= 4 ? 4 : 2;+#else+template <int K>+constexpr int kMinBlocksX = 2;+#endif+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 && __CUDA_ARCH__ < 1000+template <int K>+constexpr int kMinBlocksMv = K <= 4 ? 2 : 3;+#else+template <int K>+constexpr int kMinBlocksMv = 2;+#endif+#if defined(STRATA_USE_HIP)+#define EXL3_BOUNDS(n) __launch_bounds__(kThreads)+#else+#define EXL3_BOUNDS(n) __launch_bounds__(kThreads, n)+#endif++// the shared memory of a block_gemv block+template <int K>+struct BlockSmem {+ __half xb[2 * kTok * kCols]; // two chunk buffers, kTok activations each+ uint32_t stage[kWarps * 8 * K]; // a tile per warp (K != 4)+ float out[kTok * kCols];+};++template <int K, int cb>+__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int k0, int k1, const Xin& x,+ const uint16_t* __restrict__ suh, int nc, BlockSmem<K>& sm) {+ constexpr int W = Words<K>::W, R = Words<K>::R;+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;+ const size_t sstride = (size_t) ld * W;+ const uint32_t* tile = tr + ((size_t) (k0 / 16) * ld + tc0 + warp) * W;+ uint32_t* st = sm.stage + warp * W;+ const bool rot = warp < nc;+ const int nq = (k1 - k0) / kCols;+ uint32_t wc[8][R], wn[8][R];+ float xv[4], xs4[4];+ load_tiles<K>(tile, sstride, lane, wc);+ if (rot) chunk_load(x, suh, k0, lane, xv, xs4);+#if defined(EXL3_TC)+ float c[4] = {0.0f, 0.0f, 0.0f, 0.0f};+#else+ float acc[kTok][2];+#pragma unroll+ for (int cc = 0; cc < kTok; ++cc) acc[cc][0] = acc[cc][1] = 0.0f;+#endif+ for (int q = 0; q < nq; ++q) {+ __half* xq = sm.xb + (q & 1) * (kTok * kCols);+ if (rot) chunk_store(xv, xs4, lane, xq + warp * kCols);+ __syncthreads();+ if (q + 1 < nq) {+ load_tiles<K>(tile + (size_t) (q + 1) * 8 * sstride, sstride, lane, wn);+ if (rot) chunk_load(x, suh, k0 + (q + 1) * kCols, lane, xv, xs4);+ }+#pragma unroll+ for (int s = 0; s < 8; ++s) {+ uint32_t f[4];+ slice_frags<K, cb>(wc[s], st, lane, f);+#if defined(EXL3_TC)+ uint32_t b0 = 0u, b1 = 0u;+ if (g < nc) {+ const __half* xr = xq + g * kCols + s * 16 + 2 * t;+ b0 = *reinterpret_cast<const uint32_t*>(xr);+ b1 = *reinterpret_cast<const uint32_t*>(xr + 8);+ }+ // A: rows (outputs) g / g + 8, columns (inputs) 2t.. / 2t + 8..: steps 0,1 / 4,5 / 2,3 / 6,7+ mma_f32(c, f[0], f[2], f[1], f[3], b0, b1);+#else+ float v[8];+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 d = __half22float2(u2h(f[i]));+ v[2 * i] = d.x;+ v[2 * i + 1] = d.y;+ }+#pragma unroll+ for (int cc = 0; cc < kTok; ++cc)+ if (cc < nc) {+ const __half* xr = xq + cc * kCols + s * 16 + 2 * t;+ const float x0 = __half2float(xr[0]), x1 = __half2float(xr[1]);+ const float x2 = __half2float(xr[8]), x3 = __half2float(xr[9]);+ acc[cc][0] = dot4_acc(acc[cc][0], v, x0, x1, x2, x3);+ acc[cc][1] = dot4_acc(acc[cc][1], v + 4, x0, x1, x2, x3);+ }+#endif+ }+#pragma unroll+ for (int s = 0; s < 8; ++s)+#pragma unroll+ for (int r = 0; r < R; ++r) wc[s][r] = wn[s][r];+ }+ const int o = warp * 16 + g;+#if defined(EXL3_TC)+ // c: the tile's outputs g (c[0], c[1]) and g + 8 (c[2], c[3]) for activations 2t (c[0], c[2]) and 2t + 1+ if (2 * t < nc) {+ sm.out[(2 * t) * kCols + o] = c[0];+ sm.out[(2 * t) * kCols + o + 8] = c[2];+ }+ if (2 * t + 1 < nc) {+ sm.out[(2 * t + 1) * kCols + o] = c[1];+ sm.out[(2 * t + 1) * kCols + o + 8] = c[3];+ }+#else+ // the 4 lanes of one g hold parts of the same two outputs+#pragma unroll+ for (int cc = 0; cc < kTok; ++cc)+ if (cc < nc) {+#pragma unroll+ for (int h = 0; h < 2; ++h) {+ acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 1);+ acc[cc][h] += __shfl_xor_sync(0xffffffffu, acc[cc][h], 2);+ }+ if (t == 0) {+ sm.out[cc * kCols + o] = acc[cc][0];+ sm.out[cc * kCols + o + 8] = acc[cc][1];+ }+ }+#endif+ __syncthreads();+}++// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs+__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,+ float* y) {+ const int lane = threadIdx.x & 31;+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int c = lane * 4 + i;+ y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);+ }+}++// ---------------------------------------------------------------- mv+struct KJob {+ const uint32_t* tr;+ const uint16_t* suh;+ const uint16_t* svh;+ const float* xf;+ const Q81* xq;+ float* y;+ const float* bias;+ float* part; // nc x S x n partial sums (S > 1)+ unsigned* cnt; // n / 128 counters, zero between launches+ float alpha;+ int k, n, ld, S, kr;+ int nc; // the activations (1..kTok), xf / xq / y advanced by these per activation+ size_t xf_cs, xq_cs, y_cs; // (floats, bytes, floats)+};+struct KBatch {+ KJob j[kMaxMvJobs];+ int blk_end[kMaxMvJobs];+ int n;+ const int* skip; // set_skip's flag: non-zero = a pipelined window's token past a wrong draft (nullptr: none)+};++// a launch's blocks: its jobs' (output block, input split) pairs, each for all of the job's nc activations at once+// (block_gemv). S > 1: the partial sums go to the scratch and the last block of an output block adds them up in split+// order (deterministic) and runs the epilogue+template <int K, int cb>+__global__ void EXL3_BOUNDS(kMinBlocksMv<K>) mv_kernel(const KBatch b) {+ __shared__ __align__(16) BlockSmem<K> sm;+ __shared__ int last;+ int ji = 0;+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;+ const KJob& J = b.j[ji];+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);+ const int g = local / J.S, sp = local % J.S;+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);+ const int warp = threadIdx.x >> 5;+ // a skipped token (set_skip): no work. The flag changes only between launches (skip_gate, on the same stream), so+ // every block of the launch returns and no split counter moves+ if (b.skip != nullptr && *b.skip) return;+ Xin x;+ if (warp < J.nc) {+ if (J.xf != nullptr) x.f = J.xf + warp * J.xf_cs;+ else x.q = (const Q81*) ((const uint8_t*) J.xq + warp * J.xq_cs);+ }+ block_gemv<K, cb>(J.tr, J.ld, g * 8, k0, k1, x, J.suh, J.nc, sm);+ const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;+ if (J.S == 1) {+ if (warp < J.nc)+ epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);+ return;+ }+ if (threadIdx.x < kCols)+ for (int c = 0; c < J.nc; ++c)+ J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.out[c * kCols + threadIdx.x];+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < kCols)+ for (int c = 0; c < J.nc; ++c) {+ float s = 0.0f;+ for (int i = 0; i < J.S; ++i)+ s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];+ sm.out[c * kCols + threadIdx.x] = s;+ }+ __syncthreads();+ if (warp < J.nc)+ epilogue(sm.out + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);+ if (threadIdx.x == 0) J.cnt[g] = 0u;+}++// ---------------------------------------------------------------- the routed experts+struct KMoe {+ const unsigned long long* plan_ptr;+ const float* plan_w;+ const float* cpu_part;+ const int* cpu_flag;+ const float* x;+ float* h; // [k][n_ff]: the gate's values, then the swiglu+ float* u; // [k][n_ff]: the up's values+ const float* sh_out;+ float* out;+ float* part; // [k][n_embd]+ unsigned* cnt; // n_embd / 128+ unsigned* cnt_gu; // k x n_ff / 128: the gate / up pairs+ size_t suh[3], svh[3], tr[3];+ int k, n_embd, n_ff;+ float limit;+ const int* skip; // set_skip's flag (nullptr: none)+};++// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window+// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...+__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {+ const int lane = threadIdx.x & 31;+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));+}+// ... the clamped swiglu ...+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {+ if (limit > 0.0f) {+ gv = fminf(gv, limit);+ uv = fminf(fmaxf(uv, -limit), limit);+ }+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);+}+// ... the second block of a (gate, up) pair to finish (its counter back at zero for the next launch) ...+__device__ __forceinline__ bool pair_last(unsigned* cnt, int& last) {+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(cnt, 1u) == 1u;+ __syncthreads();+ if (!last) return false;+ __threadfence();+ if (threadIdx.x == 0) *cnt = 0u;+ return true;+}+// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU+// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {+ float m = 0.0f;+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);+ return __fadd_rn(m, sh);+}++// grid (n_ff / 128, k, 2): expert blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values; the pair's+// second block to finish turns them into the clamped swiglu, in h+template <int K, int cb>+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_gate_up_kernel(const KMoe a) {+ __shared__ __align__(16) BlockSmem<K> sm;+ __shared__ int last;+ const int ei = blockIdx.y, g = blockIdx.x, r = blockIdx.z;+ const unsigned long long ptr = a.plan_ptr[ei];+ if (ptr == 0ull) return;+ if (a.skip != nullptr && *a.skip) return; // a skipped token (set_skip, as mv_kernel)+ const uint8_t* blob = (const uint8_t*) ptr;+ const int warp = threadIdx.x >> 5;+ Xin x;+ x.f = a.x;+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,+ (const uint16_t*) (blob + a.suh[r]), 1, sm);+ const size_t row = (size_t) ei * a.n_ff + g * kCols;+ if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[r]) + g * kCols, (r == 0 ? a.h : a.u) + row);+ if (!pair_last(a.cnt_gu + ei * (a.n_ff / kCols) + g, last)) return;+ if (threadIdx.x < kCols)+ a.h[row + threadIdx.x] = swiglu(*(volatile const float*) &a.h[row + threadIdx.x],+ *(volatile const float*) &a.u[row + threadIdx.x], a.limit);+}++// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds+// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert+template <int K, int cb>+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) moe_down_kernel(const KMoe a) {+ __shared__ __align__(16) BlockSmem<K> sm;+ __shared__ int last;+ const int ei = blockIdx.y, g = blockIdx.x;+ const unsigned long long ptr = a.plan_ptr[ei];+ const int warp = threadIdx.x >> 5;+ float* part = a.part + (size_t) ei * a.n_embd + g * kCols;+ if (a.skip != nullptr && *a.skip) return; // a skipped token (set_skip, as mv_kernel)+ if (ptr != 0ull) {+ const uint8_t* blob = (const uint8_t*) ptr;+ Xin x;+ x.f = a.h + (size_t) ei * a.n_ff;+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,+ (const uint16_t*) (blob + a.suh[2]), 1, sm);+ if (warp == 0) block_out(sm.out, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);+ } else if (threadIdx.x < kCols) {+ part[threadIdx.x] = 0.0f;+ }+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < kCols) {+ const int r = g * kCols + threadIdx.x;+ float p[8];+ for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];+ a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,+ a.sh_out != nullptr ? a.sh_out[r] : 0.0f);+ }+ if (threadIdx.x == 0) a.cnt[g] = 0u;+}++// ---------------------------------------------------------------- a verify window's routed experts+// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at+// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kTok of its tokens at a time (as+// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token+// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit+struct KWin {+ const unsigned long long* grp_ptr;+ const int *grp_start, *grp_ent, *n_groups;+ const unsigned long long* plan_ptr;+ const float* plan_w;+ const float* cpu_part;+ size_t cpu_stride;+ const int* cpu_flag;+ const float* x;+ size_t x_stride;+ float* h; // [T * 8][n_ff]: the gate's values, then the swiglu+ float* u; // [T * 8][n_ff]: the up's values+ float* part; // [T * 8][n_embd]+ const float* sh_out; // [T][n_embd]+ float* out;+ size_t out_stride;+ unsigned* cnt_gu; // groups x n_ff / 128: the gate / up pairs+ size_t suh[3], svh[3], tr[3];+ int k, n_embd, n_ff;+ float limit;+};++// grid (n_ff / 128, groups at most, 2): group blockIdx.y's gate (z 0, into h) or up (z 1, into u) for 128 h values of+// each of its entries, kTok entries a pass; the pair's second block to finish turns them into the clamped swiglu+template <int K, int cb>+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_gu_kernel(const KWin a) {+ __shared__ __align__(16) BlockSmem<K> sm;+ __shared__ int last;+ const int grp = blockIdx.y, g = blockIdx.x, r = blockIdx.z;+ if (grp >= *a.n_groups) return;+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];+ const int warp = threadIdx.x >> 5;+ const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];+ float* dst = r == 0 ? a.h : a.u;+ for (int mm = m0; mm < m1; mm += kTok) {+ const int nc = min(kTok, m1 - mm);+ Xin x;+ if (warp < nc) x.f = a.x + (size_t) (a.grp_ent[mm + warp] >> 3) * a.x_stride;+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd, x,+ (const uint16_t*) (blob + a.suh[r]), nc, sm);+ if (warp < nc)+ block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,+ dst + (size_t) a.grp_ent[mm + warp] * a.n_ff + g * kCols);+ __syncthreads();+ }+ if (!pair_last(a.cnt_gu + grp * (a.n_ff / kCols) + g, last)) return;+ if (threadIdx.x < kCols)+ for (int m = m0; m < m1; ++m) {+ const size_t row = (size_t) a.grp_ent[m] * a.n_ff + g * kCols + threadIdx.x;+ a.h[row] = swiglu(*(volatile const float*) &a.h[row], *(volatile const float*) &a.u[row], a.limit);+ }+}++// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows+template <int K, int cb>+__global__ void EXL3_BOUNDS(kMinBlocksX<K>) win_down_kernel(const KWin a) {+ __shared__ __align__(16) BlockSmem<K> sm;+ const int grp = blockIdx.y, g = blockIdx.x;+ if (grp >= *a.n_groups) return;+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];+ const int warp = threadIdx.x >> 5;+ const int m0 = a.grp_start[grp], m1 = a.grp_start[grp + 1];+ for (int mm = m0; mm < m1; mm += kTok) {+ const int nc = min(kTok, m1 - mm);+ Xin x;+ if (warp < nc) x.f = a.h + (size_t) a.grp_ent[mm + warp] * a.n_ff;+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff, x,+ (const uint16_t*) (blob + a.suh[2]), nc, sm);+ if (warp < nc)+ block_out(sm.out + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,+ a.part + (size_t) a.grp_ent[mm + warp] * a.n_embd + g * kCols);+ __syncthreads();+ }+}++// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)+__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {+ const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;+ float p[8];+ for (int i = 0; i < a.k; ++i) {+ const int e = t * 8 + i;+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;+ }+ a.out[(size_t) t * a.out_stride + r] =+ combine(a.plan_w + t * 8, p, a.k,+ a.cpu_flag != nullptr && a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.cpu_stride + r : nullptr,+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);+}++__global__ void win_wait_kernel(const unsigned* p, unsigned v) {+ if (threadIdx.x != 0) return;+ while (*(volatile const unsigned*) p != v) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700+ __nanosleep(200);+#endif+ }+ __threadfence_system();+}++// ---------------------------------------------------------------- reconstruction+// grid (n / 128, k / 128): the 128 x 128 block of W = diag(suh) H W' H diag(svh), written transposed (dst[c][r])+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) recon_rows_kernel(const uint32_t* __restrict__ tr, int ld,+ const uint16_t* __restrict__ suh,+ const uint16_t* __restrict__ svh, int k, uint16_t* dst,+ bool bf16) {+ constexpr int W = 8 * K;+ constexpr int P = kCols + 2; // padded row (halves)+ __shared__ __half tile[kCols * P];+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ const int nb = blockIdx.x, kb = blockIdx.y;+ // decode: warp w takes the 16-row slices w (8 tiles each)+ for (int ks = warp; ks < 8; ks += kWarps) {+ const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;+ for (int t = 0; t < 8; ++t) {+ float v[8];+ decode8<K, cb>(src + t * W, lane, v);+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);+ const int c = t * 16 + (lane >> 2) + 8 * (j >> 2);+ tile[r * P + c] = __float2half_rn(v[j]);+ }+ }+ }+ __syncthreads();+ // H along the outputs (each row of 128 is one block)+ for (int r = warp; r < kCols; r += kWarps) {+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[r * P + lane * 4 + i]);+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) tile[r * P + lane * 4 + i] = __float2half_rn(v[i]);+ }+ __syncthreads();+ // H along the inputs, the scales, the transposed store+ for (int c = warp; c < kCols; c += kWarps) {+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = __half2float(tile[(lane * 4 + i) * P + c]);+ had128(v, lane);+ const float sv = h2f(svh[nb * kCols + c]);+ uint16_t* d = dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float w = v[i] * sv * h2f(suh[kb * kCols + lane * 4 + i]);+ if (bf16) {+ const uint32_t u = __float_as_uint(w);+ d[i] = (uint16_t) ((u + 0x7fffu + ((u >> 16) & 1u)) >> 16); // round to nearest even+ } else {+ d[i] = __half_as_ushort(__float2half_rn(w));+ }+ }+ }+}++// grid (n / 128, k / 16): W' row-major, one tile per warp+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) recon_inner_kernel(const uint32_t* __restrict__ tr, int ld, int n,+ uint16_t* dst) {+ constexpr int W = 8 * K;+ const int lane = threadIdx.x & 31, t = threadIdx.x >> 5;+ const int tc = blockIdx.x * 8 + t, ks = blockIdx.y;+ float v[8];+ decode8<K, cb>(tr + ((size_t) ks * ld + tc) * W, lane, v);+#pragma unroll+ for (int j = 0; j < 8; ++j) {+ const int r = ks * 16 + (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);+ const int c = tc * 16 + (lane >> 2) + 8 * (j >> 2);+ dst[(size_t) r * n + c] = __half_as_ushort(__float2half_rn(v[j]));+ }+}++// grid (n / 128, k / 128): W' of the 128 x 128 block, transposed (dst[c][r]) - the decoded values alone. Each warp+// decodes its slice's 8 tiles into the block's tile (half2 pairs of rows, conflict-free), then writes 256-byte rows.+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) inner_rows_kernel(const uint32_t* __restrict__ tr, int ld, int k,+ uint16_t* __restrict__ dst) {+ constexpr int W = 8 * K;+ constexpr int P = kCols + 8; // a column's 128 rows, padded (halves; 16-byte rows)+ __shared__ __align__(16) __half tile[kCols * P]; // tile[c * P + r]+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;+ const int nb = blockIdx.x, kb = blockIdx.y;+ for (int ks = warp; ks < 8; ks += kWarps) {+ const uint32_t* src = tr + ((size_t) (kb * 8 + ks) * ld + nb * 8) * W;+ const int r = ks * 16 + 2 * t;+#pragma unroll+ for (int tc = 0; tc < 8; ++tc) {+ uint32_t f[4];+ frag8<K, cb>(src + tc * W, lane, f);+ const int c = tc * 16 + g;+ *reinterpret_cast<uint32_t*>(&tile[c * P + r]) = f[0];+ *reinterpret_cast<uint32_t*>(&tile[c * P + r + 8]) = f[1];+ *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r]) = f[2];+ *reinterpret_cast<uint32_t*>(&tile[(c + 8) * P + r + 8]) = f[3];+ }+ }+ __syncthreads();+ for (int c = warp; c < kCols; c += kWarps)+ *reinterpret_cast<uint2*>(dst + (size_t) (nb * kCols + c) * k + kb * kCols + lane * 4) =+ *reinterpret_cast<const uint2*>(&tile[c * P + lane * 4]);+}++// one warp per 128 values of a row: the rotated FP16 input of an EXL3 matrix (rows_rot_f16) ...+__global__ void __launch_bounds__(kThreads) rows_rot_f16_kernel(const float* __restrict__ x, int ldx,+ const int* __restrict__ idx, int nrows, int ncols,+ const uint16_t* __restrict__ suh, uint16_t* out) {+ const int lane = threadIdx.x & 31, nb = ncols / kCols;+ const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);+ if (w >= nrows * nb) return;+ const int r = w / nb, b = w % nb;+ Xin xin;+ xin.f = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;+ float v[4], sc[4];+ chunk_load(xin, suh, b * kCols, lane, v, sc);+ chunk_store(v, sc, lane, reinterpret_cast<__half*>(out) + (size_t) r * ncols + b * kCols);+}+// ... and its output rotation in place (block_out's arithmetic)+__global__ void __launch_bounds__(kThreads) rows_rot_out_kernel(float* y, int ldy, int nrows, int ncols,+ const uint16_t* __restrict__ svh) {+ const int lane = threadIdx.x & 31, nb = ncols / kCols;+ const int w = blockIdx.x * kWarps + (threadIdx.x >> 5);+ if (w >= nrows * nb) return;+ const int r = w / nb, b = w % nb;+ float* p = y + (size_t) r * ldy + b * kCols;+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = p[lane * 4 + i];+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) p[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[b * kCols + lane * 4 + i]));+}++__global__ void rows_f16_kernel(const float* __restrict__ x, int ldx, const int* __restrict__ idx, int ncols,+ uint16_t* __restrict__ out) {+ const int r = blockIdx.y;+ const float* src = x + (size_t) (idx != nullptr ? idx[r] : r) * ldx;+ uint16_t* dst = out + (size_t) r * ncols;+ for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < ncols; c += gridDim.x * blockDim.x)+ dst[c] = __half_as_ushort(__float2half_rn(src[c]));+}++// ---------------------------------------------------------------- the prompt path's tensor-core GEMMs (sm_80+)+// A block: BM rows x 128 outputs (one output rotation block). Per 128 inputs the block's rows are scaled by suh,+// rotated and stored FP16 in shared memory (the A operands); per 16-row slice each warp decodes its 16 x 16 tile+// straight into mma.m16n8k16's B fragments - a lane's 8 steps ARE the fragment order (steps 0,1 / 2,3 rows 2t+{0,1} /+// 2t+8+{0,1} of column g, steps 4..7 the same of column g + 8) - and multiplies every m-tile of the block with them.+// The output rotation and svh run on the F32 tile in shared memory.+#if !defined(STRATA_USE_HIP)+namespace tc {++constexpr int BM = 32, MT = BM / 16, AP = 136; // rows a block, its m-tiles, the A rows' pitch (halves)++__device__ __forceinline__ void mma16816(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) {+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800+ asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, "+ "{%0,%1,%2,%3};\n"+ : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1));+#endif+}+// the lane's 4 B registers of tile t (words staged in st): b[0..1] columns 0..7, b[2..3] columns 8..15+template <int K, int cb>+__device__ __forceinline__ void tile_frags(const uint32_t* st, int lane, uint32_t* b) {+ uint32_t f[4];+ frag8<K, cb>(st, lane, f);+#pragma unroll+ for (int i = 0; i < 4; ++i) b[i] = f[i];+}++// A[r][j] = FP16(H(x_r * suh)[j]), j < 128, for the block's rows (zero past nrows). x_r: F32 row idx[r] of xf (ldx)+// when xf is given, else FP16 row r of xh (ldx). Every warp, BM / 8 rows each.+__device__ __forceinline__ void load_a(const float* __restrict__ xf, const int* __restrict__ idx,+ const __half* __restrict__ xh, int ldx, int r0, int nrows, int k0,+ const uint16_t* __restrict__ suh, __half* A) {+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ for (int rr = warp; rr < BM; rr += kWarps) {+ float v[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ const int r = r0 + rr;+ if (rr < nrows) {+ const int c = k0 + lane * 4;+ if (xf != nullptr) {+ const float* src = xf + (size_t) idx[r] * ldx + c;+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = src[i];+ } else {+ const __half* src = xh + (size_t) r * ldx + c;+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = __half2float(src[i]);+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[c + i]);+ had128(v, lane);+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) A[rr * AP + lane * 4 + i] = __float2half_rn(v[i]);+ }+}++// the m-tile's A registers for the 16-column slice s of A+__device__ __forceinline__ void a_frags(const __half* A, int mt, int s, int lane, uint32_t* a) {+ const int g = lane >> 2, t = lane & 3;+ const __half* p = A + (mt * 16 + g) * AP + s * 16 + 2 * t;+ a[0] = *reinterpret_cast<const uint32_t*>(p);+ a[1] = *reinterpret_cast<const uint32_t*>(p + 8 * AP);+ a[2] = *reinterpret_cast<const uint32_t*>(p + 8);+ a[3] = *reinterpret_cast<const uint32_t*>(p + 8 * AP + 8);+}++// acc (this warp's 16 columns of the block's BM rows) -> T[BM][128] F32+__device__ __forceinline__ void store_acc(const float (&acc)[MT][2][4], float* T) {+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5, g = lane >> 2, t = lane & 3;+#pragma unroll+ for (int mt = 0; mt < MT; ++mt)+#pragma unroll+ for (int ns = 0; ns < 2; ++ns) {+ const int c = warp * 16 + ns * 8 + 2 * t, r = mt * 16 + g;+ T[r * 128 + c] = acc[mt][ns][0];+ T[r * 128 + c + 1] = acc[mt][ns][1];+ T[(r + 8) * 128 + c] = acc[mt][ns][2];+ T[(r + 8) * 128 + c + 1] = acc[mt][ns][3];+ }+}++struct GArgs {+ const uint8_t* wbase; // expert e's blob at wbase + e * stride (dense: 0, the offsets are pointers)+ size_t stride;+ size_t tr[3], suh[3], svh[3];+ int ld[3]; // tiles per 16-row slice of each matrix+ const int* bounds; // device: expert e's rows [bounds[e], bounds[e + 1]) (null: one matrix, rows [0, m))+ int row0, m; // local row of a sorted row: bounds[e] - row0 + r+ int k, n; // of the matrix (gate/up: n_embd, n_ff)+ const float* xf; // F32 input rows by index (idx[local row]) ...+ const int* idx;+ const __half* xh; // ... or FP16 rows by local row+ int ldx;+ float* y; // F32 output rows (ldy), y = beta y + out ...+ int ldy;+ float beta;+ __half* h; // ... or (gate/up) the swiglu as FP16 rows (ldh)+ int ldh;+ float limit;+};++// one matrix: y = beta y + H(H(x * suh) W') * svh. grid (row blocks, n / 128, experts)+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) gemm_kernel(const GArgs a) {+ constexpr int W = 8 * K;+ __shared__ __align__(16) unsigned char sm[BM * 128 * 4]; // A (BM x AP halves), then the F32 tile+ __shared__ uint32_t stage[kWarps][64];+ __half* A = reinterpret_cast<__half*>(sm);+ float* T = reinterpret_cast<float*>(sm);+ const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;+ const int e0 = a.bounds != nullptr ? a.bounds[e] - a.row0 : 0;+ const int ne = a.bounds != nullptr ? a.bounds[e + 1] - a.bounds[e] : a.m;+ const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);+ if (nrows <= 0) return;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const uint32_t* tr = (const uint32_t*) (blob + a.tr[0]);+ const uint16_t* suh = (const uint16_t*) (blob + a.suh[0]);+ const uint16_t* svh = (const uint16_t*) (blob + a.svh[0]);+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ float acc[MT][2][4] = {};+ for (int k0 = 0; k0 < a.k; k0 += 128) {+ __syncthreads();+ load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, suh, A);+ __syncthreads();+#pragma unroll 1+ for (int s = 0; s < 8; ++s) {+ const int ks = k0 / 16 + s;+ const uint32_t* src = tr + ((size_t) ks * a.ld[0] + g * 8 + warp) * W;+ __syncwarp();+ for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];+ __syncwarp();+ uint32_t b[4];+ tile_frags<K, cb>(stage[warp], lane, b);+#pragma unroll+ for (int mt = 0; mt < MT; ++mt) {+ uint32_t af[4];+ a_frags(A, mt, s, lane, af);+ mma16816(acc[mt][0], af, b[0], b[1]);+ mma16816(acc[mt][1], af, b[2], b[3]);+ }+ }+ }+ __syncthreads();+ store_acc(acc, T);+ __syncthreads();+ for (int rr = warp; rr < nrows; rr += kWarps) {+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = T[rr * 128 + lane * 4 + i];+ had128(v, lane);+ float* yr = a.y + (size_t) (r0 + rr) * a.ldy + g * 128 + lane * 4;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float o = v[i] * h2f(svh[g * 128 + lane * 4 + i]);+ yr[i] = a.beta != 0.0f ? a.beta * yr[i] + o : o;+ }+ }+}++// gate and up (matrices 0 and 1) over the same rows, the clamped swiglu, FP16 h. grid (row blocks, n / 128, experts)+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) gate_up_kernel(const GArgs a) {+ constexpr int W = 8 * K;+ __shared__ __align__(16) unsigned char sm[2 * BM * 128 * 4]; // A_g, A_u, then the two F32 tiles+ __shared__ uint32_t stage[kWarps][64];+ __half* A[2] = {reinterpret_cast<__half*>(sm), reinterpret_cast<__half*>(sm) + BM * AP};+ float* T[2] = {reinterpret_cast<float*>(sm), reinterpret_cast<float*>(sm) + BM * 128};+ const int e = blockIdx.z, rb = blockIdx.x, g = blockIdx.y;+ const int e0 = a.bounds[e] - a.row0, ne = a.bounds[e + 1] - a.bounds[e];+ const int r0 = e0 + rb * BM, nrows = min(BM, ne - rb * BM);+ if (nrows <= 0) return;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ float acc[2][MT][2][4] = {};+ for (int k0 = 0; k0 < a.k; k0 += 128) {+ __syncthreads();+ for (int m = 0; m < 2; ++m)+ load_a(a.xf, a.idx, a.xh, a.ldx, r0, nrows, k0, (const uint16_t*) (blob + a.suh[m]), A[m]);+ __syncthreads();+#pragma unroll 1+ for (int s = 0; s < 8; ++s) {+ const int ks = k0 / 16 + s;+#pragma unroll+ for (int m = 0; m < 2; ++m) {+ const uint32_t* src = (const uint32_t*) (blob + a.tr[m]) + ((size_t) ks * a.ld[m] + g * 8 + warp) * W;+ __syncwarp();+ for (int i = lane; i < W; i += 32) stage[warp][i] = src[i];+ __syncwarp();+ uint32_t b[4];+ tile_frags<K, cb>(stage[warp], lane, b);+#pragma unroll+ for (int mt = 0; mt < MT; ++mt) {+ uint32_t af[4];+ a_frags(A[m], mt, s, lane, af);+ mma16816(acc[m][mt][0], af, b[0], b[1]);+ mma16816(acc[m][mt][1], af, b[2], b[3]);+ }+ }+ }+ }+ __syncthreads();+ store_acc(acc[0], T[0]);+ store_acc(acc[1], T[1]);+ __syncthreads();+ const uint16_t* svg = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[0]) + g * 128;+ const uint16_t* svu = (const uint16_t*) (a.wbase + (size_t) e * a.stride + a.svh[1]) + g * 128;+ for (int rr = warp; rr < nrows; rr += kWarps) {+ float vg[4], vu[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ vg[i] = T[0][rr * 128 + lane * 4 + i];+ vu[i] = T[1][rr * 128 + lane * 4 + i];+ }+ had128(vg, lane);+ had128(vu, lane);+ __half* hr = a.h + (size_t) (r0 + rr) * a.ldh + g * 128 + lane * 4;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ float gv = vg[i] * h2f(svg[lane * 4 + i]), uv = vu[i] * h2f(svu[lane * 4 + i]);+ if (a.limit > 0.0f) {+ gv = fminf(gv, a.limit);+ uv = fminf(fmaxf(uv, -a.limit), a.limit);+ }+ hr[i] = __float2half_rn((gv / (1.0f + expf(-gv))) * uv);+ }+ }+}++} // namespace tc+#endif++// ---------------------------------------------------------------- dispatch+// F(K, cb) for K 1..8 and the three codebooks+template <template <int, int> class F, typename... A>+bool dispatch(int K, int cb, A&&... args) {+#define EXL3_CASE(KK) \+ case KK: \+ if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...); \+ else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...); \+ else F<KK, 0>::run(std::forward<A>(args)...); \+ return true;+ switch (K) {+ EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)+ default: return false;+ }+#undef EXL3_CASE+}++template <int K, int cb> struct RunMv {+ static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }+};+template <int K, int cb> struct RunWin {+ static void run(const KWin& a, int groups, cudaStream_t s) {+ win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups, 2u), kThreads, 0, s>>>(a);+ }+};+template <int K, int cb> struct RunWinDown {+ static void run(const KWin& a, int groups, cudaStream_t s) {+ win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, 0, s>>>(a);+ }+};+template <int K, int cb> struct RunGU {+ static void run(const KMoe& a, cudaStream_t s) {+ moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k, 2u), kThreads, 0, s>>>(a);+ }+};+template <int K, int cb> struct RunDown {+ static void run(const KMoe& a, cudaStream_t s) {+ moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ }+};+template <int K, int cb> struct RunRows {+ static void run(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {+ recon_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.suh, m.svh, m.k, dst, bf16);+ }+};+#if !defined(STRATA_USE_HIP)+template <int K, int cb> struct RunGemm {+ static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) { tc::gemm_kernel<K, cb><<<grid, kThreads, 0, s>>>(a); }+};+template <int K, int cb> struct RunGateUp {+ static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) {+ tc::gate_up_kernel<K, cb><<<grid, kThreads, 0, s>>>(a);+ }+};+#endif+template <int K, int cb> struct RunInnerRows {+ static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {+ inner_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.k, dst);+ }+};+template <int K, int cb> struct RunInner {+ static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {+ recon_inner_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / 16)), kThreads, 0, s>>>(+ m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.n, dst);+ }+};++// ---------------------------------------------------------------- per-stream scratch+constexpr size_t kPartFloats = (size_t) 2 << 20; // split partials: 8 MB+constexpr size_t kCounters = 16384;+constexpr size_t kCntGU = 8192; // the counters from here on: the experts' gate / up pairs+constexpr size_t kHFloats = (size_t) 8 * 8192; // the experts' h+constexpr size_t kUFloats = (size_t) 128 * 4096; // the experts' up values: a window's 16 tokens x 8, n_ff <= 4096+struct Scratch {+ float* part = nullptr;+ unsigned* cnt = nullptr;+ float* h = nullptr;+ float* u = nullptr;+};+std::mutex g_mu;+std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;++Scratch* scratch(cudaStream_t s) {+ int dev = 0;+ cudaGetDevice(&dev);+ // (the last stream this thread used: an engine thread per device launches without the lock)+ thread_local int t_dev = -1;+ thread_local cudaStream_t t_s = nullptr;+ thread_local Scratch* t_sc = nullptr;+ if (t_sc != nullptr && t_dev == dev && t_s == s) return t_sc;+ std::lock_guard<std::mutex> lk(g_mu);+ auto it = g_scratch.find({dev, s});+ if (it != g_scratch.end()) {+ t_dev = dev, t_s = s, t_sc = &it->second;+ return &it->second;+ }+ cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;+ if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {+ std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");+ return nullptr;+ }+ Scratch sc;+ if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||+ cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||+ cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||+ cudaMalloc(&sc.u, kUFloats * sizeof(float)) != cudaSuccess ||+ cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {+ std::fprintf(stderr, "exl3: scratch did not allocate\n");+ return nullptr;+ }+ cudaStreamSynchronize(s);+ return &(g_scratch[{dev, s}] = sc);+}++int sm_count() {+ int dev = 0, sms = 0;+ cudaGetDevice(&dev);+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);+ return std::max(1, sms);+}++// Whether the current device decodes with v1.1.8's kernels (exl3_legacy.cu): NVIDIA before sm_80, where this file's+// tiles run on the CUDA cores and were slower; STRATA_GLM_EXL3_DECODE=legacy | new picks one on any GPU (A/B). The+// token path and the verify windows take the same family on a device, so a window stays bit for bit its token path.+// Asked once per device.+bool use_legacy() {+#if defined(STRATA_USE_HIP)+ return false;+#else+ static std::atomic<int> cache[64]; // 0 not asked, 1 v1.1.8's kernels, 2 this file's+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 64) return false;+ int c = cache[dev].load(std::memory_order_acquire);+ if (c == 0) {+ int major = 0;+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ const char* v = std::getenv("STRATA_GLM_EXL3_DECODE");+ c = v != nullptr && std::strcmp(v, "legacy") == 0 ? 1 : v != nullptr && std::strcmp(v, "new") == 0 ? 2+ : major < 8 ? 1+ : 2;+ if (c == 1) std::fprintf(stderr, "exl3: CUDA%d decodes with v1.1.8's kernels\n", dev);+ cache[dev].store(c, std::memory_order_release);+ }+ return c == 1;+#endif+}++// the blocks of mv_kernel<K> an SM of the current device keeps at once (its registers and shared memory), asked once+// per device and K+template <int K, int cb> struct OccMv {+ static void run(int& n) {+ if (cudaOccupancyMaxActiveBlocksPerMultiprocessor(&n, mv_kernel<K, cb>, kThreads, 0) != cudaSuccess) {+ cudaGetLastError();+ n = 2;+ }+ }+};+int mv_occ(int K) {+ static std::atomic<int> cache[64][9]; // 0: not asked+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 64 || K < 1 || K > 8) return 2;+ int n = cache[dev][K].load(std::memory_order_acquire);+ if (n == 0) {+ n = 2;+ dispatch<OccMv>(K, 2, n);+ n = std::max(1, n);+ cache[dev][K].store(n, std::memory_order_release);+ }+ return n;+}++// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block (the same for every+// activation count, so a job of several sums each activation as a job of one). About two blocks per SM; where an SM+// keeps three or more (`occ`: the RTX 3090's at K > 4) four a SM, down to 128 inputs a block - its decode sets the pace+// there, and the more, shorter blocks balance it over the SMs (RTX 3090, K6 / K5, us: a KDA layer's 8192 x 4096+// output 47.6 -> 45.5, the dense layers' 4096 x 12288 gate 58.9 -> 52.5 and 12288 x 4096 down 59.3 -> 52.2, the+// shared expert's 4096 x 2048 17.4 -> 16.5); with two a SM, a launch that would spill a short second wave (more blocks+// than the SMs keep, less than one and a half times) takes twice the splits (RTX 5060 Ti, K5: that gate 99.4 -> 84.8+// us, that down 100.6 -> 84.9)+// call_groups: the output blocks the call's jobs of this code count together against the target, not the job's own -+// where an SM keeps three or more, all of them (RTX 3090, K6, a KDA layer's q, k and v, three 4096 x 8192 jobs: 1536+// blocks 119.6 us, 384 blocks 105.9); at two a SM only jobs of the same shape (RTX 5060 Ti: those q, k and v 191 ->+// 185 us, a shared expert's gate and up 35.9 -> 35.0; the MLA layer's q_a and kv_a together would lose 8%). Never the+// activations: a window's token gets the token path's split+void mv_split(const Mat& m, int sms, int occ, int& S, int& kr, int call_groups = 0) {+ const bool many = occ >= 3;+ const int groups = std::max(m.n / kCols, call_groups);+ const int kr_min = many ? 128 : 256, target = (many ? 4 : 2) * sms, P = occ * sms;+ S = (m.k + kMaxKR - 1) / kMaxKR;+ while (groups * S < target && m.k / (S * 2) >= kr_min) S *= 2;+ if (!many && groups * S > P && 2 * groups * S < 3 * P && m.k / (S * 2) >= kr_min) S *= 2;+ kr = ((m.k + S - 1) / S + 127) / 128 * 128;+ S = (m.k + kr - 1) / kr;+}++} // namespace++Mat view(const Mat& m, int c0, int nc) {+ Mat v = m;+ v.ld = m.ld > 0 ? m.ld : m.n / 16;+ v.trellis = m.trellis + (size_t) (c0 / 16) * 8 * m.K;+ v.svh = m.svh + c0;+ v.n = nc;+ return v;+}++bool supported(const Mat& m) {+ return m.trellis != nullptr && m.suh != nullptr && m.svh != nullptr && m.k > 0 && m.n > 0 && m.k % 128 == 0 &&+ m.n % 128 == 0 && m.K >= 1 && m.K <= 8 && m.cb >= 0 && m.cb <= 2;+}++void set_skip(const int* flag) { g_skip = flag; }++int mv_splits(const Mat& m) {+ if (!supported(m)) return 0;+ int S = 1, kr = 0;+ mv_split(m, sm_count(), mv_occ(m.K), S, kr);+ return S;+}++int mv_cols(const Mat& m) {+ if (!supported(m)) return 0;+#if !defined(STRATA_USE_HIP)+ if (use_legacy()) return 4; // (v1.1.8's kernels: the runs of four their callers have always used)+#endif+ const int S = mv_splits(m);+ if (S <= 0) return 0;+ if (S == 1) return kTok;+ return (int) std::max<size_t>(1, std::min<size_t>(kTok, kPartFloats / ((size_t) S * m.n)));+}++int launch_errors() {+#if !defined(STRATA_USE_HIP)+ return g_launch_errors.load(std::memory_order_relaxed) + legacy::launch_errors();+#else+ return g_launch_errors.load(std::memory_order_relaxed);+#endif+}++bool prepare(cudaStream_t s) {+ EXL3_LEGACY(prepare(s));+ return scratch(s) != nullptr;+}++bool several_ok() {+ EXL3_LEGACY(several_ok());+ return true;+}++bool mv(const MvJob* jobs, int n, cudaStream_t s) {+ EXL3_LEGACY(mv(jobs, n, s));+ if (n <= 0 || n > kMaxMvJobs) return false;+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ const int sms = sm_count();+ // the launch units: a job's activations in runs of up to kTok (one pass over its weights each)+ struct Unit {+ int j, c0, nc;+ };+ Unit units[kMaxMvJobs * kMaxMvCols];+ int nu = 0;+ const int run = kTok;+ for (int j = 0; j < n; ++j) {+ const Mat& m = *jobs[j].m;+ if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||+ jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {+ std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,+ m.K, m.cb, jobs[j].ncol);+ return false;+ }+ for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};+ }+ // each job's call_groups (mv_split): the output blocks of the call's jobs of its code - of its shape too where an SM+ // keeps fewer than three+ int cg[kMaxMvJobs] = {};+ for (int j = 0; j < n; ++j) {+ const Mat& a = *jobs[j].m;+ const bool many = mv_occ(a.K) >= 3;+ for (int q = 0; q < n; ++q) {+ const Mat& b = *jobs[q].m;+ if (b.K == a.K && b.cb == a.cb && (many || (b.k == a.k && b.n == a.n))) cg[j] += b.n / kCols;+ }+ }+ bool done[sizeof(units) / sizeof(units[0])] = {};+ for (int i = 0; i < nu; ++i) {+ if (done[i]) continue;+ const Mat& m0 = *jobs[units[i].j].m;+ // a launch: the units of one (K, codebook), as many as KBatch and the split scratch take (the next launch on+ // the stream may use the scratch again: this one's last blocks are done with it by then)+ KBatch b{};+ int blocks = 0, nb = 0;+ size_t part_at = 0, cnt_at = 0;+ for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {+ const MvJob& job = jobs[units[u].j];+ const Mat& m = *job.m;+ if (done[u] || m.K != m0.K || m.cb != m0.cb) continue;+ const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;+ int S = 1, kr = 0;+ mv_split(m, sms, mv_occ(m.K), S, kr, cg[units[u].j]);+ const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;+ if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {+ if (nb > 0) continue; // the next launch+ std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);+ return false;+ }+ done[u] = true;+ KJob& J = b.j[nb];+ J.tr = m.trellis;+ J.suh = m.suh;+ J.svh = m.svh;+ J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;+ J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;+ J.y = job.y + c0 * job.y_cs;+ J.bias = job.bias;+ J.alpha = job.alpha;+ J.k = m.k;+ J.n = m.n;+ J.ld = m.ld > 0 ? m.ld : m.n / 16;+ J.S = S;+ J.kr = kr;+ J.part = sc->part + part_at;+ J.cnt = sc->cnt + cnt_at;+ J.nc = nc;+ J.xf_cs = job.xf_cs;+ J.xq_cs = job.xq_cs;+ J.y_cs = job.y_cs;+ part_at += need;+ if (S > 1) cnt_at += (size_t) groups;+ blocks += groups * S;+ b.blk_end[nb++] = blocks;+ }+ b.n = nb;+ b.skip = g_skip;+ if (!dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)) return false;+ }+ launch_check("mv");+ return true;+}++static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||+ a.n_ff > kMaxMoeK || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||+ (size_t) (a.n_embd / kCols) > kCntGU || kCntGU + (size_t) a.k * (a.n_ff / kCols) > kCounters) {+ std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);+ return false;+ }+ m = KMoe{};+ m.plan_ptr = a.plan_ptr;+ m.plan_w = a.plan_w;+ m.cpu_part = a.cpu_part;+ m.cpu_flag = a.cpu_flag;+ m.h = sc->h;+ m.u = sc->u;+ m.part = sc->part;+ m.cnt = sc->cnt;+ m.cnt_gu = sc->cnt + kCntGU;+ for (int r = 0; r < 3; ++r) {+ m.suh[r] = a.lay.suh[r];+ m.svh[r] = a.lay.svh[r];+ m.tr[r] = a.lay.trellis[r];+ }+ m.k = a.k;+ m.n_embd = a.n_embd;+ m.n_ff = a.n_ff;+ m.limit = a.limit;+ m.skip = g_skip;+ return true;+}++void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {+ EXL3_LEGACY(moe_gate_up(a, x, h, s));+ KMoe m;+ if (!moe_args(a, m, s)) return;+ if (a.lay.K[0] != a.lay.K[1]) {+ std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);+ return;+ }+ m.x = x;+ if (h != nullptr) m.h = h;+ if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);+ launch_check("moe_gate_up");+}++void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {+ EXL3_LEGACY(moe_down(a, h, sh_out, out, s));+ KMoe m;+ if (!moe_args(a, m, s)) return;+ if (h != nullptr) m.h = const_cast<float*>(h);+ m.sh_out = sh_out;+ m.out = out;+ if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);+ launch_check("moe_down");+}++bool win_experts(const WinArgs& a, cudaStream_t s) {+ EXL3_LEGACY(win_experts(a, s));+ if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxMoeK ||+ a.n_ff > kMaxMoeK || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {+ std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,+ a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);+ return false;+ }+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)+ if ((size_t) a.T * 8 * a.n_ff > kUFloats || kCntGU + (size_t) groups * (a.n_ff / kCols) > kCounters) {+ std::fprintf(stderr, "exl3 win_experts: %d tokens are more than the scratch takes\n", a.T);+ return false;+ }+ KWin w{};+ w.grp_ptr = a.grp_ptr;+ w.grp_start = a.grp_start;+ w.grp_ent = a.grp_ent;+ w.n_groups = a.n_groups;+ w.plan_ptr = a.plan_ptr;+ w.plan_w = a.plan_w;+ w.cpu_part = a.cpu_part;+ w.cpu_stride = a.cpu_stride != 0 ? a.cpu_stride : (size_t) a.n_embd;+ w.cpu_flag = a.cpu_flag;+ w.x = a.x;+ w.x_stride = a.x_stride;+ w.h = a.h;+ w.u = sc->u;+ w.cnt_gu = sc->cnt + kCntGU;+ w.part = a.part;+ w.sh_out = a.sh_out;+ w.out = a.out;+ w.out_stride = a.out_stride;+ for (int r = 0; r < 3; ++r) {+ w.suh[r] = a.lay.suh[r];+ w.svh[r] = a.lay.svh[r];+ w.tr[r] = a.lay.trellis[r];+ }+ w.k = a.k;+ w.n_embd = a.n_embd;+ w.n_ff = a.n_ff;+ w.limit = a.limit;+ if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||+ !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))+ return false;+ if (a.wait_ptr != nullptr) win_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);+ win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);+ launch_check("win_experts");+ return true;+}++bool mma_supported() {+#if defined(STRATA_USE_HIP)+ return false;+#else+ int dev = 0, major = 0;+ cudaGetDevice(&dev);+ cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);+ return major >= 8;+#endif+}++void set_gate_up(const SetArgs& sa, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh,+ cudaStream_t s) {+#if !defined(STRATA_USE_HIP)+ if (sa.lay.K[0] != sa.lay.K[1]) {+ std::fprintf(stderr, "exl3 set_gate_up: gate K %d != up K %d\n", sa.lay.K[0], sa.lay.K[1]);+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;+ tc::GArgs a{};+ a.wbase = sa.wbase;+ a.stride = sa.stride;+ for (int r = 0; r < 2; ++r) {+ a.tr[r] = sa.lay.trellis[r];+ a.suh[r] = sa.lay.suh[r];+ a.svh[r] = sa.lay.svh[r];+ a.ld[r] = sa.n_ff / 16;+ }+ a.bounds = sa.bounds;+ a.row0 = sa.row0;+ a.k = sa.n_embd;+ a.n = sa.n_ff;+ a.xf = x;+ a.idx = row_idx;+ a.ldx = ldx;+ a.h = (__half*) h16;+ a.ldh = ldh;+ a.limit = sa.limit;+ const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_ff / kCols), (unsigned) sa.n_exp);+ if (!dispatch<RunGateUp>(sa.lay.K[0], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_gate_up: K\n");+ launch_check("set_gate_up");+#else+ (void) sa; (void) x; (void) ldx; (void) row_idx; (void) h16; (void) ldh; (void) s;+#endif+}++void set_down(const SetArgs& sa, const uint16_t* h16, int ldh, float* y, int ldy, cudaStream_t s) {+#if !defined(STRATA_USE_HIP)+ if (sa.n_exp <= 0 || sa.max_rows <= 0) return;+ tc::GArgs a{};+ a.wbase = sa.wbase;+ a.stride = sa.stride;+ a.tr[0] = sa.lay.trellis[2];+ a.suh[0] = sa.lay.suh[2];+ a.svh[0] = sa.lay.svh[2];+ a.ld[0] = sa.n_embd / 16;+ a.bounds = sa.bounds;+ a.row0 = sa.row0;+ a.k = sa.n_ff;+ a.n = sa.n_embd;+ a.xh = (const __half*) h16;+ a.ldx = ldh;+ a.y = y;+ a.ldy = ldy;+ const dim3 grid((unsigned) ((sa.max_rows + tc::BM - 1) / tc::BM), (unsigned) (sa.n_embd / kCols), (unsigned) sa.n_exp);+ if (!dispatch<RunGemm>(sa.lay.K[2], sa.lay.cb, a, grid, s)) std::fprintf(stderr, "exl3 set_down: K\n");+ launch_check("set_down");+#else+ (void) sa; (void) h16; (void) ldh; (void) y; (void) ldy; (void) s;+#endif+}++void gemm_rows(const Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta, cudaStream_t s) {+#if !defined(STRATA_USE_HIP)+ if (rows <= 0) return;+ tc::GArgs a{};+ a.wbase = nullptr;+ a.tr[0] = (size_t) m.trellis;+ a.suh[0] = (size_t) m.suh;+ a.svh[0] = (size_t) m.svh;+ a.ld[0] = m.ld > 0 ? m.ld : m.n / 16;+ a.m = rows;+ a.k = m.k;+ a.n = m.n;+ a.xh = (const __half*) x16;+ a.ldx = ldx;+ a.y = y;+ a.ldy = ldy;+ a.beta = beta;+ const dim3 grid((unsigned) ((rows + tc::BM - 1) / tc::BM), (unsigned) (m.n / kCols), 1);+ if (!supported(m) || !dispatch<RunGemm>(m.K, m.cb, a, grid, s)) {+ std::fprintf(stderr, "exl3 gemm_rows: unsupported matrix\n");+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ launch_check("gemm_rows");+#else+ (void) m; (void) x16; (void) ldx; (void) rows; (void) y; (void) ldy; (void) beta; (void) s;+#endif+}++void reconstruct_rows(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {+ if (!supported(m) || !dispatch<RunRows>(m.K, m.cb, m, dst, bf16, s)) {+ std::fprintf(stderr, "exl3 reconstruct_rows: unsupported matrix\n");+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ launch_check("reconstruct_rows");+}++void reconstruct_inner_rows(const Mat& m, uint16_t* dst, cudaStream_t s) {+ if (!supported(m) || !dispatch<RunInnerRows>(m.K, m.cb, m, dst, s)) {+ std::fprintf(stderr, "exl3 reconstruct_inner_rows: unsupported matrix\n");+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ launch_check("reconstruct_inner_rows");+}++void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, const uint16_t* suh, uint16_t* out,+ cudaStream_t s) {+ if (nrows <= 0) return;+ const long long warps = (long long) nrows * (ncols / kCols);+ rows_rot_f16_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(x, ldx, idx, nrows, ncols, suh,+ out);+ launch_check("rows_rot_f16");+}++void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s) {+ if (nrows <= 0) return;+ const long long warps = (long long) nrows * (ncols / kCols);+ rows_rot_out_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(y, ldy, nrows, ncols, svh);+ launch_check("rows_rot_out");+}++void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {+ if (nrows <= 0) return;+ rows_f16_kernel<<<dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s>>>(x, ldx, idx,+ ncols, out);+ launch_check("rows_f16");+}++void reconstruct_inner(const Mat& m, uint16_t* dst, cudaStream_t s) {+ if (!supported(m) || !dispatch<RunInner>(m.K, m.cb, m, dst, s)) {+ std::fprintf(stderr, "exl3 reconstruct_inner: unsupported matrix\n");+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ launch_check("reconstruct_inner");+}++} // namespace strata::kernels::exl3diff --git a/src/dsv41/vendor/exl3_cpu.cpp b/src/dsv41/vendor/exl3_cpu.cppnew file mode 100644index 000000000..11eca4ebe--- /dev/null+++ b/src/dsv41/vendor/exl3_cpu.cpp@@ -0,0 +1,1269 @@+// src/kernels/cpu/exl3_cpu.cpp - EXL3 routed experts on the CPU (see include/strata/kernels/cpu/exl3_cpu.hpp).+//+// The vector kernels follow exllamav3's CPU expert kernel (MIT, turboderp - exllamav3_ext/cpu/moe_mul1.cpp, its+// AVX-512 BW and AVX2 tiers): a tile row's 16 states come out of the tile's words with cross-lane permutes and a+// per-lane funnel shift from compile-time tables, the codebook product's bytes are added with vpmaddubsw against ones+// and vpmaddwd against ones. Unlike there, the activations stay F32 (no int8 rows) and the affine offset is the+// device's FP16 constant, so the CPU lane computes what the device kernels compute up to the FP16 rounding of each+// weight. The AVX-512 and AVX2 tiers do the same FMAs in the same order: they give the same bits (the bands,+// the split-K pieces, the GEMM's sums and the vectorised transforms alike).+#include "strata/kernels/cpu/exl3_cpu.hpp"++#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/f16_bits.hpp"++#include <algorithm>+#include <array>+#include <cmath>+#include <cstdlib>+#include <cstring>+#include <utility>+#include <vector>++#if defined(__x86_64__) || defined(_M_X64)+#define EXL3_X86 1+#include <immintrin.h>+#if defined(_MSC_VER)+#include <intrin.h>+#else+#include <cpuid.h>+#endif+#endif++#if defined(EXL3_X86) && (defined(__GNUC__) || defined(__clang__))+#define EXL3_AVX512 __attribute__((target("avx512f,avx512bw,fma")))+#define EXL3_AVX2 __attribute__((target("avx2,fma,f16c")))+#define EXL3_AVX2_INLINE __attribute__((target("avx2,fma,f16c"), always_inline)) inline+#elif defined(_MSC_VER)+#define EXL3_AVX512+#define EXL3_AVX2+#define EXL3_AVX2_INLINE __forceinline+#else+#define EXL3_AVX512+#define EXL3_AVX2+#define EXL3_AVX2_INLINE inline+#endif++namespace strata::kernels::cpu {+namespace {++constexpr uint32_t kMul1 = 0x83DCD12Du;++float k_inv() { return f32_from_f16(0x1eee); }+float k_zero() { return 1024.0f * f32_from_f16(0x1eee) + f32_from_f16(0xc931); } // w = s * k_inv + k_zero++// the 128-point Sylvester transform over sqrt(128), in place+void had128(float* v) {+ for (int h = 1; h < 128; h <<= 1)+ for (int i = 0; i < 128; i += 2 * h)+ for (int j = i; j < i + h; ++j) {+ const float a = v[j], b = v[j + h];+ v[j] = a + b;+ v[j + h] = a - b;+ }+ const float s = 0.088388347648f;+ for (int i = 0; i < 128; ++i) v[i] *= s;+}++// step p of a tile: row (p/8 % 4) * 2 + (p & 1) + 8 (p/2 & 1), column p/32 + 8 (p/4 & 1); inv[row * 16 + col] = p+constexpr std::array<uint16_t, 256> make_inv() {+ std::array<uint16_t, 256> inv{};+ for (int p = 0; p < 256; ++p) {+ const int lane = p >> 3, j = p & 7;+ const int r = (lane & 3) * 2 + (j & 1) + 8 * ((j >> 1) & 1);+ const int c = (lane >> 2) + 8 * (j >> 2);+ inv[(size_t) (r * 16 + c)] = (uint16_t) p;+ }+ return inv;+}++// a tile row's 16 states: column c's 16-bit window ends at bit b1 = (p + 1 + 256) K of the tile's MSB-first stream;+// its words i0 (the window's first bit) and i1 (its last), and the right shift of (w[i0] : w[i1]) that aligns it+template <int K>+struct RowTab {+ int32_t i0[16][16] = {}, i1[16][16] = {}, sr[16][16] = {}, sl[16][16] = {};+ uint16_t hi0[16] = {}, hi1[16] = {}; // columns whose word is in the second register pair (>= 32)+};+template <int K>+constexpr RowTab<K> make_rowtab() {+ RowTab<K> t{};+ constexpr int W = 8 * K;+ const auto inv = make_inv();+ for (int r = 0; r < 16; ++r)+ for (int c = 0; c < 16; ++c) {+ const int p = inv[(size_t) (r * 16 + c)];+ const int b1 = (p + 1 + 256) * K;+ const int w1 = ((b1 - 1) >> 5) % W, w0 = ((b1 - 16) >> 5) % W;+ const int sh = (((b1 - 1) >> 5) + 1) * 32 - b1;+ t.i0[r][c] = w0;+ t.i1[r][c] = w1;+ t.sr[r][c] = sh;+ t.sl[r][c] = 32 - sh; // a shift by 32 clears the lane: a window inside one word+ if (w0 >= 32) t.hi0[r] = (uint16_t) (t.hi0[r] | (1u << c));+ if (w1 >= 32) t.hi1[r] = (uint16_t) (t.hi1[r] | (1u << c));+ }+ return t;+}++// ---------------------------------------------------------------- scalar+template <int K>+void band_scalar(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* acc /* 128 */) {+ constexpr int W = 8 * K;+ static constexpr RowTab<K> T = make_rowtab<K>();+ const int ld = m.n / 16;+ for (int i = 0; i < 128; ++i) acc[i] = 0.0f;+ for (int ks = ks0; ks < ks1; ++ks)+ for (int t = 0; t < 8; ++t) {+ const uint32_t* w = m.trellis + ((size_t) ks * ld + tc0 + t) * W;+ for (int r = 0; r < 16; ++r) {+ const float xv = xh[ks * 16 + r];+ for (int c = 0; c < 16; ++c) {+ const uint64_t a = w[T.i0[r][c]], b = w[T.i1[r][c]];+ const uint32_t st = (uint32_t) (((a << 32) | b) >> T.sr[r][c]) & 0xffffu;+ const uint32_t x = st * kMul1;+ const int s = (int) ((x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24));+ acc[t * 16 + c] += (float) s * xv;+ }+ }+ }+}++// ---------------------------------------------------------------- AVX-512+#if defined(EXL3_X86)+template <int K>+EXL3_AVX512 inline __m512i gather_words(const RowTab<K>& T, const int32_t (&idx)[16][16], const uint16_t* hi, int r,+ __m512i p0, __m512i p1, __m512i p2, __m512i p3) {+ const __m512i ix = _mm512_loadu_si512((const void*) idx[r]);+ __m512i v = _mm512_permutex2var_epi32(p0, ix, p1);+ if constexpr (8 * K > 32) {+ if (hi[r] != 0) v = _mm512_mask_blend_epi32((__mmask16) hi[r], v, _mm512_permutex2var_epi32(p2, ix, p3));+ }+ (void) T;+ return v;+}++template <int K>+EXL3_AVX512 void band_avx512(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* out /* 128 */) {+ constexpr int W = 8 * K;+ static constexpr RowTab<K> T = make_rowtab<K>();+ constexpr auto ld_mask = [](int n) -> __mmask16 {+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);+ };+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const __m512i mult = _mm512_set1_epi32((int) kMul1);+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);+ const __m512i low16 = _mm512_set1_epi32(0xffff);+ __m512 acc[8];+ for (int t = 0; t < 8; ++t) acc[t] = _mm512_setzero_ps();+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ // the band's 8 tiles of this slice are contiguous; the next slices' are one row of tiles on+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ const float* xk = xh + ks * 16;+ for (int t = 0; t < 8; ++t) {+ const uint32_t* w = row + t * W;+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();+ __m512 a = acc[t];+#pragma GCC unroll 16+ for (int r = 0; r < 16; ++r) {+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);+ const __m512i st = _mm512_and_si512(+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),+ low16);+ const __m512i bs = _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16);+ a = _mm512_fmadd_ps(_mm512_cvtepi32_ps(bs), _mm512_set1_ps(xk[r]), a);+ }+ acc[t] = a;+ }+ }+ for (int t = 0; t < 8; ++t) _mm512_storeu_ps(out + t * 16, acc[t]);+}++// band_avx512 for N activations at once (xh[i] -> out[i]), as exllamav3's CPU kernel takes up to 4 token rows: each+// tile row's 16 states are pulled out and byte-summed once, then one FMA per activation - activation i's FMAs are+// band_avx512's for it alone, in its order, so out[i] is the same bit for bit. An extra activation costs one FMA of+// the row's ~12 instructions. (The tiles' accumulators live in memory, loaded once a tile and slice as there: only+// the N of the tile in hand take registers.)+template <int K, int N>+EXL3_AVX512 void band_avx512_n(const Exl3Part& m, const float* const* xh, int tc0, int ks0, int ks1,+ float* const* out /* N x 128 */) {+ constexpr int W = 8 * K;+ static constexpr RowTab<K> T = make_rowtab<K>();+ constexpr auto ld_mask = [](int n) -> __mmask16 {+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);+ };+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const __m512i mult = _mm512_set1_epi32((int) kMul1);+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);+ const __m512i low16 = _mm512_set1_epi32(0xffff);+ __m512 acc[8][N];+ for (int t = 0; t < 8; ++t)+ for (int i = 0; i < N; ++i) acc[t][i] = _mm512_setzero_ps();+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ const float* xk[N];+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;+ for (int t = 0; t < 8; ++t) {+ const uint32_t* w = row + t * W;+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();+ __m512 a[N];+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) a[i] = acc[t][i];+#pragma GCC unroll 16+ for (int r = 0; r < 16; ++r) {+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);+ const __m512i st = _mm512_and_si512(+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),+ low16);+ const __m512 f = _mm512_cvtepi32_ps(+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16));+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) a[i] = _mm512_fmadd_ps(f, _mm512_set1_ps(xk[i][r]), a[i]);+ }+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) acc[t][i] = a[i];+ }+ }+ for (int i = 0; i < N; ++i)+ for (int t = 0; t < 8; ++t) _mm512_storeu_ps(out[i] + t * 16, acc[t][i]);+}++// rows_avx512_n: slices [ks0, ks1) of the tile columns [t0, t1) - a split-K piece, or a run of its columns - for N+// activations (xh[i] -> acc[i], indexed by column; columns 16 t0 .. 16 t1 overwritten). The tile rows are read in+// memory order, where a band's tiles are 1 KB runs a whole tile row apart: 40 threads of a 2-socket Xeon 6152 read a K4+// expert from DRAM in 0.14 ms in pieces, 0.21 in bands (~90 vs ~60 GB/s), the kernel the same. A tile's 16 sums per+// activation live in acc between slices; each row's decode and FMAs are band_avx512's, so acc[i] is the N = 1 call's+// for xh[i] alone, bit for bit, whatever the columns' run. The prefetch runs 16 tiles ahead in the job's own order+// (into the next slice past the run's end, not the next job's columns)+template <int K, int N>+EXL3_AVX512 void rows_avx512_n(const Exl3Part& m, const float* const* xh, int ks0, int ks1, int t0, int t1,+ float* const* acc) {+ constexpr int W = 8 * K;+ static constexpr RowTab<K> T = make_rowtab<K>();+ constexpr auto ld_mask = [](int n) -> __mmask16 {+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);+ };+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const __m512i mult = _mm512_set1_epi32((int) kMul1);+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);+ const __m512i low16 = _mm512_set1_epi32(0xffff);+ for (int i = 0; i < N; ++i)+ for (int c = 16 * t0; c < 16 * t1; c += 16) _mm512_storeu_ps(acc[i] + c, _mm512_setzero_ps());+ const int span = t1 - t0;+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ float xk[N][16];+ for (int i = 0; i < N; ++i)+ for (int r = 0; r < 16; ++r) xk[i][r] = xh[i][ks * 16 + r];+ for (int t = t0; t < t1; ++t) {+ const uint32_t* w = row + (size_t) t * W;+ const uint32_t* pf = t + 16 < t1 || span == ld ? w + 16 * W : row + row_words + (size_t) (t + 16 - span) * W;+ for (int l = 0; l < W * 4; l += 64) _mm_prefetch((const char*) pf + l, _MM_HINT_T0);+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();+ __m512 a[N];+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) a[i] = _mm512_loadu_ps(acc[i] + t * 16);+#pragma GCC unroll 16+ for (int r = 0; r < 16; ++r) {+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);+ const __m512i st = _mm512_and_si512(+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),+ low16);+ const __m512 f = _mm512_cvtepi32_ps(+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16));+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) a[i] = _mm512_fmadd_ps(f, _mm512_set1_ps(xk[i][r]), a[i]);+ }+#pragma GCC unroll 8+ for (int i = 0; i < N; ++i) _mm512_storeu_ps(acc[i] + t * 16, a[i]);+ }+ }+}++// band_gemm: a band's byte-sum products for many activations - each chunk of kGemmK inputs decoded ONCE into F32+// (wc, in L2), then multiplied as a GEMM: tiles of TB activations x 64 columns, the sums in registers, two FMAs a+// load. The prompt path's CPU rows, where an expert serves tens to hundreds of tokens: one decode (~6 instructions a+// 16-weight row) for all of them instead of one per kExl3CpuMaxN.+constexpr int kGemmK = 256;++template <int K>+EXL3_AVX512 void decode_chunk(const Exl3Part& m, int tc0, int ks0, int ks1, float* wc /* (ks1 - ks0) 16 x 128 */) {+ constexpr int W = 8 * K;+ static constexpr RowTab<K> T = make_rowtab<K>();+ constexpr auto ld_mask = [](int n) -> __mmask16 {+ return n >= 16 ? (__mmask16) 0xffffu : n <= 0 ? (__mmask16) 0 : (__mmask16) ((1u << n) - 1u);+ };+ constexpr __mmask16 m0 = ld_mask(W), m1 = ld_mask(W - 16), m2 = ld_mask(W - 32), m3 = ld_mask(W - 48);+ const size_t row_words = (size_t) (m.n / 16) * W;+ const __m512i mult = _mm512_set1_epi32((int) kMul1);+ const __m512i ones8 = _mm512_set1_epi8(1), ones16 = _mm512_set1_epi16(1);+ const __m512i low16 = _mm512_set1_epi32(0xffff);+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ float* wr = wc + (size_t) (ks - ks0) * 16 * 128;+ for (int t = 0; t < 8; ++t) {+ const uint32_t* w = row + t * W;+ const __m512i p0 = _mm512_maskz_loadu_epi32(m0, w);+ const __m512i p1 = m1 ? _mm512_maskz_loadu_epi32(m1, w + 16) : _mm512_setzero_si512();+ const __m512i p2 = m2 ? _mm512_maskz_loadu_epi32(m2, w + 32) : _mm512_setzero_si512();+ const __m512i p3 = m3 ? _mm512_maskz_loadu_epi32(m3, w + 48) : _mm512_setzero_si512();+#pragma GCC unroll 16+ for (int r = 0; r < 16; ++r) {+ const __m512i wa = gather_words<K>(T, T.i0, T.hi0, r, p0, p1, p2, p3);+ const __m512i wb = gather_words<K>(T, T.i1, T.hi1, r, p0, p1, p2, p3);+ const __m512i st = _mm512_and_si512(+ _mm512_or_si512(_mm512_srlv_epi32(wb, _mm512_loadu_si512((const void*) T.sr[r])),+ _mm512_sllv_epi32(wa, _mm512_loadu_si512((const void*) T.sl[r]))),+ low16);+ const __m512i bs =+ _mm512_madd_epi16(_mm512_maddubs_epi16(_mm512_mullo_epi32(st, mult), ones8), ones16);+ _mm512_store_ps(wr + r * 128 + t * 16, _mm512_cvtepi32_ps(bs));+ }+ }+ }+}++// acc[t][0..128) += sum_{k < kr} x[t][k] wc[k][..] for TB activations+template <int TB>+EXL3_AVX512 void gemm_tile(const float* wc, int kr, const float* const* x, float* const* acc) {+ for (int cb = 0; cb < 128; cb += 64) {+ __m512 a[TB][4];+ for (int t = 0; t < TB; ++t)+ for (int j = 0; j < 4; ++j) a[t][j] = _mm512_loadu_ps(acc[t] + cb + j * 16);+ const float* w = wc + cb;+ for (int k = 0; k < kr; ++k, w += 128) {+ const __m512 w0 = _mm512_load_ps(w), w1 = _mm512_load_ps(w + 16), w2 = _mm512_load_ps(w + 32),+ w3 = _mm512_load_ps(w + 48);+#pragma GCC unroll 8+ for (int t = 0; t < TB; ++t) {+ const __m512 xv = _mm512_set1_ps(x[t][k]);+ a[t][0] = _mm512_fmadd_ps(xv, w0, a[t][0]);+ a[t][1] = _mm512_fmadd_ps(xv, w1, a[t][1]);+ a[t][2] = _mm512_fmadd_ps(xv, w2, a[t][2]);+ a[t][3] = _mm512_fmadd_ps(xv, w3, a[t][3]);+ }+ }+ for (int t = 0; t < TB; ++t)+ for (int j = 0; j < 4; ++j) _mm512_storeu_ps(acc[t] + cb + j * 16, a[t][j]);+ }+}++// nb bands from tile column tc0 (acc[t]: the activation's 128 nb outputs from there): each chunk of the inputs is+// taken for every band in turn, so the activations' chunk (n x kGemmK floats) comes from L2 for the nb bands - an+// expert's rows are its own tokens, and with a job a band they came from DRAM again for every band (~16x the+// expert's weights in bytes)+template <int K>+EXL3_AVX512 void band_gemm_avx512(const Exl3Part& m, const float* const* xh, int n, int tc0, int nb,+ float* const* acc, float* wc) {+ constexpr int KS = kGemmK / 16;+ constexpr int TB = 6; // 24 sums + 4 weights + the activation: 29 of the 32 registers+ for (int t = 0; t < n; ++t) std::memset(acc[t], 0, (size_t) nb * 128 * sizeof(float));+ std::vector<float*> ab((size_t) n);+ for (int ks0 = 0; ks0 < m.k / 16; ks0 += KS) {+ const int ks1 = std::min(m.k / 16, ks0 + KS), kr = (ks1 - ks0) * 16;+ for (int bb = 0; bb < nb; ++bb) {+ decode_chunk<K>(m, tc0 + 8 * bb, ks0, ks1, wc);+ for (int t = 0; t < n; ++t) ab[(size_t) t] = acc[t] + bb * 128;+ float* const* ap = ab.data();+ int t = 0;+ for (; t + TB <= n; t += TB) {+ const float* xp[TB];+ for (int i = 0; i < TB; ++i) xp[i] = xh[t + i] + ks0 * 16;+ gemm_tile<TB>(wc, kr, xp, ap + t);+ }+ if (t < n) { // the last 1..TB-1 activations in one pass too+ const float* xp[TB];+ for (int i = 0; i < n - t; ++i) xp[i] = xh[t + i] + ks0 * 16;+ switch (n - t) {+ case 1: gemm_tile<1>(wc, kr, xp, ap + t); break;+ case 2: gemm_tile<2>(wc, kr, xp, ap + t); break;+ case 3: gemm_tile<3>(wc, kr, xp, ap + t); break;+ case 4: gemm_tile<4>(wc, kr, xp, ap + t); break;+ default: gemm_tile<5>(wc, kr, xp, ap + t); break;+ }+ }+ }+ }+}++// had128 in 8 registers: every butterfly takes the operands the scalar loop takes (a lane's partner by a permute; the+// lower lane a + b, the upper b's lane t - r = a - b), so the result is had128's bit for bit+EXL3_AVX512 inline __m512 bfly(__m512 r, __m512 t, __mmask16 upper) {+ return _mm512_mask_sub_ps(_mm512_add_ps(r, t), upper, t, r);+}+EXL3_AVX512 void had128_avx512(float* v) {+ __m512 r[8];+ for (int i = 0; i < 8; ++i) r[i] = _mm512_loadu_ps(v + 16 * i);+ for (int i = 0; i < 8; ++i) {+ r[i] = bfly(r[i], _mm512_permute_ps(r[i], 0b10110001), (__mmask16) 0xAAAA); // h = 1+ r[i] = bfly(r[i], _mm512_permute_ps(r[i], 0b01001110), (__mmask16) 0xCCCC); // h = 2+ r[i] = bfly(r[i], _mm512_shuffle_f32x4(r[i], r[i], 0b10110001), (__mmask16) 0xF0F0); // h = 4+ r[i] = bfly(r[i], _mm512_shuffle_f32x4(r[i], r[i], 0b01001110), (__mmask16) 0xFF00); // h = 8+ }+ for (int h = 1; h < 8; h <<= 1) // h = 16, 32, 64: whole registers+ for (int i = 0; i < 8; i += 2 * h)+ for (int j = i; j < i + h; ++j) {+ const __m512 a = r[j], b = r[j + h];+ r[j] = _mm512_add_ps(a, b);+ r[j + h] = _mm512_sub_ps(a, b);+ }+ const __m512 s = _mm512_set1_ps(0.088388347648f);+ for (int i = 0; i < 8; ++i) _mm512_storeu_ps(v + 16 * i, _mm512_mul_ps(r[i], s));+}++// exl3_cpu_prepare_block / exl3_cpu_band_out with the FP16 scales converted 16 at a time (vcvtph2ps: exact, as+// f32_from_f16) and the block's sum in eight double lanes+EXL3_AVX512 float prepare_block_avx512(const uint16_t* suh, const float* x, float* xh) {+ for (int i = 0; i < 128; i += 16) {+ const __m512 sc = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) (suh + i)));+ _mm512_storeu_ps(xh + i, _mm512_mul_ps(_mm512_loadu_ps(x + i), sc));+ }+ had128_avx512(xh);+ __m512d s0 = _mm512_setzero_pd(), s1 = _mm512_setzero_pd();+ for (int i = 0; i < 128; i += 16) {+ s0 = _mm512_add_pd(s0, _mm512_cvtps_pd(_mm256_loadu_ps(xh + i)));+ s1 = _mm512_add_pd(s1, _mm512_cvtps_pd(_mm256_loadu_ps(xh + i + 8)));+ }+ return (float) _mm512_reduce_add_pd(_mm512_add_pd(s0, s1));+}+EXL3_AVX512 void band_out_avx512(const float* acc, float ki, float kz, const uint16_t* svh, float* y) {+ const __m512 vk = _mm512_set1_ps(ki), vz = _mm512_set1_ps(kz);+ for (int i = 0; i < 128; i += 16) _mm512_storeu_ps(y + i, _mm512_fmadd_ps(_mm512_loadu_ps(acc + i), vk, vz));+ had128_avx512(y);+ for (int i = 0; i < 128; i += 16) {+ const __m512 sc = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i*) (svh + i)));+ _mm512_storeu_ps(y + i, _mm512_mul_ps(_mm512_loadu_ps(y + i), sc));+ }+}++// ---------------------------------------------------------------- AVX2+// The AVX-512 tier on 8-lane registers, a tile row's 16 columns as two halves of 8 (after exllamav3's AVX2 tier).+// Column c' of a half reads the tile's words K c' + base (mod 8K) - its 8 words lie in one residue class of the word+// index mod K, and its funnel shift is the same for all 8 (the layout's row-shift invariant). So each tile's K+// registers are regrouped once into its K classes, and every half's gather is then ONE vpermd of a class register+// (none where the lanes are in order already) instead of a permute and a blend per register the half reads: odd K+// takes each class lane from exactly one register (blends only), K 2 / 4 / 8 are shuffles / a 4x4 / an 8x8 transpose,+// K 6 a permute and a blend per register. exllamav3 gets the same single permute from a planar order repacked at load+// (`_copy_repacked`); here the regrouping costs a few register ops a tile and the RAM tier and the GPUs keep the+// tiles as they are. The shifts are immediates, and the byte sum and the FMA are the AVX-512 tier's, lane for lane+// in the same (slice, row) order: the two tiers give the same bits.++// the lane of tile word w in its class register (class w % K; avx2_classes builds them)+constexpr int avx2_slot(int K, int w) {+ return (K & 1) ? (w & 7)+ : K == 2 ? (w & 4) | ((w >> 2) & 2) | ((w >> 1) & 1)+ : K == 4 ? (w >> 3) + 4 * ((w >> 2) & 1)+ : K == 8 ? (w >> 3)+ : w / K;+}++// per row and half: the right shift, whether the 16-bit windows sit in one word (no second gather), and for the+// window's first and last word the class register and the vpermd lanes that put its 8 words in column order+template <int K>+struct Avx2Tab {+ int sh[16][2] = {};+ bool one[16][2] = {};+ int cls[16][2][2] = {};+ bool ident[16][2][2] = {};+ int32_t idx[16][2][2][8] = {};+ bool ok = true; // the invariants above hold for every row and half+};+template <int K>+constexpr Avx2Tab<K> make_avx2tab() {+ Avx2Tab<K> t{};+ const RowTab<K> rt = make_rowtab<K>();+ for (int r = 0; r < 16; ++r)+ for (int h = 0; h < 2; ++h) {+ const int c0 = 8 * h;+ t.sh[r][h] = rt.sr[r][c0];+ t.one[r][h] = true;+ for (int c = c0; c < c0 + 8; ++c) {+ if (rt.sr[r][c] != t.sh[r][h]) t.ok = false;+ if (rt.i0[r][c] != rt.i1[r][c]) t.one[r][h] = false;+ }+ if (t.one[r][h] != (t.sh[r][h] <= 16)) t.ok = false;+ for (int wd = 0; wd < 2; ++wd) {+ const auto& ix = wd == 0 ? rt.i0 : rt.i1;+ t.cls[r][h][wd] = ix[r][c0] % K;+ t.ident[r][h][wd] = true;+ for (int c = 0; c < 8; ++c) {+ if (ix[r][c0 + c] % K != t.cls[r][h][wd]) t.ok = false;+ t.idx[r][h][wd][c] = avx2_slot(K, ix[r][c0 + c]);+ if (t.idx[r][h][wd][c] != c) t.ident[r][h][wd] = false;+ }+ }+ }+ return t;+}+template <int K>+inline constexpr Avx2Tab<K> kAvx2Tab = make_avx2tab<K>();++// odd K: lane l of class rho is register j's, the one j with (8 j + l) % K == rho+template <int K, int RHO, int J>+constexpr int avx2_odd_mask() {+ int m = 0;+ for (int l = 0; l < 8; ++l)+ if ((8 * J + l) % K == RHO) m |= 1 << l;+ return m;+}+template <int K, int RHO, int J = 1>+EXL3_AVX2_INLINE __m256i avx2_odd_class(const __m256i (&R)[K], __m256i acc) {+ if constexpr (J == K) {+ return acc;+ } else {+ constexpr int m = avx2_odd_mask<K, RHO, J>(); // (clang's _mm256_blend_epi32 is a macro: no template commas)+ return avx2_odd_class<K, RHO, J + 1>(R, _mm256_blend_epi32(acc, R[J], m));+ }+}+template <int K, int... RHO>+EXL3_AVX2_INLINE void avx2_odd_classes(const __m256i (&R)[K], __m256i (&C)[K], std::integer_sequence<int, RHO...>) {+ ((C[RHO] = avx2_odd_class<K, RHO>(R, R[0])), ...);+}+// any K (used for K 6): class rho's word rho + K s at lane s - each register's words permuted there, then blended+template <int K, int RHO, int J>+constexpr int avx2_gen_mask() {+ int m = 0;+ for (int s = 0; s < 8; ++s)+ if ((RHO + K * s) / 8 == J) m |= 1 << s;+ return m;+}+template <int K, int RHO, int J, int S>+constexpr int avx2_gen_lane() {+ return (RHO + K * S) / 8 == J ? (RHO + K * S) % 8 : 0;+}+template <int K, int RHO, int J = 0>+EXL3_AVX2_INLINE __m256i avx2_gen_class(const __m256i (&R)[K], __m256i acc) {+ if constexpr (J == K) {+ return acc;+ } else if constexpr (avx2_gen_mask<K, RHO, J>() == 0) {+ return avx2_gen_class<K, RHO, J + 1>(R, acc);+ } else {+ const __m256i p = _mm256_permutevar8x32_epi32(+ R[J], _mm256_setr_epi32(avx2_gen_lane<K, RHO, J, 0>(), avx2_gen_lane<K, RHO, J, 1>(),+ avx2_gen_lane<K, RHO, J, 2>(), avx2_gen_lane<K, RHO, J, 3>(),+ avx2_gen_lane<K, RHO, J, 4>(), avx2_gen_lane<K, RHO, J, 5>(),+ avx2_gen_lane<K, RHO, J, 6>(), avx2_gen_lane<K, RHO, J, 7>()));+ constexpr int m = avx2_gen_mask<K, RHO, J>(); // (a macro under clang, as above)+ return avx2_gen_class<K, RHO, J + 1>(R, _mm256_blend_epi32(acc, p, m));+ }+}+template <int K, int... RHO>+EXL3_AVX2_INLINE void avx2_gen_classes(const __m256i (&R)[K], __m256i (&C)[K], std::integer_sequence<int, RHO...>) {+ ((C[RHO] = avx2_gen_class<K, RHO>(R, _mm256_setzero_si256())), ...);+}+// a 4x4 transpose in each 128-bit half: o[q] = lanes q and q + 4 of r0..r3+EXL3_AVX2_INLINE void avx2_t4(__m256i r0, __m256i r1, __m256i r2, __m256i r3, __m256i (&o)[4]) {+ const __m256i t0 = _mm256_unpacklo_epi32(r0, r1), t1 = _mm256_unpackhi_epi32(r0, r1);+ const __m256i t2 = _mm256_unpacklo_epi32(r2, r3), t3 = _mm256_unpackhi_epi32(r2, r3);+ o[0] = _mm256_unpacklo_epi64(t0, t2);+ o[1] = _mm256_unpackhi_epi64(t0, t2);+ o[2] = _mm256_unpacklo_epi64(t1, t3);+ o[3] = _mm256_unpackhi_epi64(t1, t3);+}+// a tile's 8K words as its K class registers (word w in C[w % K] at lane avx2_slot(K, w))+template <int K>+EXL3_AVX2_INLINE void avx2_classes(const uint32_t* w, __m256i (&C)[K]) {+ __m256i R[K];+ for (int j = 0; j < K; ++j) R[j] = _mm256_loadu_si256((const __m256i*) (w + 8 * j));+ if constexpr (K == 1) {+ C[0] = R[0];+ } else if constexpr (K % 2 == 1) {+ avx2_odd_classes<K>(R, C, std::make_integer_sequence<int, K>{});+ } else if constexpr (K == 2) {+ const __m256 a = _mm256_castsi256_ps(R[0]), b = _mm256_castsi256_ps(R[1]);+ C[0] = _mm256_castps_si256(_mm256_shuffle_ps(a, b, 0x88)); // words 0 2 8 10 | 4 6 12 14+ C[1] = _mm256_castps_si256(_mm256_shuffle_ps(a, b, 0xDD)); // words 1 3 9 11 | 5 7 13 15+ } else if constexpr (K == 4) {+ avx2_t4(R[0], R[1], R[2], R[3], C);+ } else if constexpr (K == 8) {+ __m256i A[4], B[4];+ avx2_t4(R[0], R[1], R[2], R[3], A);+ avx2_t4(R[4], R[5], R[6], R[7], B);+ for (int q = 0; q < 4; ++q) {+ C[q] = _mm256_permute2x128_si256(A[q], B[q], 0x20);+ C[q + 4] = _mm256_permute2x128_si256(A[q], B[q], 0x31);+ }+ } else {+ avx2_gen_classes<K>(R, C, std::make_integer_sequence<int, K>{});+ }+}++// the 8 words (first or last of each window) of row R's half H, in column order+template <int K, int R, int H, int WD>+EXL3_AVX2_INLINE __m256i avx2_gather(const __m256i (&C)[K]) {+ constexpr const Avx2Tab<K>& T = kAvx2Tab<K>;+ if constexpr (T.ident[R][H][WD]) {+ return C[T.cls[R][H][WD]];+ } else {+ return _mm256_permutevar8x32_epi32(C[T.cls[R][H][WD]],+ _mm256_loadu_si256((const __m256i*) T.idx[R][H][WD]));+ }+}++struct Avx2Consts {+ __m256i mult, ones8, ones16, low16;+};+EXL3_AVX2_INLINE Avx2Consts avx2_consts() {+ return {_mm256_set1_epi32((int) kMul1), _mm256_set1_epi8(1), _mm256_set1_epi16(1), _mm256_set1_epi32(0xffff)};+}++// row R's half H: its 8 states, and their codebook byte sums as F32+template <int K, int R, int H>+EXL3_AVX2_INLINE __m256 avx2_sums(const __m256i (&C)[K], const Avx2Consts& k) {+ constexpr const Avx2Tab<K>& T = kAvx2Tab<K>;+ constexpr int sh = T.sh[R][H];+ const __m256i b = avx2_gather<K, R, H, 1>(C);+ __m256i st;+ if constexpr (T.one[R][H] && sh == 16) {+ st = _mm256_srli_epi32(b, 16);+ } else if constexpr (T.one[R][H]) {+ st = _mm256_and_si256(_mm256_srli_epi32(b, sh), k.low16);+ } else {+ const __m256i a = avx2_gather<K, R, H, 0>(C);+ st = _mm256_and_si256(_mm256_or_si256(_mm256_srli_epi32(b, sh), _mm256_slli_epi32(a, 32 - sh)), k.low16);+ }+ return _mm256_cvtepi32_ps(+ _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_mullo_epi32(st, k.mult), k.ones8), k.ones16));+}++template <int K, int R>+EXL3_AVX2_INLINE void avx2_rows(const __m256i (&C)[K], const float* xk, __m256& lo, __m256& hi, const Avx2Consts& k) {+ if constexpr (R < 16) {+ const __m256 xv = _mm256_broadcast_ss(xk + R);+ lo = _mm256_fmadd_ps(avx2_sums<K, R, 0>(C, k), xv, lo);+ hi = _mm256_fmadd_ps(avx2_sums<K, R, 1>(C, k), xv, hi);+ avx2_rows<K, R + 1>(C, xk, lo, hi, k);+ }+}++template <int K>+EXL3_AVX2 void band_avx2(const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* out /* 128 */) {+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");+ constexpr int W = 8 * K;+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const Avx2Consts k = avx2_consts();+ __m256 acc[8][2]; // the band's 8 tiles (16 registers: in memory, a tile's two taken at a time)+ for (int t = 0; t < 8; ++t) acc[t][0] = acc[t][1] = _mm256_setzero_ps();+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ // the band's 8 tiles of this slice are contiguous; the next slices' are one row of tiles on+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ const float* xk = xh + ks * 16;+ for (int t = 0; t < 8; ++t) {+ __m256i C[K];+ avx2_classes<K>(row + t * W, C);+ __m256 lo = acc[t][0], hi = acc[t][1];+ avx2_rows<K, 0>(C, xk, lo, hi, k);+ acc[t][0] = lo;+ acc[t][1] = hi;+ }+ }+ for (int t = 0; t < 8; ++t) {+ _mm256_storeu_ps(out + t * 16, acc[t][0]);+ _mm256_storeu_ps(out + t * 16 + 8, acc[t][1]);+ }+}++// band_avx2 for N activations at once, as band_avx512_n: a half row's states summed once, then one FMA per+// activation - activation i's FMAs are band_avx2's for it alone, in its order, so out[i] is the same bit for bit.+// Up to 3 activations both halves of a tile at once (2N accumulators, one broadcast a row and activation); more, a+// tile's halves one after the other (N accumulators, not 2N: 16 registers) - 2 activations 1.55 -> 1.34 ms an+// expert-token on one Zen 3 core, 3: 1.18 -> 1.11+template <int N, int... I>+EXL3_AVX2_INLINE void avx2_fma_n(__m256 f, const float* const (&xk)[N], int r, __m256 (&a)[N],+ std::integer_sequence<int, I...>) {+ ((a[I] = _mm256_fmadd_ps(f, _mm256_broadcast_ss(xk[I] + r), a[I])), ...);+}+template <int K, int N, int H, int R>+EXL3_AVX2_INLINE void avx2_rows_n(const __m256i (&C)[K], const float* const (&xk)[N], __m256 (&a)[N],+ const Avx2Consts& k) {+ if constexpr (R < 16) {+ avx2_fma_n<N>(avx2_sums<K, R, H>(C, k), xk, R, a, std::make_integer_sequence<int, N>{});+ avx2_rows_n<K, N, H, R + 1>(C, xk, a, k);+ }+}+template <int N, int... I>+EXL3_AVX2_INLINE void avx2_fma_n2(__m256 f0, __m256 f1, const float* const (&xk)[N], int r, __m256 (&lo)[N],+ __m256 (&hi)[N], std::integer_sequence<int, I...>) {+ ((lo[I] = _mm256_fmadd_ps(f0, _mm256_broadcast_ss(xk[I] + r), lo[I]),+ hi[I] = _mm256_fmadd_ps(f1, _mm256_broadcast_ss(xk[I] + r), hi[I])),+ ...);+}+template <int K, int N, int R>+EXL3_AVX2_INLINE void avx2_rows_n2(const __m256i (&C)[K], const float* const (&xk)[N], __m256 (&lo)[N],+ __m256 (&hi)[N], const Avx2Consts& k) {+ if constexpr (R < 16) {+ avx2_fma_n2<N>(avx2_sums<K, R, 0>(C, k), avx2_sums<K, R, 1>(C, k), xk, R, lo, hi,+ std::make_integer_sequence<int, N>{});+ avx2_rows_n2<K, N, R + 1>(C, xk, lo, hi, k);+ }+}+template <int K, int N>+EXL3_AVX2 void band_avx2_n(const Exl3Part& m, const float* const* xh, int tc0, int ks0, int ks1,+ float* const* out /* N x 128 */) {+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");+ constexpr int W = 8 * K;+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const Avx2Consts k = avx2_consts();+ __m256 acc[8][2][N];+ for (int t = 0; t < 8; ++t)+ for (int h = 0; h < 2; ++h)+ for (int i = 0; i < N; ++i) acc[t][h][i] = _mm256_setzero_ps();+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ const float* xk[N];+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;+ for (int t = 0; t < 8; ++t) {+ __m256i C[K];+ avx2_classes<K>(row + t * W, C);+ if constexpr (N <= 3) {+ __m256 lo[N], hi[N];+ for (int i = 0; i < N; ++i) lo[i] = acc[t][0][i], hi[i] = acc[t][1][i];+ avx2_rows_n2<K, N, 0>(C, xk, lo, hi, k);+ for (int i = 0; i < N; ++i) acc[t][0][i] = lo[i], acc[t][1][i] = hi[i];+ } else {+ __m256 a[N];+ for (int i = 0; i < N; ++i) a[i] = acc[t][0][i];+ avx2_rows_n<K, N, 0, 0>(C, xk, a, k);+ for (int i = 0; i < N; ++i) acc[t][0][i] = a[i], a[i] = acc[t][1][i];+ avx2_rows_n<K, N, 1, 0>(C, xk, a, k);+ for (int i = 0; i < N; ++i) acc[t][1][i] = a[i];+ }+ }+ }+ for (int i = 0; i < N; ++i)+ for (int t = 0; t < 8; ++t) {+ _mm256_storeu_ps(out[i] + t * 16, acc[t][0][i]);+ _mm256_storeu_ps(out[i] + t * 16 + 8, acc[t][1][i]);+ }+}++// #188's split-K piece (rows_avx512_n) on the AVX2 tier: slices [ks0, ks1) of EVERY column for N activations, the+// tile rows in memory order, a tile's sums in acc between slices; each row's decode and FMAs are band_avx2's, so acc[i]+// is the band kernel's for its columns and xh[i] alone, bit for bit (and rows_avx512_n's)+template <int K, int N>+EXL3_AVX2 void rows_avx2_n(const Exl3Part& m, const float* const* xh, int ks0, int ks1, int t0, int t1,+ float* const* acc) {+ static_assert(kAvx2Tab<K>.ok, "EXL3 tile layout: a half row's words must share a residue class and a shift");+ constexpr int W = 8 * K;+ const int ld = m.n / 16;+ const size_t row_words = (size_t) ld * W;+ const Avx2Consts k = avx2_consts();+ for (int i = 0; i < N; ++i)+ for (int c = 16 * t0; c < 16 * t1; c += 8) _mm256_storeu_ps(acc[i] + c, _mm256_setzero_ps());+ const int span = t1 - t0;+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ const float* xk[N];+ for (int i = 0; i < N; ++i) xk[i] = xh[i] + ks * 16;+ for (int t = t0; t < t1; ++t) {+ const uint32_t* w = row + (size_t) t * W;+ const uint32_t* pf = t + 16 < t1 || span == ld ? w + 16 * W : row + row_words + (size_t) (t + 16 - span) * W;+ for (int l = 0; l < W * 4; l += 64) _mm_prefetch((const char*) pf + l, _MM_HINT_T0);+ __m256i C[K];+ avx2_classes<K>(w, C);+ if constexpr (N == 1) {+ __m256 lo = _mm256_loadu_ps(acc[0] + t * 16), hi = _mm256_loadu_ps(acc[0] + t * 16 + 8);+ avx2_rows<K, 0>(C, xk[0], lo, hi, k);+ _mm256_storeu_ps(acc[0] + t * 16, lo);+ _mm256_storeu_ps(acc[0] + t * 16 + 8, hi);+ } else if constexpr (N <= 3) {+ __m256 lo[N], hi[N];+ for (int i = 0; i < N; ++i)+ lo[i] = _mm256_loadu_ps(acc[i] + t * 16), hi[i] = _mm256_loadu_ps(acc[i] + t * 16 + 8);+ avx2_rows_n2<K, N, 0>(C, xk, lo, hi, k);+ for (int i = 0; i < N; ++i)+ _mm256_storeu_ps(acc[i] + t * 16, lo[i]), _mm256_storeu_ps(acc[i] + t * 16 + 8, hi[i]);+ } else {+ __m256 a[N];+ for (int i = 0; i < N; ++i) a[i] = _mm256_loadu_ps(acc[i] + t * 16);+ avx2_rows_n<K, N, 0, 0>(C, xk, a, k);+ for (int i = 0; i < N; ++i) _mm256_storeu_ps(acc[i] + t * 16, a[i]), a[i] = _mm256_loadu_ps(acc[i] + t * 16 + 8);+ avx2_rows_n<K, N, 1, 0>(C, xk, a, k);+ for (int i = 0; i < N; ++i) _mm256_storeu_ps(acc[i] + t * 16 + 8, a[i]);+ }+ }+ }+}++// #188's band GEMM (decode_chunk + gemm_tile) on the AVX2 tier: a chunk of a band's byte sums decoded once into F32,+// then TB activations x 32 columns a pass, the sums in registers (12 of the 16 with TB 3) - each column's FMAs in the+// band kernel's order (k ascending; x w + a as band_avx2's s x + a), so the same bits as gemm_tile on AVX-512+template <int K, int R>+EXL3_AVX2_INLINE void avx2_decode_rows(const __m256i (&C)[K], float* wr, int t, const Avx2Consts& k) {+ if constexpr (R < 16) {+ _mm256_store_ps(wr + R * 128 + t * 16, avx2_sums<K, R, 0>(C, k));+ _mm256_store_ps(wr + R * 128 + t * 16 + 8, avx2_sums<K, R, 1>(C, k));+ avx2_decode_rows<K, R + 1>(C, wr, t, k);+ }+}+template <int K>+EXL3_AVX2 void decode_chunk_avx2(const Exl3Part& m, int tc0, int ks0, int ks1, float* wc /* (ks1 - ks0) 16 x 128 */) {+ constexpr int W = 8 * K;+ const size_t row_words = (size_t) (m.n / 16) * W;+ const Avx2Consts k = avx2_consts();+ const uint32_t* row = m.trellis + (size_t) ks0 * row_words + (size_t) tc0 * W;+ for (int ks = ks0; ks < ks1; ++ks, row += row_words) {+ const char* pf = (const char*) (row + 4 * row_words);+ for (int l = 0; l < 8 * W * 4; l += 64) _mm_prefetch(pf + l, _MM_HINT_T0);+ float* wr = wc + (size_t) (ks - ks0) * 16 * 128;+ for (int t = 0; t < 8; ++t) {+ __m256i C[K];+ avx2_classes<K>(row + t * W, C);+ avx2_decode_rows<K, 0>(C, wr, t, k);+ }+ }+}+// acc[t][0..128) += sum_{k < kr} x[t][k] wc[k][..] for TB activations+template <int TB>+EXL3_AVX2 void gemm_tile_avx2(const float* wc, int kr, const float* const* x, float* const* acc) {+ for (int cb = 0; cb < 128; cb += 32) {+ __m256 a[TB][4];+ for (int t = 0; t < TB; ++t)+ for (int j = 0; j < 4; ++j) a[t][j] = _mm256_loadu_ps(acc[t] + cb + j * 8);+ const float* w = wc + cb;+ for (int q = 0; q < kr; ++q, w += 128) {+ const __m256 w0 = _mm256_load_ps(w), w1 = _mm256_load_ps(w + 8), w2 = _mm256_load_ps(w + 16),+ w3 = _mm256_load_ps(w + 24);+ for (int t = 0; t < TB; ++t) {+ const __m256 xv = _mm256_broadcast_ss(x[t] + q);+ a[t][0] = _mm256_fmadd_ps(xv, w0, a[t][0]);+ a[t][1] = _mm256_fmadd_ps(xv, w1, a[t][1]);+ a[t][2] = _mm256_fmadd_ps(xv, w2, a[t][2]);+ a[t][3] = _mm256_fmadd_ps(xv, w3, a[t][3]);+ }+ }+ for (int t = 0; t < TB; ++t)+ for (int j = 0; j < 4; ++j) _mm256_storeu_ps(acc[t] + cb + j * 8, a[t][j]);+ }+}+template <int K>+EXL3_AVX2 void band_gemm_avx2(const Exl3Part& m, const float* const* xh, int n, int tc0, int nb, float* const* acc,+ float* wc) {+ constexpr int KS = kGemmK / 16;+ constexpr int TB = 3; // 12 sums + 4 weights of the 16 registers+ for (int t = 0; t < n; ++t) std::memset(acc[t], 0, (size_t) nb * 128 * sizeof(float));+ std::vector<float*> ab((size_t) n);+ for (int ks0 = 0; ks0 < m.k / 16; ks0 += KS) {+ const int ks1 = std::min(m.k / 16, ks0 + KS), kr = (ks1 - ks0) * 16;+ for (int bb = 0; bb < nb; ++bb) {+ decode_chunk_avx2<K>(m, tc0 + 8 * bb, ks0, ks1, wc);+ for (int t = 0; t < n; ++t) ab[(size_t) t] = acc[t] + bb * 128;+ float* const* ap = ab.data();+ int t = 0;+ for (; t + TB <= n; t += TB) {+ const float* xp[TB];+ for (int i = 0; i < TB; ++i) xp[i] = xh[t + i] + ks0 * 16;+ gemm_tile_avx2<TB>(wc, kr, xp, ap + t);+ }+ if (t < n) {+ const float* xp[TB];+ for (int i = 0; i < n - t; ++i) xp[i] = xh[t + i] + ks0 * 16;+ if (n - t == 1) gemm_tile_avx2<1>(wc, kr, xp, ap + t);+ else gemm_tile_avx2<2>(wc, kr, xp, ap + t);+ }+ }+ }+}++// had128 on 16 registers (exllamav3's hadamard_128_avx2): every butterfly takes the scalar loop's operands (a lane's+// partner by a permute; the lower lane a + b, the upper t - r = a - b), so the result is had128's bit for bit+EXL3_AVX2 void had128_avx2(float* v) {+ __m256 r[16];+ for (int i = 0; i < 16; ++i) r[i] = _mm256_loadu_ps(v + 8 * i);+ for (int i = 0; i < 16; ++i) {+ __m256 t = _mm256_permute_ps(r[i], 0b10110001); // h = 1+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b10101010);+ t = _mm256_permute_ps(r[i], 0b01001110); // h = 2+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b11001100);+ t = _mm256_permute2f128_ps(r[i], r[i], 0x01); // h = 4+ r[i] = _mm256_blend_ps(_mm256_add_ps(r[i], t), _mm256_sub_ps(t, r[i]), 0b11110000);+ }+ for (int h = 1; h < 16; h <<= 1) // h = 8 .. 64: whole registers+ for (int i = 0; i < 16; i += 2 * h)+ for (int j = i; j < i + h; ++j) {+ const __m256 a = r[j], b = r[j + h];+ r[j] = _mm256_add_ps(a, b);+ r[j + h] = _mm256_sub_ps(a, b);+ }+ const __m256 s = _mm256_set1_ps(0.088388347648f);+ for (int i = 0; i < 16; ++i) _mm256_storeu_ps(v + 8 * i, _mm256_mul_ps(r[i], s));+}++// prepare_block_avx512 / band_out_avx512 on the AVX2 tier, the same operations in the same order: the FP16 scales by+// vcvtph2ps (exact), the block's sum in the same eight double lanes (lanes 0-7 of the 16-float steps' first half, 8-15+// of the second, added) and reduced as GCC's _mm512_reduce_add_pd does (upper four + lower four, then the two pairs,+// then the last two), band_out's FMA - the same bits as the AVX-512 tier+EXL3_AVX2 float prepare_block_avx2(const uint16_t* suh, const float* x, float* xh) {+ for (int i = 0; i < 128; i += 8) {+ const __m256 sc = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (suh + i)));+ _mm256_storeu_ps(xh + i, _mm256_mul_ps(_mm256_loadu_ps(x + i), sc));+ }+ had128_avx2(xh);+ __m256d s0a = _mm256_setzero_pd(), s0b = _mm256_setzero_pd(), s1a = _mm256_setzero_pd(), s1b = _mm256_setzero_pd();+ for (int i = 0; i < 128; i += 16) {+ s0a = _mm256_add_pd(s0a, _mm256_cvtps_pd(_mm_loadu_ps(xh + i)));+ s0b = _mm256_add_pd(s0b, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 4)));+ s1a = _mm256_add_pd(s1a, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 8)));+ s1b = _mm256_add_pd(s1b, _mm256_cvtps_pd(_mm_loadu_ps(xh + i + 12)));+ }+ const __m256d lo = _mm256_add_pd(s0a, s1a), hi = _mm256_add_pd(s0b, s1b); // the 8 lanes of s0 + s1+ const __m256d t3 = _mm256_add_pd(hi, lo);+ const __m128d t6 = _mm_add_pd(_mm256_extractf128_pd(t3, 1), _mm256_castpd256_pd128(t3));+ return (float) (_mm_cvtsd_f64(t6) + _mm_cvtsd_f64(_mm_unpackhi_pd(t6, t6)));+}+EXL3_AVX2 void band_out_avx2(const float* acc, float ki, float kz, const uint16_t* svh, float* y) {+ const __m256 vk = _mm256_set1_ps(ki), vz = _mm256_set1_ps(kz);+ for (int i = 0; i < 128; i += 8) _mm256_storeu_ps(y + i, _mm256_fmadd_ps(_mm256_loadu_ps(acc + i), vk, vz));+ had128_avx2(y);+ for (int i = 0; i < 128; i += 8) {+ const __m256 sc = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*) (svh + i)));+ _mm256_storeu_ps(y + i, _mm256_mul_ps(_mm256_loadu_ps(y + i), sc));+ }+}++// AVX-512 F and BW with FMA, the OS saving the state (STRATA_FORCE_AVX2 is the tier choice's, below)+bool cpu_avx512_exl3() {+ unsigned r[4] = {0, 0, 0, 0};+ auto cpuid = [&](unsigned leaf, unsigned sub) {+#if defined(_MSC_VER)+ int x[4];+ __cpuidex(x, (int) leaf, (int) sub);+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];+#else+ __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);+#endif+ };+ cpuid(0, 0);+ if (r[0] < 7) return false;+ cpuid(1, 0);+ if (!((r[2] >> 27) & 1u) || !((r[2] >> 12) & 1u)) return false; // OSXSAVE, FMA+#if defined(_MSC_VER)+ const unsigned long long xcr0 = _xgetbv(0);+#else+ unsigned lo = 0, hi = 0;+ __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));+ const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo;+#endif+ if ((xcr0 & 0xE6) != 0xE6) return false; // the OS saves the AVX-512 state+ cpuid(7, 0);+ return ((r[1] >> 16) & 1u) && ((r[1] >> 30) & 1u); // AVX-512F, AVX-512BW+}+// F16C (the AVX2 tier's FP16 scale conversions; every AVX2 CPU has it, asked anyway)+bool cpu_f16c() {+ unsigned r[4] = {0, 0, 0, 0};+#if defined(_MSC_VER)+ int x[4];+ __cpuidex(x, 1, 0);+ for (int i = 0; i < 4; ++i) r[i] = (unsigned) x[i];+#else+ __cpuid_count(1, 0, r[0], r[1], r[2], r[3]);+#endif+ return ((r[2] >> 29) & 1u) != 0;+}+#endif++// the tiers this CPU runs (bit t: tier t - 0 scalar, 1 AVX2 + FMA + F16C, 2 AVX-512 F/BW), and the one the lane takes:+// the widest, but AVX2 on an AVX-512 CPU with STRATA_FORCE_AVX2=1 (to compare the two vector tiers)+int tier_mask() {+ static const int mask = [] {+ int t = 1;+#if defined(EXL3_X86)+ if (cpu_avx2_ok() && cpu_f16c()) t |= 2;+ if (cpu_avx512_exl3()) t |= 4;+#endif+ return t;+ }();+ return mask;+}+int detect_tier() {+ const int t = tier_mask();+ const char* f = std::getenv("STRATA_FORCE_AVX2");+ if ((t & 4) && !(f != nullptr && f[0] == '1')) return 2;+ return (t & 2) ? 1 : 0;+}++template <int K>+void band_on(int tier, const Exl3Part& m, const float* xh, int tc0, int ks0, int ks1, float* acc) {+#if defined(EXL3_X86)+ if (tier == 2) {+ band_avx512<K>(m, xh, tc0, ks0, ks1, acc);+ return;+ }+ if (tier == 1) {+ band_avx2<K>(m, xh, tc0, ks0, ks1, acc);+ return;+ }+#endif+ band_scalar<K>(m, xh, tc0, ks0, ks1, acc);+}++// n activations (1..kExl3CpuMaxN) on one band+template <int K>+void band_on_n(int tier, const Exl3Part& m, const float* const* xh, int n, int tc0, int ks0, int ks1,+ float* const* acc) {+#if defined(EXL3_X86)+ if (tier == 2 && n > 1) {+ switch (n) {+ case 2: band_avx512_n<K, 2>(m, xh, tc0, ks0, ks1, acc); return;+ case 3: band_avx512_n<K, 3>(m, xh, tc0, ks0, ks1, acc); return;+ case 4: band_avx512_n<K, 4>(m, xh, tc0, ks0, ks1, acc); return;+ case 5: band_avx512_n<K, 5>(m, xh, tc0, ks0, ks1, acc); return;+ case 6: band_avx512_n<K, 6>(m, xh, tc0, ks0, ks1, acc); return;+ case 7: band_avx512_n<K, 7>(m, xh, tc0, ks0, ks1, acc); return;+ default: band_avx512_n<K, 8>(m, xh, tc0, ks0, ks1, acc); return;+ }+ }+ if (tier == 1 && n > 1) {+ switch (n) {+ case 2: band_avx2_n<K, 2>(m, xh, tc0, ks0, ks1, acc); return;+ case 3: band_avx2_n<K, 3>(m, xh, tc0, ks0, ks1, acc); return;+ case 4: band_avx2_n<K, 4>(m, xh, tc0, ks0, ks1, acc); return;+ case 5: band_avx2_n<K, 5>(m, xh, tc0, ks0, ks1, acc); return;+ case 6: band_avx2_n<K, 6>(m, xh, tc0, ks0, ks1, acc); return;+ case 7: band_avx2_n<K, 7>(m, xh, tc0, ks0, ks1, acc); return;+ default: band_avx2_n<K, 8>(m, xh, tc0, ks0, ks1, acc); return;+ }+ }+#endif+ for (int i = 0; i < n; ++i) band_on<K>(tier, m, xh[i], tc0, ks0, ks1, acc[i]);+}++// a split-K piece's tile columns [t0, t1) for n activations (1..kExl3CpuMaxN); the scalar loop: each band's+// band_scalar (t0, t1 multiples of 8 there)+template <int K>+void rows_on_n(int tier, const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int t0, int t1,+ float* const* acc) {+#if defined(EXL3_X86)+ if (tier == 2) {+ switch (n) {+ case 1: rows_avx512_n<K, 1>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 2: rows_avx512_n<K, 2>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 3: rows_avx512_n<K, 3>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 4: rows_avx512_n<K, 4>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 5: rows_avx512_n<K, 5>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 6: rows_avx512_n<K, 6>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 7: rows_avx512_n<K, 7>(m, xh, ks0, ks1, t0, t1, acc); return;+ default: rows_avx512_n<K, 8>(m, xh, ks0, ks1, t0, t1, acc); return;+ }+ }+ if (tier == 1) {+ switch (n) {+ case 1: rows_avx2_n<K, 1>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 2: rows_avx2_n<K, 2>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 3: rows_avx2_n<K, 3>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 4: rows_avx2_n<K, 4>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 5: rows_avx2_n<K, 5>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 6: rows_avx2_n<K, 6>(m, xh, ks0, ks1, t0, t1, acc); return;+ case 7: rows_avx2_n<K, 7>(m, xh, ks0, ks1, t0, t1, acc); return;+ default: rows_avx2_n<K, 8>(m, xh, ks0, ks1, t0, t1, acc); return;+ }+ }+#endif+ for (int i = 0; i < n; ++i)+ for (int c0 = 16 * t0; c0 < 16 * t1; c0 += 128) band_scalar<K>(m, xh[i], c0 / 16, ks0, ks1, acc[i] + c0);+}++void band_acc_tier(int tier, const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc) {+ const int tc0 = c0 / 16;+ switch (m.K) {+ case 1: band_on<1>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 2: band_on<2>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 3: band_on<3>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 4: band_on<4>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 5: band_on<5>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 6: band_on<6>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 7: band_on<7>(tier, m, xh, tc0, ks0, ks1, acc); break;+ case 8: band_on<8>(tier, m, xh, tc0, ks0, ks1, acc); break;+ default:+ for (int i = 0; i < 128; ++i) acc[i] = 0.0f;+ }+}++void band_acc_n_tier(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,+ float* const* acc) {+ const int tc0 = c0 / 16;+ for (int i0 = 0; i0 < n; i0 += kExl3CpuMaxN) {+ const int c = std::min(kExl3CpuMaxN, n - i0);+ switch (m.K) {+ case 1: band_on_n<1>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 2: band_on_n<2>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 3: band_on_n<3>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 4: band_on_n<4>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 5: band_on_n<5>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 6: band_on_n<6>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 7: band_on_n<7>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ case 8: band_on_n<8>(tier, m, xh + i0, c, tc0, ks0, ks1, acc + i0); break;+ default:+ for (int i = 0; i < c; ++i)+ for (int j = 0; j < 128; ++j) acc[i0 + i][j] = 0.0f;+ }+ }+}++} // namespace++int exl3_cpu_tier() {+ static const int tier = detect_tier();+ return tier;+}++bool exl3_cpu_vector() { return exl3_cpu_tier() > 0; }++float exl3_cpu_prepare_block(const uint16_t* suh, const float* x, float* xh) {+#if defined(EXL3_X86)+ if (exl3_cpu_tier() == 2) return prepare_block_avx512(suh, x, xh);+ if (exl3_cpu_tier() == 1) return prepare_block_avx2(suh, x, xh);+#endif+ for (int i = 0; i < 128; ++i) xh[i] = x[i] * f32_from_f16(suh[i]);+ had128(xh);+ double sum = 0.0;+ for (int i = 0; i < 128; ++i) sum += xh[i];+ return (float) sum;+}++float exl3_cpu_prepare(const Exl3Part& m, const float* x, float* xh) {+ double sum = 0.0;+ for (int b = 0; b < m.k; b += 128) sum += exl3_cpu_prepare_block(m.suh + b, x + b, xh + b);+ return (float) sum;+}++void exl3_cpu_band_acc(const Exl3Part& m, const float* xh, int c0, int ks0, int ks1, float* acc) {+ band_acc_tier(exl3_cpu_tier(), m, xh, c0, ks0, ks1, acc);+}++void exl3_cpu_band_acc_n(const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,+ float* const* acc) {+ band_acc_n_tier(exl3_cpu_tier(), m, xh, n, c0, ks0, ks1, acc);+}++bool exl3_cpu_band_acc_on(int tier, const Exl3Part& m, const float* const* xh, int n, int c0, int ks0, int ks1,+ float* const* acc) {+ if (tier < 0 || tier > 2 || ((tier_mask() >> tier) & 1) == 0 || n < 1) return false;+ if (n == 1) band_acc_tier(tier, m, xh[0], c0, ks0, ks1, acc[0]);+ else band_acc_n_tier(tier, m, xh, n, c0, ks0, ks1, acc);+ return true;+}++void exl3_cpu_rows_acc_cols_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, int c0, int c1,+ float* const* acc) {+ const int tier = exl3_cpu_tier();+ const int t0 = c0 / 16, t1 = c1 / 16;+ for (int i0 = 0; i0 < n; i0 += kExl3CpuMaxN) {+ const int c = std::min(kExl3CpuMaxN, n - i0);+ switch (m.K) {+ case 1: rows_on_n<1>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 2: rows_on_n<2>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 3: rows_on_n<3>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 4: rows_on_n<4>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 5: rows_on_n<5>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 6: rows_on_n<6>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 7: rows_on_n<7>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ case 8: rows_on_n<8>(tier, m, xh + i0, c, ks0, ks1, t0, t1, acc + i0); break;+ default:+ for (int i = 0; i < c; ++i)+ for (int j = c0; j < c1; ++j) acc[i0 + i][j] = 0.0f;+ }+ }+}++void exl3_cpu_rows_acc_n(const Exl3Part& m, const float* const* xh, int n, int ks0, int ks1, float* const* acc) {+ exl3_cpu_rows_acc_cols_n(m, xh, n, ks0, ks1, 0, m.n, acc);+}++size_t exl3_cpu_gemm_scratch() { return (size_t) kGemmK * 128; }++void exl3_cpu_band_gemm(const Exl3Part& m, const float* const* xh, int n, int c0, int nb, float* const* acc,+ float* wc) {+#if defined(EXL3_X86)+ if (exl3_cpu_tier() == 2) {+ switch (m.K) {+ case 1: band_gemm_avx512<1>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 2: band_gemm_avx512<2>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 3: band_gemm_avx512<3>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 4: band_gemm_avx512<4>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 5: band_gemm_avx512<5>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 6: band_gemm_avx512<6>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 7: band_gemm_avx512<7>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 8: band_gemm_avx512<8>(m, xh, n, c0 / 16, nb, acc, wc); return;+ default: break;+ }+ }+ if (exl3_cpu_tier() == 1) {+ switch (m.K) {+ case 1: band_gemm_avx2<1>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 2: band_gemm_avx2<2>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 3: band_gemm_avx2<3>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 4: band_gemm_avx2<4>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 5: band_gemm_avx2<5>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 6: band_gemm_avx2<6>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 7: band_gemm_avx2<7>(m, xh, n, c0 / 16, nb, acc, wc); return;+ case 8: band_gemm_avx2<8>(m, xh, n, c0 / 16, nb, acc, wc); return;+ default: break;+ }+ }+#endif+ (void) wc;+ for (int i = 0; i < n; ++i)+ for (int b = 0; b < nb; ++b) exl3_cpu_band_acc(m, xh[i], c0 + 128 * b, 0, m.k / 16, acc[i] + 128 * b);+}++void exl3_cpu_rows_acc(const Exl3Part& m, const float* xh, int ks0, int ks1, float* acc) {+ exl3_cpu_rows_acc_n(m, &xh, 1, ks0, ks1, &acc);+}++void exl3_cpu_band_out(const Exl3Part& m, const float* acc, float xsum, int c0, float* y) {+ const float ki = k_inv(), kz = k_zero() * xsum;+#if defined(EXL3_X86)+ if (exl3_cpu_tier() == 2) {+ band_out_avx512(acc, ki, kz, m.svh + c0, y);+ return;+ }+ if (exl3_cpu_tier() == 1) {+ band_out_avx2(acc, ki, kz, m.svh + c0, y);+ return;+ }+#endif+ for (int i = 0; i < 128; ++i) y[i] = acc[i] * ki + kz;+ had128(y);+ for (int i = 0; i < 128; ++i) y[i] *= f32_from_f16(m.svh[c0 + i]);+}++void exl3_cpu_band(const Exl3Part& m, const float* xh, float xsum, int c0, float* y) {+ float acc[128];+ exl3_cpu_band_acc(m, xh, c0, 0, m.k / 16, acc);+ exl3_cpu_band_out(m, acc, xsum, c0, y);+}++} // namespace strata::kernels::cpudiff --git a/src/dsv41/vendor/exl3_legacy.cu b/src/dsv41/vendor/exl3_legacy.cunew file mode 100644index 000000000..64ca50fd8--- /dev/null+++ b/src/dsv41/vendor/exl3_legacy.cu@@ -0,0 +1,965 @@+// src/kernels/cuda/exl3_legacy.cu - v1.1.8's EXL3 decode kernels, kept for NVIDIA GPUs before sm_80 (V100, T4,+// RTX 20): #180's decode kernels run the tiles on the tensor cores from sm_80 on, and their CUDA-core fallback was+// slower here than these (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). exl3.cu's mv / moe_gate_up / moe_down+// / win_experts / prepare / several_ok call these on such a GPU (STRATA_GLM_EXL3_DECODE=legacy|new picks one on any+// GPU); the prompt path is exl3.cu's on every GPU. The code below is v1.1.8's as it was, the prompt path's kernels+// left out.+//+// The codebooks and the trellis window arithmetic follow exllamav3 (MIT, turboderp - exllamav3_ext/quant/+// codebook.cuh, exl3_dq.cuh, hadamard_inner.cuh); the kernels around them are shaped for this engine's decode:+//+// * a block owns 128 outputs (8 tiles - exactly one output rotation block) over a slice of at most 4096 inputs;+// its 8 warps take the slice's 16-row tile rows in turn, each staging the row's 8 tiles in shared memory with+// coalesced loads and decoding 8 weights per lane (one fragment: 2 columns x 4 rows), F32 accumulation;+// * the input rotation runs per block on the slice it reads (F32 or q8_1 in, FP16 in shared memory), so the+// engine's activation buffers are used as they are;+// * a matrix with more than 4096 inputs, or too few output blocks to fill the device, splits its inputs over+// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in+// split order (deterministic) and runs the output rotation.+#if !defined(STRATA_USE_HIP) // (AMD runs exl3.cu's kernels; EXL3 does not run there anyway)+#include "strata/kernels/exl3.hpp"++#include <cuda_fp16.h>+#include <cuda_runtime.h>++#include <algorithm>+#include <atomic>+#include <cstdio>+#include <map>+#include <mutex>+#include <utility>++namespace strata::kernels::exl3::legacy {+namespace {++constexpr int kThreads = 256, kWarps = 8;+constexpr int kCols = 128; // outputs per block: 8 tiles, one rotation block+constexpr int kMaxKR = 4096; // inputs per block (the rotated slice lives in shared memory)+constexpr float kHad = 0.088388347648f; // 1 / sqrt(128)++struct Q81 { // the engine's q8_1 block (ggml block_q8_1): d, sum, 32 int8+ __half2 ds;+ int8_t qs[32];+};++std::atomic<int> g_launch_errors{0};+void launch_check(const char* what) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "exl3 %s: %s\n", what, cudaGetErrorString(e));+ g_launch_errors.fetch_add(1, std::memory_order_relaxed);+ }+}++// ---------------------------------------------------------------- the codebooks+__device__ __forceinline__ uint32_t bytesum_plus(uint32_t x, uint32_t acc) {+#if defined(STRATA_USE_HIP)+ return acc + (x & 0xffu) + ((x >> 8) & 0xffu) + ((x >> 16) & 0xffu) + (x >> 24);+#else+ return __dp4a(x, 0x01010101u, acc);+#endif+}++// two states -> two values, FP16 arithmetic as exllamav3's decode_3inst_2 (the tensor-core kernels take the half2)+template <int cb>+__device__ __forceinline__ __half2 decode2h(uint32_t s0, uint32_t s1) {+ if constexpr (cb == 2) {+ const uint32_t a = bytesum_plus(s0 * 0x83DCD12Du, 0x6400u);+ const uint32_t b = bytesum_plus(s1 * 0x83DCD12Du, 0x6400u);+ const __half2 h = __halves2half2(__ushort_as_half((unsigned short) a), __ushort_as_half((unsigned short) b));+ return __hfma2(h, __half2half2(__ushort_as_half(0x1eee)), __half2half2(__ushort_as_half(0xc931)));+ } else {+ uint32_t x0 = cb == 1 ? s0 * 0xCBAC1FEDu : s0 * 89226354u + 64248484u;+ uint32_t x1 = cb == 1 ? s1 * 0xCBAC1FEDu : s1 * 89226354u + 64248484u;+ x0 = (x0 & 0x8fff8fffu) ^ 0x3b603b60u;+ x1 = (x1 & 0x8fff8fffu) ^ 0x3b603b60u;+ const __half2 lo = __halves2half2(__ushort_as_half((unsigned short) (x0 & 0xffffu)),+ __ushort_as_half((unsigned short) (x1 & 0xffffu)));+ const __half2 hi = __halves2half2(__ushort_as_half((unsigned short) (x0 >> 16)),+ __ushort_as_half((unsigned short) (x1 >> 16)));+ return __hadd2(lo, hi);+ }+}+template <int cb>+__device__ __forceinline__ float2 decode2(uint32_t s0, uint32_t s1) {+ return __half22float2(decode2h<cb>(s0, s1));+}++// step p's state: the 16-bit window ending at bit (p + 1) K of the tile's MSB-first bit stream (w: its 8 K words),+// wrapping around the tile (the + 256 K keeps the start non-negative)+template <int K>+__device__ __forceinline__ uint32_t state_at(const uint32_t* w, int p) {+ constexpr int W = 8 * K;+ const int b1 = (p + 1 + 256) * K;+ const int i1 = (b1 - 1) >> 5, i0 = (b1 - 16) >> 5;+ const int sh = ((i1 + 1) << 5) - b1;+ const uint64_t m = ((uint64_t) w[i0 % W] << 32) | (uint64_t) w[i1 % W];+ return (uint32_t) (m >> sh) & 0xffffu;+}++// lane's 8 steps (lane * 8 + j): v[j] sits at tile row (lane & 3) * 2 + (j & 1) + 8 (j >> 1 & 1), column+// (lane >> 2) + 8 (j >> 2)+template <int K, int cb>+__device__ __forceinline__ void decode8(const uint32_t* w, int lane, float* v) {+ const int p0 = lane * 8;+#pragma unroll+ for (int j = 0; j < 8; j += 2) {+ const float2 d = decode2<cb>(state_at<K>(w, p0 + j), state_at<K>(w, p0 + j + 1));+ v[j] = d.x;+ v[j + 1] = d.y;+ }+}++// ---------------------------------------------------------------- the rotation+// the 128-point transform of the 128 values a warp holds 4 per lane (value lane * 4 + i), over sqrt(128)+__device__ __forceinline__ void had128(float* v, int lane) {+ const float s0 = v[0] + v[1], d0 = v[0] - v[1], s1 = v[2] + v[3], d1 = v[2] - v[3];+ v[0] = s0 + s1;+ v[1] = d0 + d1;+ v[2] = s0 - s1;+ v[3] = d0 - d1;+#pragma unroll+ for (int m = 1; m < 32; m <<= 1) {+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float o = __shfl_xor_sync(0xffffffffu, v[i], m);+ v[i] = (lane & m) ? o - v[i] : v[i] + o;+ }+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] *= kHad;+}++__device__ __forceinline__ float h2f(uint16_t b) { return __half2float(__ushort_as_half(b)); }++// xs[i] = H(x * suh)[k0 + i] for i < kr (kr % 128 == 0); x from xf (F32) or xq (q8_1); every warp of the block+__device__ void rotate_in(const float* __restrict__ xf, const Q81* __restrict__ xq, const uint16_t* __restrict__ suh,+ int k0, int kr, __half* xs) {+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ for (int c = warp; c < kr / 128; c += kWarps) {+ const int base = k0 + c * 128 + lane * 4;+ float v[4];+ if (xf != nullptr) {+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = xf[base + i];+ } else {+ const Q81* b = xq + base / 32;+ const float d = __low2float(b->ds);+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = d * (float) b->qs[base % 32 + i];+ }+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] *= h2f(suh[base + i]);+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) xs[c * 128 + lane * 4 + i] = __float2half_rn(v[i]);+ }+}++// ---------------------------------------------------------------- the block GEMV+// a lane's 4 weights of one column times the activation's 4 inputs, added to a: the rounding spelled out, so the+// one-activation and the several-activation GEMVs (block_gemv, block_gemv_n) give each activation the same bits+__device__ __forceinline__ float dot4_acc(float a, const float* v, float x0, float x1, float x2, float x3) {+ return __fadd_rn(a, __fmaf_rn(v[3], x3, __fmaf_rn(v[2], x2, __fmaf_rn(v[1], x1, __fmul_rn(v[0], x0)))));+}++// red[c] (c < 128) = sum over the 16-row slices [ks0, ks1) of x' . W'[:, tc0 * 16 + c], x' = xs from slice ks0 on.+// tr: the matrix's first tile, ld tiles per slice row. stage: kWarps * 64 K words, red: kWarps * 128 floats.+// Ends with a __syncthreads (red[0..127] complete).+template <int K, int cb>+__device__ void block_gemv(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,+ uint32_t* stage, float* red) {+ constexpr int W = 8 * K, TW = 8 * W;+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ uint32_t* st = stage + warp * TW;+ float acc[8][2];+#pragma unroll+ for (int t = 0; t < 8; ++t) acc[t][0] = acc[t][1] = 0.0f;+ const int kk = (lane & 3) * 2;+ for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {+ const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;+ __syncwarp();+#pragma unroll+ for (int i = lane; i < TW; i += 32) st[i] = src[i];+ __syncwarp();+ const __half* xk = xs + (ks - ks0) * 16;+ const float x0 = __half2float(xk[kk]), x1 = __half2float(xk[kk + 1]);+ const float x2 = __half2float(xk[kk + 8]), x3 = __half2float(xk[kk + 9]);+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ float v[8];+ decode8<K, cb>(st + t * W, lane, v);+ acc[t][0] = dot4_acc(acc[t][0], v, x0, x1, x2, x3);+ acc[t][1] = dot4_acc(acc[t][1], v + 4, x0, x1, x2, x3);+ }+ }+ // the 4 lanes of one lane / 4 hold the same two columns of every tile+#pragma unroll+ for (int t = 0; t < 8; ++t)+#pragma unroll+ for (int c = 0; c < 2; ++c) {+ acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 1);+ acc[t][c] += __shfl_xor_sync(0xffffffffu, acc[t][c], 2);+ }+ if ((lane & 3) == 0)+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ red[warp * kCols + t * 16 + (lane >> 2)] = acc[t][0];+ red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[t][1];+ }+ __syncthreads();+ if (threadIdx.x < kCols) {+ float s = 0.0f;+#pragma unroll+ for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];+ red[threadIdx.x] = s;+ }+ __syncthreads();+}++// block_gemv for nc activations at once (nc <= kNC; activation c's rotated slice at xs + c * kMaxKR): each lane decodes+// its 8 weights of a tile once and adds them into every activation's sums - as exllamav3's small-m GEMV and fused MoE+// kernels share a tile's decode among the rows of an MMA, but with block_gemv's own F32 arithmetic, so outc[c * 128 ++// i] (i < 128) is bit for bit block_gemv's red[i] for activation c alone. Ends with a __syncthreads.+constexpr int kNC = 4;+template <int K, int cb>+__device__ void block_gemv_n(const uint32_t* __restrict__ tr, int ld, int tc0, int ks0, int ks1, const __half* xs,+ int nc, uint32_t* stage, float* red, float* outc) {+ constexpr int W = 8 * K, TW = 8 * W;+ const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5;+ uint32_t* st = stage + warp * TW;+ float acc[kNC][8][2];+#pragma unroll+ for (int c = 0; c < kNC; ++c)+#pragma unroll+ for (int t = 0; t < 8; ++t) acc[c][t][0] = acc[c][t][1] = 0.0f;+ const int kk = (lane & 3) * 2;+ for (int ks = ks0 + warp; ks < ks1; ks += kWarps) {+ const uint32_t* src = tr + ((size_t) ks * ld + tc0) * W;+ __syncwarp();+#pragma unroll+ for (int i = lane; i < TW; i += 32) st[i] = src[i];+ __syncwarp();+ float x[kNC][4];+#pragma unroll+ for (int c = 0; c < kNC; ++c) {+ const __half* xk = xs + c * kMaxKR + (ks - ks0) * 16;+ if (c < nc) {+ x[c][0] = __half2float(xk[kk]);+ x[c][1] = __half2float(xk[kk + 1]);+ x[c][2] = __half2float(xk[kk + 8]);+ x[c][3] = __half2float(xk[kk + 9]);+ }+ }+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ float v[8];+ decode8<K, cb>(st + t * W, lane, v);+#pragma unroll+ for (int c = 0; c < kNC; ++c)+ if (c < nc) {+ acc[c][t][0] = dot4_acc(acc[c][t][0], v, x[c][0], x[c][1], x[c][2], x[c][3]);+ acc[c][t][1] = dot4_acc(acc[c][t][1], v + 4, x[c][0], x[c][1], x[c][2], x[c][3]);+ }+ }+ }+ // per activation, block_gemv's reduction+#pragma unroll+ for (int c = 0; c < kNC; ++c) {+ if (c >= nc) break;+#pragma unroll+ for (int t = 0; t < 8; ++t)+#pragma unroll+ for (int h = 0; h < 2; ++h) {+ acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 1);+ acc[c][t][h] += __shfl_xor_sync(0xffffffffu, acc[c][t][h], 2);+ }+ if ((lane & 3) == 0)+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ red[warp * kCols + t * 16 + (lane >> 2)] = acc[c][t][0];+ red[warp * kCols + t * 16 + 8 + (lane >> 2)] = acc[c][t][1];+ }+ __syncthreads();+ if (threadIdx.x < kCols) {+ float s = 0.0f;+#pragma unroll+ for (int w = 0; w < kWarps; ++w) s += red[w * kCols + threadIdx.x];+ outc[c * kCols + threadIdx.x] = s;+ }+ __syncthreads();+ }+}++// one warp: y[c] = alpha * (H(red) * svh)[c] + bias[c] for the block's 128 outputs+__device__ __forceinline__ void epilogue(const float* red, const uint16_t* svh, float alpha, const float* bias,+ float* y) {+ const int lane = threadIdx.x & 31;+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const int c = lane * 4 + i;+ y[c] = __fmaf_rn(__fmul_rn(alpha, v[i]), h2f(svh[c]), bias != nullptr ? bias[c] : 0.0f);+ }+}++// ---------------------------------------------------------------- mv+struct KJob {+ const uint32_t* tr;+ const uint16_t* suh;+ const uint16_t* svh;+ const float* xf;+ const Q81* xq;+ float* y;+ const float* bias;+ float* part; // S x n partial sums (S > 1); mv_n_kernel: nc x S x n+ unsigned* cnt; // n / 128 counters, zero between launches+ float alpha;+ int k, n, ld, S, kr;+ int nc; // mv_n_kernel: the activations (1..kNC), xf / xq / y advanced by these per activation+ size_t xf_cs, xq_cs, y_cs; // (floats, bytes, floats)+};+struct KBatch {+ KJob j[kMaxMvJobs];+ int blk_end[kMaxMvJobs];+ int n;+};++// the dynamic shared memory of the several-activation kernels (mv_n_kernel, win_gu_kernel, win_down_kernel):+// red[kWarps][128] outc[kNC][128] gu[2][kNC][128] (F32), stage[kWarps * 64 K] (words), xs[kNC][kMaxKR] (FP16) - up to+// 58 KB, past the 48 KB a launch gets without asking (set_smem)+constexpr size_t smem_n_bytes(int K) {+ return (size_t) (kWarps * kCols + kNC * kCols + 2 * kNC * kCols) * 4 + (size_t) kWarps * 64 * K * 4 ++ (size_t) kNC * kMaxKR * 2;+}+struct SmemN {+ float *red, *outc, *gu;+ uint32_t* stage;+ __half* xs;+};+template <int K>+__device__ __forceinline__ SmemN smem_n(unsigned char* p) {+ SmemN s;+ s.red = (float*) p;+ s.outc = s.red + kWarps * kCols;+ s.gu = s.outc + kNC * kCols;+ s.stage = (uint32_t*) (s.gu + 2 * kNC * kCols);+ s.xs = (__half*) (s.stage + kWarps * 64 * K);+ return s;+}++template <int K, int cb>+__global__ void __launch_bounds__(kThreads) mv_kernel(const KBatch b) {+ __shared__ __half xs[kMaxKR];+ __shared__ uint32_t stage[kWarps * 64 * K];+ __shared__ float red[kWarps * kCols];+ __shared__ int last;+ int ji = 0;+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;+ const KJob& J = b.j[ji];+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);+ const int g = local / J.S, sp = local % J.S;+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);+ rotate_in(J.xf, J.xq, J.suh, k0, k1 - k0, xs);+ __syncthreads();+ block_gemv<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, xs, stage, red);+ const int warp = threadIdx.x >> 5;+ if (J.S == 1) {+ if (warp == 0)+ epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr,+ J.y + g * kCols);+ return;+ }+ if (threadIdx.x < kCols) J.part[(size_t) sp * J.n + g * kCols + threadIdx.x] = red[threadIdx.x];+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < kCols) {+ float s = 0.0f;+ for (int i = 0; i < J.S; ++i) s += *(volatile const float*) &J.part[(size_t) i * J.n + g * kCols + threadIdx.x];+ red[threadIdx.x] = s;+ }+ __syncthreads();+ if (warp == 0)+ epilogue(red, J.svh + g * kCols, J.alpha, J.bias != nullptr ? J.bias + g * kCols : nullptr, J.y + g * kCols);+ if (threadIdx.x == 0) J.cnt[g] = 0u;+}++// mv_kernel for a job's nc activations at once (a verify window's tokens): the same input split, the weights read and+// decoded once for all of them (block_gemv_n), each activation's partials and sums in mv_kernel's order - every+// activation's y is mv_kernel's for it alone, bit for bit+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) mv_n_kernel(const KBatch b) {+ extern __shared__ __align__(16) unsigned char smem_raw[];+ const SmemN sm = smem_n<K>(smem_raw);+ __shared__ int last;+ int ji = 0;+ while ((int) blockIdx.x >= b.blk_end[ji]) ++ji;+ const KJob& J = b.j[ji];+ const int local = (int) blockIdx.x - (ji > 0 ? b.blk_end[ji - 1] : 0);+ const int g = local / J.S, sp = local % J.S;+ const int k0 = sp * J.kr, k1 = min(J.k, k0 + J.kr);+ for (int c = 0; c < J.nc; ++c)+ rotate_in(J.xf != nullptr ? J.xf + c * J.xf_cs : nullptr,+ J.xq != nullptr ? (const Q81*) ((const uint8_t*) J.xq + c * J.xq_cs) : nullptr, J.suh, k0, k1 - k0,+ sm.xs + c * kMaxKR);+ __syncthreads();+ block_gemv_n<K, cb>(J.tr, J.ld, g * 8, k0 / 16, k1 / 16, sm.xs, J.nc, sm.stage, sm.red, sm.outc);+ const int warp = threadIdx.x >> 5;+ const float* bias = J.bias != nullptr ? J.bias + g * kCols : nullptr;+ if (J.S == 1) {+ if (warp < J.nc)+ epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);+ return;+ }+ if (threadIdx.x < kCols)+ for (int c = 0; c < J.nc; ++c)+ J.part[((size_t) c * J.S + sp) * J.n + g * kCols + threadIdx.x] = sm.outc[c * kCols + threadIdx.x];+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(&J.cnt[g], 1u) == (unsigned) (J.S - 1);+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < kCols)+ for (int c = 0; c < J.nc; ++c) {+ float s = 0.0f;+ for (int i = 0; i < J.S; ++i)+ s += *(volatile const float*) &J.part[((size_t) c * J.S + i) * J.n + g * kCols + threadIdx.x];+ sm.outc[c * kCols + threadIdx.x] = s;+ }+ __syncthreads();+ if (warp < J.nc) epilogue(sm.outc + warp * kCols, J.svh + g * kCols, J.alpha, bias, J.y + warp * J.y_cs + g * kCols);+ if (threadIdx.x == 0) J.cnt[g] = 0u;+}++// ---------------------------------------------------------------- the routed experts+struct KMoe {+ const unsigned long long* plan_ptr;+ const float* plan_w;+ const float* cpu_part;+ const int* cpu_flag;+ const float* x;+ float* h; // [k][n_ff]+ const float* sh_out;+ float* out;+ float* part; // [k][n_embd]+ unsigned* cnt; // n_embd / 128+ size_t suh[3], svh[3], tr[3];+ int k, n_embd, n_ff;+ float limit;+};++// the pieces the token's experts (moe_*_kernel) and a window's grouped experts (win_*_kernel) share, so each window+// token gets the token path's bits: one warp's output block of 128 (H, then svh) ...+__device__ __forceinline__ void block_out(const float* red, const uint16_t* svh, float* dst) {+ const int lane = threadIdx.x & 31;+ float v[4];+#pragma unroll+ for (int i = 0; i < 4; ++i) v[i] = red[lane * 4 + i];+ had128(v, lane);+#pragma unroll+ for (int i = 0; i < 4; ++i) dst[lane * 4 + i] = __fmul_rn(v[i], h2f(svh[lane * 4 + i]));+}+// ... the clamped swiglu ...+__device__ __forceinline__ float swiglu(float gv, float uv, float limit) {+ if (limit > 0.0f) {+ gv = fminf(gv, limit);+ uv = fminf(fmaxf(uv, -limit), limit);+ }+ return __fmul_rn(__fdiv_rn(gv, __fadd_rn(1.0f, expf(-gv))), uv);+}+// ... and one output's sum: the experts' rows in plan order (p[i]: expert i's, 0 for one not on the device), the CPU+// lane's part when flagged (cpu_part: null when not), the shared expert's row (sh: 0 without one)+__device__ __forceinline__ float combine(const float* plan_w, const float* p, int k, const float* cpu_part, float sh) {+ float m = 0.0f;+ for (int i = 0; i < k; ++i) m = __fmaf_rn(plan_w[i], p[i], m);+ if (cpu_part != nullptr) m = __fadd_rn(m, *cpu_part);+ return __fadd_rn(m, sh);+}++// grid (n_ff / 128, k): expert blockIdx.y's gate and up for 128 h values, then the clamped swiglu+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) moe_gate_up_kernel(const KMoe a) {+ __shared__ __half xs[2][kMaxKR];+ __shared__ uint32_t stage[kWarps * 64 * K];+ __shared__ float red[kWarps * kCols];+ __shared__ float gu[2][kCols];+ const int ei = blockIdx.y, g = blockIdx.x;+ const unsigned long long ptr = a.plan_ptr[ei];+ if (ptr == 0ull) return;+ const uint8_t* blob = (const uint8_t*) ptr;+ const int warp = threadIdx.x >> 5;+ for (int r = 0; r < 2; ++r) rotate_in(a.x, nullptr, (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, xs[r]);+ __syncthreads();+ for (int r = 0; r < 2; ++r) {+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, xs[r], stage, red);+ if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[r]) + g * kCols, gu[r]);+ __syncthreads();+ }+ if (threadIdx.x < kCols)+ a.h[(size_t) ei * a.n_ff + g * kCols + threadIdx.x] = swiglu(gu[0][threadIdx.x], gu[1][threadIdx.x], a.limit);+}++// grid (n_embd / 128, k): expert blockIdx.y's down rows for 128 outputs; the last block of each output block adds+// the experts up in plan order (as the engine's moe_down), then the CPU lane's part and the shared expert+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) moe_down_kernel(const KMoe a) {+ __shared__ __half xs[kMaxKR];+ __shared__ uint32_t stage[kWarps * 64 * K];+ __shared__ float red[kWarps * kCols];+ __shared__ int last;+ const int ei = blockIdx.y, g = blockIdx.x;+ const unsigned long long ptr = a.plan_ptr[ei];+ const int warp = threadIdx.x >> 5;+ float* part = a.part + (size_t) ei * a.n_embd + g * kCols;+ if (ptr != 0ull) {+ const uint8_t* blob = (const uint8_t*) ptr;+ rotate_in(a.h + (size_t) ei * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0, a.n_ff, xs);+ __syncthreads();+ block_gemv<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, xs, stage, red);+ if (warp == 0) block_out(red, (const uint16_t*) (blob + a.svh[2]) + g * kCols, part);+ } else if (threadIdx.x < kCols) {+ part[threadIdx.x] = 0.0f;+ }+ __threadfence();+ __syncthreads();+ if (threadIdx.x == 0) last = atomicAdd(&a.cnt[g], 1u) == (unsigned) (a.k - 1);+ __syncthreads();+ if (!last) return;+ __threadfence();+ if (threadIdx.x < kCols) {+ const int r = g * kCols + threadIdx.x;+ float p[8];+ for (int i = 0; i < a.k; ++i) p[i] = *(volatile const float*) &a.part[(size_t) i * a.n_embd + r];+ a.out[r] = combine(a.plan_w, p, a.k, a.cpu_flag != nullptr && *a.cpu_flag ? a.cpu_part + r : nullptr,+ a.sh_out != nullptr ? a.sh_out[r] : 0.0f);+ }+ if (threadIdx.x == 0) a.cnt[g] = 0u;+}++// ---------------------------------------------------------------- a verify window's routed experts+// gf::moe_win_experts' grouping (group g: one expert, its blob at grp_ptr[g]; its entries e = t * 8 + i at+// grp_ent[grp_start[g] ..)), each group's weights read and decoded once for up to kNC of its tokens at a time (as+// exllamav3's fused MoE runs a run of slots that picked one expert as the rows of one MMA tile), with the token+// path's arithmetic per entry: every token's output is moe_gate_up + moe_down's for it alone, bit for bit+struct KWin {+ const unsigned long long* grp_ptr;+ const int *grp_start, *grp_ent, *n_groups;+ const unsigned long long* plan_ptr;+ const float* plan_w;+ const float* cpu_part;+ const int* cpu_flag;+ const float* x;+ size_t x_stride;+ float* h; // [T * 8][n_ff]+ float* part; // [T * 8][n_embd]+ const float* sh_out; // [T][n_embd]+ float* out;+ size_t out_stride;+ size_t suh[3], svh[3], tr[3];+ int k, n_embd, n_ff;+ float limit;+};++// grid (n_ff / 128, groups at most): group blockIdx.y's gate and up for 128 h values of each entry, the clamped swiglu+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) win_gu_kernel(const KWin a) {+ extern __shared__ __align__(16) unsigned char smem_raw[];+ const SmemN sm = smem_n<K>(smem_raw);+ const int grp = blockIdx.y, g = blockIdx.x;+ if (grp >= *a.n_groups) return;+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];+ const int warp = threadIdx.x >> 5;+ const int m1 = a.grp_start[grp + 1];+ for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {+ const int nc = min(kNC, m1 - m0);+ for (int r = 0; r < 2; ++r) {+ for (int c = 0; c < nc; ++c)+ rotate_in(a.x + (size_t) (a.grp_ent[m0 + c] >> 3) * a.x_stride, nullptr,+ (const uint16_t*) (blob + a.suh[r]), 0, a.n_embd, sm.xs + c * kMaxKR);+ __syncthreads();+ block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[r]), a.n_ff / 16, g * 8, 0, a.n_embd / 16, sm.xs, nc,+ sm.stage, sm.red, sm.outc);+ if (warp < nc)+ block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[r]) + g * kCols,+ sm.gu + (r * kNC + warp) * kCols);+ __syncthreads();+ }+ if (threadIdx.x < kCols)+ for (int c = 0; c < nc; ++c)+ a.h[(size_t) a.grp_ent[m0 + c] * a.n_ff + g * kCols + threadIdx.x] =+ swiglu(sm.gu[c * kCols + threadIdx.x], sm.gu[(kNC + c) * kCols + threadIdx.x], a.limit);+ __syncthreads();+ }+}++// grid (n_embd / 128, groups at most): group blockIdx.y's down rows for 128 outputs of each entry into its part rows+template <int K, int cb>+__global__ void __launch_bounds__(kThreads) win_down_kernel(const KWin a) {+ extern __shared__ __align__(16) unsigned char smem_raw[];+ const SmemN sm = smem_n<K>(smem_raw);+ const int grp = blockIdx.y, g = blockIdx.x;+ if (grp >= *a.n_groups) return;+ const uint8_t* blob = (const uint8_t*) a.grp_ptr[grp];+ const int warp = threadIdx.x >> 5;+ const int m1 = a.grp_start[grp + 1];+ for (int m0 = a.grp_start[grp]; m0 < m1; m0 += kNC) {+ const int nc = min(kNC, m1 - m0);+ for (int c = 0; c < nc; ++c)+ rotate_in(a.h + (size_t) a.grp_ent[m0 + c] * a.n_ff, nullptr, (const uint16_t*) (blob + a.suh[2]), 0,+ a.n_ff, sm.xs + c * kMaxKR);+ __syncthreads();+ block_gemv_n<K, cb>((const uint32_t*) (blob + a.tr[2]), a.n_embd / 16, g * 8, 0, a.n_ff / 16, sm.xs, nc,+ sm.stage, sm.red, sm.outc);+ if (warp < nc)+ block_out(sm.outc + warp * kCols, (const uint16_t*) (blob + a.svh[2]) + g * kCols,+ a.part + (size_t) a.grp_ent[m0 + warp] * a.n_embd + g * kCols);+ __syncthreads();+ }+}++// grid (n_embd / 128, T): each token's outputs, moe_down_kernel's combine (an entry off the device adds 0)+__global__ void __launch_bounds__(kCols) win_combine_kernel(const KWin a) {+ const int t = blockIdx.y, r = blockIdx.x * kCols + threadIdx.x;+ float p[8];+ for (int i = 0; i < a.k; ++i) {+ const int e = t * 8 + i;+ p[i] = a.plan_ptr[e] != 0ull ? a.part[(size_t) e * a.n_embd + r] : 0.0f;+ }+ a.out[(size_t) t * a.out_stride + r] =+ combine(a.plan_w + t * 8, p, a.k, a.cpu_flag[t] ? a.cpu_part + (size_t) t * a.n_embd + r : nullptr,+ a.sh_out != nullptr ? a.sh_out[(size_t) t * a.n_embd + r] : 0.0f);+}++// ---------------------------------------------------------------- dispatch+// F(K, cb) for K 1..8 and the three codebooks+template <template <int, int> class F, typename... A>+bool dispatch(int K, int cb, A&&... args) {+#define EXL3_CASE(KK) \+ case KK: \+ if (cb == 2) F<KK, 2>::run(std::forward<A>(args)...); \+ else if (cb == 1) F<KK, 1>::run(std::forward<A>(args)...); \+ else F<KK, 0>::run(std::forward<A>(args)...); \+ return true;+ switch (K) {+ EXL3_CASE(1) EXL3_CASE(2) EXL3_CASE(3) EXL3_CASE(4) EXL3_CASE(5) EXL3_CASE(6) EXL3_CASE(7) EXL3_CASE(8)+ default: return false;+ }+#undef EXL3_CASE+}++template <int K, int cb> struct RunMv {+ static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }+};+template <int K, int cb> struct RunMvN {+ static void run(const KBatch& b, int blocks, cudaStream_t s) {+ mv_n_kernel<K, cb><<<blocks, kThreads, smem_n_bytes(K), s>>>(b);+ }+};+template <int K, int cb> struct RunWin {+ static void run(const KWin& a, int groups, cudaStream_t s) {+ win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups), kThreads, smem_n_bytes(K), s>>>(a);+ }+};+template <int K, int cb> struct RunWinDown {+ static void run(const KWin& a, int groups, cudaStream_t s) {+ win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, smem_n_bytes(K),+ s>>>(a);+ }+};+template <int K, int cb> struct AllowSmemN {+ static void run() {+ const int b = (int) smem_n_bytes(K);+ cudaFuncSetAttribute(mv_n_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);+ cudaFuncSetAttribute(win_gu_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);+ cudaFuncSetAttribute(win_down_kernel<K, cb>, cudaFuncAttributeMaxDynamicSharedMemorySize, b);+ }+};+template <int K, int cb> struct RunGU {+ static void run(const KMoe& a, cudaStream_t s) {+ moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ }+};+template <int K, int cb> struct RunDown {+ static void run(const KMoe& a, cudaStream_t s) {+ moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ }+};+// ---------------------------------------------------------------- per-stream scratch+constexpr size_t kPartFloats = (size_t) 2 << 20; // split partials: 8 MB+constexpr size_t kCounters = 16384;+constexpr size_t kHFloats = (size_t) 8 * 8192; // the experts' h+struct Scratch {+ float* part = nullptr;+ unsigned* cnt = nullptr;+ float* h = nullptr;+};+std::mutex g_mu;+std::map<std::pair<int, cudaStream_t>, Scratch> g_scratch;++Scratch* scratch(cudaStream_t s) {+ int dev = 0;+ cudaGetDevice(&dev);+ std::lock_guard<std::mutex> lk(g_mu);+ auto it = g_scratch.find({dev, s});+ if (it != g_scratch.end()) return &it->second;+ cudaStreamCaptureStatus cs = cudaStreamCaptureStatusNone;+ if (cudaStreamIsCapturing(s, &cs) == cudaSuccess && cs != cudaStreamCaptureStatusNone) {+ std::fprintf(stderr, "exl3: scratch for a stream first seen inside a graph capture (call exl3::prepare)\n");+ return nullptr;+ }+ Scratch sc;+ if (cudaMalloc(&sc.part, kPartFloats * sizeof(float)) != cudaSuccess ||+ cudaMalloc(&sc.cnt, kCounters * sizeof(unsigned)) != cudaSuccess ||+ cudaMalloc(&sc.h, kHFloats * sizeof(float)) != cudaSuccess ||+ cudaMemsetAsync(sc.cnt, 0, kCounters * sizeof(unsigned), s) != cudaSuccess) {+ std::fprintf(stderr, "exl3: scratch did not allocate\n");+ return nullptr;+ }+ cudaStreamSynchronize(s);+ return &(g_scratch[{dev, s}] = sc);+}++int sm_count() {+ int dev = 0, sms = 0;+ cudaGetDevice(&dev);+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);+ return std::max(1, sms);+}++// the several-activation kernels may use smem_n_bytes of shared memory: asked once per device (prepare, before a graph+// capture; else at the first such launch). A device that cannot give a block that much (48 KB GPUs) runs one+// activation at a time instead+constexpr int kMaxDev = 64;+std::atomic<int> g_smem_n[kMaxDev]; // 0 not asked, 1 allowed, 2 refused+bool allow_smem_n() {+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= kMaxDev) return false;+ const int st = g_smem_n[dev].load(std::memory_order_acquire);+ if (st != 0) return st == 1;+ std::lock_guard<std::mutex> lk(g_mu);+ if (g_smem_n[dev].load(std::memory_order_relaxed) != 0) return g_smem_n[dev].load() == 1;+ int optin = 0;+ cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);+ bool ok = (size_t) optin >= smem_n_bytes(8);+ if (ok) {+ for (int K = 1; K <= 8; ++K)+ for (int cb = 0; cb <= 2; ++cb) dispatch<AllowSmemN>(K, cb);+ ok = cudaGetLastError() == cudaSuccess;+ }+ if (!ok)+ std::fprintf(stderr, "exl3: CUDA%d gives a block %d bytes of shared memory, the several-activation kernels "+ "need %zu - a verify window's tokens run one at a time there\n", dev, optin,+ smem_n_bytes(8));+ g_smem_n[dev].store(ok ? 1 : 2, std::memory_order_release);+ return ok;+}++// mv's input split for m: S blocks of kr inputs per output block, at most kMaxKR inputs a block and about two blocks+// per SM (the same for every activation count: mv_kernel and mv_n_kernel sum each activation alike)+void mv_split(const Mat& m, int sms, int& S, int& kr) {+ const int groups = m.n / kCols;+ S = (m.k + kMaxKR - 1) / kMaxKR;+ while (groups * S < 2 * sms && m.k / (S * 2) >= 256) S *= 2;+ kr = ((m.k + S - 1) / S + 127) / 128 * 128;+ S = (m.k + kr - 1) / kr;+}++} // namespace++int launch_errors() { return g_launch_errors.load(std::memory_order_relaxed); }++bool prepare(cudaStream_t s) {+ if (scratch(s) == nullptr) return false;+ (void) allow_smem_n();+ return true;+}++bool several_ok() { return allow_smem_n(); }++bool mv(const MvJob* jobs, int n, cudaStream_t s) {+ if (n <= 0 || n > kMaxMvJobs) return false;+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ const int sms = sm_count();+ // the launch units: a job of one activation is mv_kernel's, a job of several mv_n_kernel's in runs of up to kNC+ // (one mv_kernel unit an activation where the device lacks the shared memory)+ struct Unit {+ int j, c0, nc;+ };+ Unit units[kMaxMvJobs * kMaxMvCols];+ int nu = 0;+ bool several = false;+ for (int j = 0; j < n; ++j) several = several || jobs[j].ncol > 1;+ const int run = several && allow_smem_n() ? kNC : 1;+ for (int j = 0; j < n; ++j) {+ const Mat& m = *jobs[j].m;+ if (!supported(m) || jobs[j].y == nullptr || (jobs[j].xf == nullptr && jobs[j].xq == nullptr) ||+ jobs[j].ncol < 1 || jobs[j].ncol > kMaxMvCols) {+ std::fprintf(stderr, "exl3 mv: job %d unsupported (k %d n %d K %d cb %d, %d activations)\n", j, m.k, m.n,+ m.K, m.cb, jobs[j].ncol);+ return false;+ }+ for (int c0 = 0; c0 < jobs[j].ncol; c0 += run) units[nu++] = {j, c0, std::min(run, jobs[j].ncol - c0)};+ }+ bool done[sizeof(units) / sizeof(units[0])] = {};+ for (int i = 0; i < nu; ++i) {+ if (done[i]) continue;+ const Mat& m0 = *jobs[units[i].j].m;+ const bool multi = units[i].nc > 1;+ // a launch: the units of one (K, codebook, kernel), as many as KBatch and the split scratch take (the next+ // launch on the stream may use the scratch again: this one's last blocks are done with it by then)+ KBatch b{};+ int blocks = 0, nb = 0;+ size_t part_at = 0, cnt_at = 0;+ for (int u = i; u < nu && nb < kMaxMvJobs; ++u) {+ const MvJob& job = jobs[units[u].j];+ const Mat& m = *job.m;+ if (done[u] || m.K != m0.K || m.cb != m0.cb || (units[u].nc > 1) != multi) continue;+ const int c0 = units[u].c0, nc = units[u].nc, groups = m.n / kCols;+ int S = 1, kr = 0;+ mv_split(m, sms, S, kr);+ const size_t need = S > 1 ? (size_t) S * m.n * nc : 0;+ if (S > 1 && (part_at + need > kPartFloats || cnt_at + (size_t) groups > kCounters)) {+ if (nb > 0) continue; // the next launch+ std::fprintf(stderr, "exl3 mv: scratch too small for k %d n %d (%d activations)\n", m.k, m.n, nc);+ return false;+ }+ done[u] = true;+ KJob& J = b.j[nb];+ J.tr = m.trellis;+ J.suh = m.suh;+ J.svh = m.svh;+ J.xf = job.xf != nullptr ? job.xf + c0 * job.xf_cs : nullptr;+ J.xq = job.xq != nullptr ? (const Q81*) ((const uint8_t*) job.xq + c0 * job.xq_cs) : nullptr;+ J.y = job.y + c0 * job.y_cs;+ J.bias = job.bias;+ J.alpha = job.alpha;+ J.k = m.k;+ J.n = m.n;+ J.ld = m.ld > 0 ? m.ld : m.n / 16;+ J.S = S;+ J.kr = kr;+ J.part = sc->part + part_at;+ J.cnt = sc->cnt + cnt_at;+ J.nc = nc;+ J.xf_cs = job.xf_cs;+ J.xq_cs = job.xq_cs;+ J.y_cs = job.y_cs;+ part_at += need;+ if (S > 1) cnt_at += (size_t) groups;+ blocks += groups * S;+ b.blk_end[nb++] = blocks;+ }+ b.n = nb;+ if (!(multi ? dispatch<RunMvN>(m0.K, m0.cb, b, blocks, s) : dispatch<RunMv>(m0.K, m0.cb, b, blocks, s)))+ return false;+ }+ launch_check("mv");+ return true;+}++static bool moe_args(const MoeArgs& a, KMoe& m, cudaStream_t s) {+ Scratch* sc = scratch(s);+ if (sc == nullptr) return false;+ if (a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||+ a.n_ff > kMaxKR || (size_t) a.k * a.n_ff > kHFloats || (size_t) a.k * a.n_embd > kPartFloats ||+ (size_t) (a.n_embd / kCols) > kCounters) {+ std::fprintf(stderr, "exl3 moe: geometry not covered (k %d n_embd %d n_ff %d)\n", a.k, a.n_embd, a.n_ff);+ return false;+ }+ m = KMoe{};+ m.plan_ptr = a.plan_ptr;+ m.plan_w = a.plan_w;+ m.cpu_part = a.cpu_part;+ m.cpu_flag = a.cpu_flag;+ m.h = sc->h;+ m.part = sc->part;+ m.cnt = sc->cnt;+ for (int r = 0; r < 3; ++r) {+ m.suh[r] = a.lay.suh[r];+ m.svh[r] = a.lay.svh[r];+ m.tr[r] = a.lay.trellis[r];+ }+ m.k = a.k;+ m.n_embd = a.n_embd;+ m.n_ff = a.n_ff;+ m.limit = a.limit;+ return true;+}++void moe_gate_up(const MoeArgs& a, const float* x, float* h, cudaStream_t s) {+ KMoe m;+ if (!moe_args(a, m, s)) return;+ if (a.lay.K[0] != a.lay.K[1]) {+ std::fprintf(stderr, "exl3 moe_gate_up: gate K %d != up K %d\n", a.lay.K[0], a.lay.K[1]);+ return;+ }+ m.x = x;+ if (h != nullptr) m.h = h;+ if (!dispatch<RunGU>(a.lay.K[0], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_gate_up: K %d\n", a.lay.K[0]);+ launch_check("moe_gate_up");+}++void moe_down(const MoeArgs& a, const float* h, const float* sh_out, float* out, cudaStream_t s) {+ KMoe m;+ if (!moe_args(a, m, s)) return;+ if (h != nullptr) m.h = const_cast<float*>(h);+ m.sh_out = sh_out;+ m.out = out;+ if (!dispatch<RunDown>(a.lay.K[2], a.lay.cb, m, s)) std::fprintf(stderr, "exl3 moe_down: K %d\n", a.lay.K[2]);+ launch_check("moe_down");+}++bool win_experts(const WinArgs& a, cudaStream_t s) {+ if (a.T < 1 || a.k < 1 || a.k > 8 || a.n_embd % kCols != 0 || a.n_ff % kCols != 0 || a.n_embd > kMaxKR ||+ a.n_ff > kMaxKR || a.lay.K[0] != a.lay.K[1] || a.h == nullptr || a.part == nullptr || a.out == nullptr) {+ std::fprintf(stderr, "exl3 win_experts: not covered (T %d k %d n_embd %d n_ff %d, gate K %d up K %d)\n", a.T,+ a.k, a.n_embd, a.n_ff, a.lay.K[0], a.lay.K[1]);+ return false;+ }+ if (!allow_smem_n()) return false;+ KWin w{};+ w.grp_ptr = a.grp_ptr;+ w.grp_start = a.grp_start;+ w.grp_ent = a.grp_ent;+ w.n_groups = a.n_groups;+ w.plan_ptr = a.plan_ptr;+ w.plan_w = a.plan_w;+ w.cpu_part = a.cpu_part;+ w.cpu_flag = a.cpu_flag;+ w.x = a.x;+ w.x_stride = a.x_stride;+ w.h = a.h;+ w.part = a.part;+ w.sh_out = a.sh_out;+ w.out = a.out;+ w.out_stride = a.out_stride;+ for (int r = 0; r < 3; ++r) {+ w.suh[r] = a.lay.suh[r];+ w.svh[r] = a.lay.svh[r];+ w.tr[r] = a.lay.trellis[r];+ }+ w.k = a.k;+ w.n_embd = a.n_embd;+ w.n_ff = a.n_ff;+ w.limit = a.limit;+ const int groups = a.T * a.k; // at most (the blocks past the real count return at once)+ if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||+ !dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))+ return false;+ win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);+ launch_check("win_experts");+ return true;+}++} // namespace strata::kernels::exl3::legacy+#endifdiff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cunew file mode 100644index 000000000..7779231cf--- /dev/null+++ b/src/dsv41/vendor/sampler.cu@@ -0,0 +1,579 @@+// src/kernels/cuda/sampler.cu - P2.S2: the sampler chain, in llama.cpp's order.+//+// penalties -> top_k -> top_p -> min_p -> temperature -> pick+//+// THE ORDER IS THE WHOLE CONTENT OF THIS FILE. llama.cpp builds its chain by walking `params.samplers`, whose+// default is { PENALTIES, DRY, TOP_N_SIGMA, TOP_K, TYPICAL_P, TOP_P, MIN_P, XTC, TEMPERATURE } (`common/common.h`+// at 3cf03257) - ONE penalties stage, first, and TEMPERATURE AFTER THE TRUNCATION FILTERS. (Issue #53: this file+// used to apply the penalties a second time after the temperature, and min_p before top_p - both taken from the+// order of the `case` labels in `common/sampling.cpp`, which is not the order the chain runs.) Every order+// produces a valid token, so only a comparison at the distribution level can tell them apart; the parity test+// does that against an independently computed distribution.+//+// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,+// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /+// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).+#include "strata/kernels/sampler.hpp"++#include <cuda_runtime.h>++#include <cmath>+#include <cstdio>+#include <cstdlib>++namespace strata::kernels {+namespace {++// Philox 4x32-10, the counter-based generator the phase asks for. Counter-based matters because it makes the+// stream a function of (seed, position) rather than of how many draws came before - so a batch can be sampled+// in any order and a run is reproducible.+__device__ __forceinline__ uint32_t philox4x32_round(uint32_t& c0, uint32_t& c1, uint32_t& c2, uint32_t& c3,+ uint32_t k0, uint32_t k1) {+ const uint32_t hi0 = __umulhi(0x9E3779B9u, c0);+ const uint32_t hi1 = __umulhi(0xBB67AE85u, c2);+ const uint32_t lo0 = 0x9E3779B9u * c0;+ const uint32_t lo1 = 0xBB67AE85u * c2;+ const uint32_t n0 = hi1 ^ c1 ^ k0;+ const uint32_t n1 = lo1;+ const uint32_t n2 = hi0 ^ c3 ^ k1;+ const uint32_t n3 = lo0;+ c0 = n0; c1 = n1; c2 = n2; c3 = n3;+ return 0;+}++__device__ __forceinline__ float philox_uniform(uint64_t seed, uint64_t counter) {+ uint32_t c0 = (uint32_t) counter, c1 = (uint32_t) (counter >> 32);+ uint32_t c2 = (uint32_t) seed, c3 = (uint32_t) (seed >> 32);+ for (int i = 0; i < 10; ++i) {+ philox4x32_round(c0, c1, c2, c3, (uint32_t) i, 0u);+ }+ // 24 bits of mantissa, so the value is uniform in [0,1) with no rounding to 1.0+ return (float) (c0 >> 8) * (1.0f / 16777216.0f);+}++// `count_in_history` and the penalty application, transcribed from `llama_sampler_penalties_apply`.+// The repeat penalty MULTIPLIES for non-positive logits and DIVIDES for positive ones - dividing+// unconditionally is the natural reading of the source paper and it INVERTS the penalty on half the+// vocabulary. The presence penalty is `float(count > 0)`, a boolean, not the count.+__device__ __forceinline__ int history_count(const int* __restrict__ h, int n, int v) {+ int c = 0;+ for (int i = 0; i < n; ++i) if (h[i] == v) ++c;+ return c;+}++__device__ __forceinline__ float apply_penalties(float logit, int count, const SamplerParams& p) {+ if (count <= 0) return logit;+ if (logit <= 0.0f) logit *= p.penalty_repeat;+ else logit /= p.penalty_repeat;+ logit -= (float) count * p.penalty_freq + (count > 0 ? 1.0f : 0.0f) * p.penalty_present;+ return logit;+}++/// **THE GREEDY ARGMAX, ONE BLOCK PER TOKEN, COVERING THE VOCABULARY.**+///+/// **WHY THIS IS A SEPARATE KERNEL AND NOT A BRANCH.** `sampler_kernel` is launched as a grid over TOKENS+/// with 64 threads and a `if (t >= n_tokens) return;` at the top. The decode path has `n_tokens == 1`, so+/// that launch was `<<<1, 64>>>`, 63 threads exited on the first line, and ONE THREAD walked all 248,320+/// logits in a dependent loop on one SM of 48. Measured in isolation (`bench/micro/sampler_cost.cu`):+/// **3.11 ms per token**, 5.7% of an ~54 ms token, and the whole of round 309's `sample` phase - the two+/// synchronisations around it are 0.03 ms each.+///+/// The obvious repair is to parallelise the scan inside `sampler_kernel`, and it is WRONG: with one thread+/// per token, a block reduction over the vocabulary has nothing to reduce, and the threads that returned+/// early are not there for `__syncthreads` or `__shfl_down_sync`. The first attempt did exactly that and+/// produced the token `5120` thirty-two times. The grid has to be over tokens with the BLOCK over the+/// vocabulary, which is a different launch configuration and therefore a different kernel.+///+/// **THE TIE RULE IS UNCHANGED AND THAT IS THE WHOLE CORRECTNESS ARGUMENT.** The serial scan walked `v`+/// ascending with `if (s > bv)`, so the LOWEST index wins a tie. Each thread keeps that rule over its own+/// strided subset and the reduction resolves two candidates by taking the larger value and, on equality, the+/// SMALLER index - the same total order, so `sampler_parity` and C1 see no change.+__global__ void sampler_greedy_kernel(const float* __restrict__ logits, int n_vocab,+ const int* __restrict__ history, int history_len, const SamplerParams p,+ int pmin, int plen, int* __restrict__ out) {+ const int t = blockIdx.x;+ const float* l = logits + (size_t) t * n_vocab;+ (void) pmin;+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;+ int hlen = 0;+ if (hrow) {+ hlen = plen < history_len ? plen : history_len;+ if (hlen < 0) hlen = 0;+ hrow += history_len - hlen; // the window is the TAIL+ }++ // PENALTY MEMBERSHIP AS A BITMAP. The history touches at most `hlen` tokens of a quarter-million+ // vocabulary, but the naive `history_count` per candidate per argmax round costs O(k x n_vocab x hlen)+ // integer compares (~318 M per token at k=20, hlen=64 - measured 45 -> 31 tok/s on a real workload).+ // A shared bitmap gives an O(1) membership test, and only the (at most hlen) hits pay the count scan;+ // the counts - and therefore every sampled value - are exactly what the per-candidate scan produced.+ extern __shared__ unsigned int penal_bits[];+ const int bits_words = (int) ((n_vocab + 31) / 32);+ // The gate needs a NON-EMPTY WINDOW (`hlen > 0`): the launch sizes the shared bitmap only when penalties+ // are on, so a caller handing over a history buffer with `penalty_last_n == 0` must not touch it.+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;+ if (use_bits) {+ for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;+ __syncthreads();+ for (int i = threadIdx.x; i < hlen; i += blockDim.x)+ if (hrow[i] >= 0 && hrow[i] < n_vocab) // an id outside the vocabulary is never a candidate+ atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));+ __syncthreads();+ }+ auto hit_count = [&](int v) -> int {+ if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;+ return history_count(hrow, hlen, v);+ };++ // `n_vocab` is the "no candidate" index: it loses every comparison to a real one, so a thread with no+ // elements contributes nothing rather than contributing a bogus zero.+ float bv = __int_as_float(0xff800000); // -inf+ int best = n_vocab;+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {+ const float s = apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);+ if (s > bv) { bv = s; best = v; }+ }+ for (int off = 16; off > 0; off >>= 1) {+ const float ov = __shfl_down_sync(0xFFFFFFFFu, bv, off);+ const int oi = __shfl_down_sync(0xFFFFFFFFu, best, off);+ if (ov > bv || (ov == bv && oi < best)) { bv = ov; best = oi; }+ }+ __shared__ float sv[32];+ __shared__ int si[32];+ const int warp = (int) (threadIdx.x >> 5), lane = (int) (threadIdx.x & 31);+ if (lane == 0) { sv[warp] = bv; si[warp] = best; }+ __syncthreads();+ if (warp == 0) {+ const int nw = (int) ((blockDim.x + 31) >> 5);+ float wv = lane < nw ? sv[lane] : __int_as_float(0xff800000);+ int wi = lane < nw ? si[lane] : n_vocab;+ for (int off = 16; off > 0; off >>= 1) {+ const float ov = __shfl_down_sync(0xFFFFFFFFu, wv, off);+ const int oi = __shfl_down_sync(0xFFFFFFFFu, wi, off);+ if (ov > wv || (ov == wv && oi < wi)) { wv = ov; wi = oi; }+ }+ // A tie between two `-inf` candidates leaves `wi == n_vocab`, and the serial version answered 0.+ if (lane == 0) out[t] = (wi < n_vocab) ? wi : 0;+ }+}++/// **THE SAMPLED PATH, ONE BLOCK PER TOKEN.** The kernel below replaced a version that ran the whole chain+/// in ONE THREAD per token (`<<<ceil(T/64), 64>>>`, so a 4-token window fielded four threads): `top_k` alone+/// was `k` sequential scans of the vocabulary with an inner sweep over the already-taken list - 20 x 248,320+/// iterations of dependent work on one SM - and a verify window measured **1.6 s in the sampler**, which made+/// every temperature-bearing request ~30x slower than a greedy one. The selection is `k` argmax rounds, and+/// an argmax over the vocabulary parallelises exactly like `sampler_greedy_kernel` (block over the vocab), so+/// the rounds run back to back inside a block-per-token launch: the per-token cost falls to+/// `k x n_vocab / 1024` plus `k` block reductions.+///+/// THE SEMANTICS ARE THE SERIAL ONES, EXACTLY. Each round's argmax resolves ties to the LOWEST index (the+/// serial scan's strict `>` keeps the first maximum it meets), so the kept sequence - both its set and its+/// order - is unchanged; `top_p`'s cut reads that order in double arithmetic as before; temperature and the+/// Philox draw apply after the cut. `sampler_parity` pins all of it against the host reference.+// MODE 0 = the draw (sample_tokens, unchanged); 1 = speculative sampling's draft (sample_draft: the draw from the+// draft block's logits, on a Philox stream of its own, and its kept list in ql[t]); 2 = its verify (sample_verify:+// the draft draft[t] with probability min(1, p/q), else a draw from max(0, p - q)). The selection is shared.+constexpr uint64_t kSpecDraftKey = 0x5bd1e9955bd1e995ull, kSpecAcceptKey = 0x9e3779b97f4a7c15ull,+ kSpecResidKey = 0xc2b2ae3d27d4eb4full;+// one block of 1024 threads per row (the launches below): the bound keeps every mode at 64 registers a thread, so a+// block fits the 64K-register file - unbounded, sm_75 (Turing) compiled the draft mode to 68 and its launch failed+// ("too many resources requested for launch") wherever the NextN block sat on a Turing card+template <int MODE>+__global__ void __launch_bounds__(1024) sampler_kernel(const float* __restrict__ logits, int n_vocab, int n_tokens,+ const int* __restrict__ history, int history_len, const SamplerParams p,+ int* __restrict__ out, SpecList* __restrict__ ql, const int* __restrict__ draft) {+ const int t = blockIdx.x;+ if (t >= n_tokens) return;+ const float* l = logits + (size_t) t * n_vocab;++ // Temperature is needed by BOTH stages below, so it is computed here; the chain still APPLIES it after+ // the truncation filters - the survivors are chosen on the raw logits and only then scaled.+ const float inv_t = p.temperature > 0.0f ? 1.0f / p.temperature : 0.0f;++ // The penalty window is the last `penalty_last_n` entries of this row's history (disabled at this+ // launch: `sample_tokens` refuses a non-zero `penalty_last_n` without a history buffer).+ const int* hrow = history ? history + (size_t) t * history_len : nullptr;+ int hlen = 0;+ if (hrow) {+ hlen = p.penalty_last_n < history_len ? p.penalty_last_n : history_len;+ if (hlen < 0) hlen = 0;+ hrow += history_len - hlen; // the window is the TAIL+ }++ // the membership bitmap, as in `sampler_greedy_kernel` - see the cost note there. The gate needs an+ // NON-EMPTY WINDOW too: the launch sizes the bitmap only when penalties are on, so a caller that hands over+ // a stale history buffer with `penalty_last_n == 0` must not touch it.+ extern __shared__ unsigned int penal_bits[];+ const int bits_words = (int) ((n_vocab + 31) / 32);+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;+ if (use_bits) {+ for (int w = threadIdx.x; w < bits_words; w += blockDim.x) penal_bits[w] = 0u;+ __syncthreads();+ for (int i = threadIdx.x; i < hlen; i += blockDim.x)+ if (hrow[i] >= 0 && hrow[i] < n_vocab) // an id outside the vocabulary is never a candidate+ atomicOr(&penal_bits[hrow[i] >> 5], 1u << (hrow[i] & 31));+ __syncthreads();+ }+ auto hit_count = [&](int v) -> int {+ if (!use_bits || !(penal_bits[v >> 5] & (1u << (v & 31)))) return 0;+ return history_count(hrow, hlen, v);+ };++ // top_k in 1..64 is taken as given; 0 ("off") and anything wider mean the widest shortlist the kernel+ // keeps, 64. Every row writes out[t]: a verify window reads all of them.+ const int KMAX = 64;+ int k = (p.top_k > 0 && p.top_k < KMAX) ? p.top_k : KMAX;+ if (k > n_vocab) k = n_vocab;++ // ---- top_k: an exact RADIX SELECT, then a rank sort of the k survivors. The order is the serial one:+ // descending by (penalised) logit, ties to the lower index - i.e. descending by the composite key+ // (ordered(score) << 32) | ~index, which is distinct for every token. So: find the k-th largest ordered score+ // T byte by byte (4 histogram passes over the vocabulary), keep every token above T and the lowest-indexed+ // ones equal to T, then rank the k of them by the composite key. The k block-argmax rounds this replaces+ // re-scanned the taken list for every token in every round: 1.5 ms a sample at top_k 20 and 11 ms at 64 on an+ // RTX 4090 (the tail card's critical path in every sampled decode step).+ // THE COMMON CASE takes two passes: a coarse histogram of the top 11 bits finds the bin that holds the k-th+ // largest; every token in that bin or above (at most kCap) is copied to shared memory and ranked there by the+ // composite key. A flat distribution with more than kCap tokens there takes the full radix select below.+ __shared__ int sel_ids[KMAX];+ __shared__ float sel_logit[KMAX];+ __shared__ unsigned int sh_prefix, sh_remaining, sh_ncand, sh_tie_base;+ // ONE shared pool, its parts reused phase by phase: the penalty bitmap (dynamic shared memory, 31 KB at a Qwen+ // vocabulary) must still fit beside it under the 48 KB a launch gets without opting in.+ // [0, 2048) the coarse histogram, then (once its bin is known) the candidates' scores and ids;+ // in the fallback, the byte histogram and the k candidates+ // [2048, 3072) the block scan+ constexpr int kBins = 2048, kCap = 1024; // the coarse histogram: the ordered score's top 11 bits+ __shared__ unsigned int pool[kBins + 1024];+ unsigned int* coarse = pool;+ unsigned int* ccand_u = pool;+ int* ccand_i = (int*) pool + kCap;+ unsigned int* hist = pool;+ unsigned int* cand_u = pool + 256;+ int* cand_i = (int*) pool + 256 + KMAX;+ unsigned int* scan = pool + kBins;+ __shared__ int sh_bin;+ __shared__ unsigned int sh_ncoarse;+ // a score's ordered bits: larger float -> larger unsigned; NaN below everything (never kept before a number)+ auto ordered = [](float x) -> unsigned int {+ unsigned int b = __float_as_uint(x);+ if ((b & 0x7fffffffu) > 0x7f800000u) return 0u;+ if (b == 0x80000000u) b = 0u; // -0 ties with +0, as the float comparison it replaces+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);+ };+ auto score = [&](int v) -> float {+ return apply_penalties(p.logit_bias ? l[v] + p.logit_bias[v] : l[v], hit_count(v), p);+ };+ for (int b = threadIdx.x; b < kBins; b += blockDim.x) coarse[b] = 0u;+ if (threadIdx.x == 0) sh_ncoarse = 0u;+ __syncthreads();+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) atomicAdd(&coarse[ordered(score(v)) >> 21], 1u);+ __syncthreads();+ { // the bin holding the k-th largest: suffix counts over the bins (2 bins a thread, a block scan)+ const int nt = (int) blockDim.x, per = (kBins + nt - 1) / nt;+ const int b_hi = kBins - 1 - (int) threadIdx.x * per; // this thread's bins, from the top down+ unsigned int mine = 0u;+ for (int j = 0; j < per && b_hi - j >= 0; ++j) mine += coarse[b_hi - j];+ scan[threadIdx.x] = mine;+ __syncthreads();+ for (int off = 1; off < nt; off <<= 1) {+ const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;+ __syncthreads();+ scan[threadIdx.x] += add;+ __syncthreads();+ }+ unsigned int above = scan[threadIdx.x] - mine; // tokens in the bins above this thread's+ if (above < (unsigned int) k && above + mine >= (unsigned int) k) {+ for (int j = 0; j < per && b_hi - j >= 0; ++j) {+ above += coarse[b_hi - j];+ if (above >= (unsigned int) k) { sh_bin = b_hi - j; sh_tie_base = above; break; }+ }+ }+ __syncthreads();+ }+ const int bin = sh_bin;+ const bool coarse_ok = sh_tie_base <= (unsigned int) kCap; // the tokens in that bin and above fit+ if (coarse_ok) {+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {+ const unsigned int u = ordered(score(v));+ if ((int) (u >> 21) >= bin) {+ const unsigned int at = atomicAdd(&sh_ncoarse, 1u);+ if (at < (unsigned int) kCap) { ccand_u[at] = u; ccand_i[at] = v; }+ }+ }+ __syncthreads();+ const int nc = (int) min(sh_ncoarse, (unsigned int) kCap);+ for (int i = threadIdx.x; i < nc; i += blockDim.x) {+ const unsigned int ui = ccand_u[i];+ const int ii = ccand_i[i];+ int r = 0;+ for (int j = 0; j < nc && r < k; ++j) {+ const unsigned int uj = ccand_u[j];+ r += uj > ui || (uj == ui && ccand_i[j] < ii);+ }+ if (r < k) {+ sel_ids[r] = ii;+ sel_logit[r] = score(ii);+ }+ }+ __syncthreads();+ } else {+ if (threadIdx.x == 0) {+ sh_prefix = 0u;+ sh_remaining = (unsigned int) k;+ sh_ncand = 0u;+ }+ unsigned int mask = 0u;+ for (int pass = 0; pass < 4; ++pass) {+ const int shift = 24 - 8 * pass;+ for (int b = threadIdx.x; b < 256; b += blockDim.x) hist[b] = 0u;+ __syncthreads();+ const unsigned int prefix = sh_prefix;+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {+ const unsigned int u = ordered(score(v));+ if ((u & mask) == prefix) atomicAdd(&hist[(u >> shift) & 255u], 1u);+ }+ __syncthreads();+ if (threadIdx.x == 0) { // the digit that holds the remaining-th largest, from the top+ unsigned int cum = 0u, rem = sh_remaining;+ int d = 255;+ for (; d > 0; --d) {+ if (cum + hist[d] >= rem) break;+ cum += hist[d];+ }+ sh_remaining = rem - cum;+ sh_prefix = prefix | ((unsigned int) d << shift);+ }+ mask |= 255u << shift;+ __syncthreads();+ }+ const unsigned int T = sh_prefix; // the k-th largest ordered score+ const unsigned int need = sh_remaining; // how many tokens equal to T are kept (the lowest-indexed ones)+ // every token above T (fewer than k of them)+ for (int v = threadIdx.x; v < n_vocab; v += blockDim.x) {+ const unsigned int u = ordered(score(v));+ if (u > T) {+ const unsigned int at = atomicAdd(&sh_ncand, 1u);+ if (at < (unsigned int) KMAX) { cand_u[at] = u; cand_i[at] = v; }+ }+ }+ __syncthreads();+ if (threadIdx.x == 0) sh_tie_base = sh_ncand;+ // the ties in index order: each thread counts its own contiguous range, an exclusive scan orders the ranges+ {+ const int nt = (int) blockDim.x;+ const int chunk = (n_vocab + nt - 1) / nt;+ const int lo = (int) threadIdx.x * chunk, hi = min(n_vocab, lo + chunk);+ unsigned int mine = 0u;+ for (int v = lo; v < hi; ++v) mine += ordered(score(v)) == T;+ scan[threadIdx.x] = mine;+ __syncthreads();+ for (int off = 1; off < nt; off <<= 1) { // Hillis-Steele inclusive scan+ const unsigned int add = (int) threadIdx.x >= off ? scan[threadIdx.x - off] : 0u;+ __syncthreads();+ scan[threadIdx.x] += add;+ __syncthreads();+ }+ unsigned int rank = scan[threadIdx.x] - mine; // ties before this range+ for (int v = lo; v < hi && rank < need; ++v)+ if (ordered(score(v)) == T) {+ const unsigned int at = sh_tie_base + rank;+ if (at < (unsigned int) KMAX) { cand_u[at] = T; cand_i[at] = v; }+ ++rank;+ }+ }+ __syncthreads();+ // rank sort: candidate i's place is the number of candidates with a larger composite key+ if ((int) threadIdx.x < k) {+ const unsigned int ui = cand_u[threadIdx.x];+ const int ii = cand_i[threadIdx.x];+ int r = 0;+ for (int j = 0; j < k; ++j) {+ const unsigned int uj = cand_u[j];+ r += uj > ui || (uj == ui && cand_i[j] < ii);+ }+ sel_ids[r] = ii;+ sel_logit[r] = score(ii);+ }+ __syncthreads();+ }++ // ---- top_p over the top_k list (penalised logits, descending as the selection produced them), then min_p,+ // then temperature and one Philox draw - llama.cpp's order (issue #53). Every thread computes the same chain+ // redundantly over `sel_*` - the arithmetic is the serial kernel's, instruction for instruction - so they+ // agree on `pick` and thread 0 writes it.+ // (one thread: the chain is serial and in double precision, which consumer cards run at 1/64 rate - 1024+ // threads computing it redundantly cost more than the whole selection above)+ if (threadIdx.x != 0) return;+ int n_keep = k;+ float mx = sel_logit[0];+ for (int i = 1; i < k; ++i) mx = fmaxf(mx, sel_logit[i]);+ if (p.top_p < 1.0f) {+ double sum = 0.0;+ for (int i = 0; i < k; ++i) sum += exp((double) sel_logit[i] - (double) mx);+ double cum = 0.0;+ int cut = k;+ for (int i = 0; i < k; ++i) {+ cum += exp((double) sel_logit[i] - (double) mx) / sum;+ if (cum >= (double) p.top_p) { cut = i + 1; break; }+ }+ if (cut < p.min_keep) cut = p.min_keep < k ? p.min_keep : k;+ n_keep = cut;+ }+ // ---- min_p on top_p's survivors: the descending prefix whose probability is at least `min_p` of the top+ // token's. In logit space the threshold is `sel_logit[0] + logf(min_p)` - equivalent to `p >= min_p * p_max`+ // without the overflow an exp of raw logits risks. 0 disables, and the head itself always survives+ // (`expf(0) == 1 >= min_p` for min_p in 0..1), so the count never reaches zero.+ if (p.min_p > 0.0f) {+ const float thresh = sel_logit[0] + logf(p.min_p);+ for (int i = 0; i < n_keep; ++i)+ if (sel_logit[i] < thresh) { n_keep = i; break; }+ }+ // temperature only: the penalties were applied once, before the selection (issue #53: they were applied a+ // second time here, after the temperature scaling - llama.cpp's chain has one penalties stage)+ auto scaled = [&](int i) { return sel_logit[i] * inv_t; };+ float smx = scaled(0);+ for (int i = 1; i < n_keep; ++i) smx = fmaxf(smx, scaled(i));+ double sum = 0.0;+ for (int i = 0; i < n_keep; ++i) sum += exp((double) scaled(i) - (double) smx);+ if constexpr (MODE == 0) {+ const float u = philox_uniform(p.seed, p.counter + (uint64_t) t);+ double cum = 0.0;+ int pick = sel_ids[n_keep - 1];+ for (int i = 0; i < n_keep; ++i) {+ cum += exp((double) scaled(i) - (double) smx) / sum;+ if ((double) u < cum) { pick = sel_ids[i]; break; }+ }+ if (threadIdx.x == 0) out[t] = pick;+ } else {+ double w[KMAX]; // the kept list's probabilities, as the plain draw computes them+ for (int i = 0; i < n_keep; ++i) w[i] = exp((double) scaled(i) - (double) smx) / sum;+ // the plain draw over w with a given uniform (MODE 0's loop)+ auto draw = [&](float u) -> int {+ double cum = 0.0;+ int pick = sel_ids[n_keep - 1];+ for (int i = 0; i < n_keep; ++i) {+ cum += w[i];+ if ((double) u < cum) { pick = sel_ids[i]; break; }+ }+ return pick;+ };+ if constexpr (MODE == 1) {+ SpecList* q = ql + t;+ q->n = n_keep;+ for (int i = 0; i < n_keep; ++i) { q->ids[i] = sel_ids[i]; q->p[i] = w[i]; }+ out[t] = draw(philox_uniform(p.seed ^ kSpecDraftKey, p.counter + (uint64_t) t));+ } else {+ const SpecList* q = ql + t;+ const int d = draft[t];+ const int qn = q->n < KMAX ? q->n : KMAX;+ auto qof = [&](int id) -> double {+ for (int j = 0; j < qn; ++j)+ if (q->ids[j] == id) return q->p[j];+ return 0.0;+ };+ double pd = 0.0;+ for (int i = 0; i < n_keep; ++i)+ if (sel_ids[i] == d) { pd = w[i]; break; }+ const double qd = qof(d);+ int pick = -1;+ if (qd <= 0.0) {+ pick = draw(philox_uniform(p.seed, p.counter + (uint64_t) t)); // not a draft from q: the plain draw+ } else if ((double) philox_uniform(p.seed ^ kSpecAcceptKey, p.counter + (uint64_t) t) * qd < pd) {+ pick = d; // kept: probability min(1, p(d)/q(d))+ } else {+ // rejected: a draw from max(0, p - q), renormalised - over p's kept list (p is 0 outside it)+ double r[KMAX], z = 0.0;+ for (int i = 0; i < n_keep; ++i) {+ const double x = w[i] - qof(sel_ids[i]);+ r[i] = x > 0.0 ? x : 0.0;+ z += r[i];+ }+ const float u = philox_uniform(p.seed ^ kSpecResidKey, p.counter + (uint64_t) t);+ if (z > 0.0) {+ double cum = 0.0;+ for (int i = 0; i < n_keep; ++i) {+ if (r[i] <= 0.0) continue;+ pick = sel_ids[i]; // the last one with residual mass, if rounding leaves cum below u+ cum += r[i] / z;+ if ((double) u < cum) break;+ }+ } else {+ pick = draw(u); // p == q to the last bit: a rejection had probability 0 - never in practice+ }+ }+ out[t] = pick;+ }+ }+}++} // namespace++void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len,+ const SamplerParams& p, int* out, void* stream) {+ if (n_tokens <= 0 || n_vocab <= 0) return;+ if (p.penalty_last_n > 0 && (history == nullptr || history_len <= 0)) {+ std::fprintf(stderr, "sample_tokens: penalty_last_n %d needs a history (got %p, len %d)\n",+ p.penalty_last_n, (const void*) history, history_len);+ std::exit(1);+ }+ const unsigned shmem = (history != nullptr && history_len > 0 && p.penalty_last_n > 0)+ ? (unsigned) ((n_vocab + 31) / 32) * sizeof(unsigned) // the penalty bitmap+ : 0;+ if (p.greedy || p.temperature <= 0.0f) {+ // One block per token, 1,024 threads over the vocabulary. See `sampler_greedy_kernel`.+ const int gthreads = 1024;+ sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(+ logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);+ } else {+ // The same block-per-token shape: the selection's k argmax rounds reduce inside the block. See+ // `sampler_kernel`'s header for what the old one-thread-per-token launch cost.+ sampler_kernel<0><<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(+ logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);+ }+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "sample_tokens launch: %s\n", cudaGetErrorString(e));+ std::exit(1);+ }+ if (stream == nullptr) cudaDeviceSynchronize();+}++namespace {+void spec_check(const char* who, int n_tokens, int n_vocab, const SamplerParams& p) {+ if (p.greedy || p.temperature <= 0.0f || p.penalty_last_n > 0 || n_vocab <= 0) {+ std::fprintf(stderr, "%s: a sampled request without penalties only (greedy %d, temperature %g, penalty_last_n "+ "%d, n_vocab %d)\n", who, (int) p.greedy, p.temperature, p.penalty_last_n, n_vocab);+ std::exit(1);+ }+ (void) n_tokens;+}+void spec_launched(const char* who, void* stream) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::fprintf(stderr, "%s launch: %s\n", who, cudaGetErrorString(e));+ std::exit(1);+ }+ if (stream == nullptr) cudaDeviceSynchronize();+}+} // namespace++void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, SpecList* q, int* out,+ void* stream) {+ if (n_tokens <= 0) return;+ spec_check("sample_draft", n_tokens, n_vocab, p);+ sampler_kernel<1><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,+ p, out, q, nullptr);+ spec_launched("sample_draft", stream);+}++void sample_verify(const float* logits, int n_tokens, int n_vocab, const SamplerParams& p, const SpecList* q,+ const int* draft, int* out, void* stream) {+ if (n_tokens <= 0) return;+ spec_check("sample_verify", n_tokens, n_vocab, p);+ sampler_kernel<2><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,+ p, out, const_cast<SpecList*>(q), draft);+ spec_launched("sample_verify", stream);+}++} // namespace strata::kernelsdiff --git a/tests/dsv41/ds4_cpu_bench.cpp b/tests/dsv41/ds4_cpu_bench.cppnew file mode 100644index 000000000..f5666216e--- /dev/null+++ b/tests/dsv41/ds4_cpu_bench.cpp@@ -0,0 +1,105 @@+// tests/ds4_cpu_bench.cpp - how fast does this CPU run DeepSeek-V4.1-Flash's EXL3 experts (5120 -> 2304 -> 5120, K 3)?+//+// ds4_cpu_bench [--threads N] [--node 0|1|-1] [--mode band|rows] [--experts N] [--per-call N] [--iters N]+//+// Random trellis words in memory bound to --node (-1: wherever the first touch puts them), --threads workers pinned to+// that node's physical cores (CPU i is on node i % 2 here, CPUs 0..43 the physical cores), each call computing+// --per-call experts' gate and up (all columns) from one activation: the time of a matrix pass and its GB/s.+#include "strata/kernels/cpu/exl3_cpu.hpp"+#include "../src/ds4/ds4_pool.hpp"++#include <sys/mman.h>+#include <sys/syscall.h>+#include <unistd.h>++#include <chrono>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++namespace kc = strata::kernels::cpu;++static void* node_alloc(size_t bytes, int node) {+ void* p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);+ if (p == MAP_FAILED) return nullptr;+ madvise(p, bytes, MADV_HUGEPAGE);+ if (node >= 0) {+ unsigned long mask = 1ul << node;+ if (syscall(SYS_mbind, p, bytes, 2 /*MPOL_BIND*/, &mask, 64, 0) != 0) std::perror("mbind");+ }+ return p;+}++int main(int argc, char** argv) {+ int threads = 22, node = 0, n_exp = 64, per_call = 2, iters = 40;+ std::string mode = "rows";+ for (int i = 1; i + 1 < argc; i += 2) {+ const std::string a = argv[i];+ if (a == "--threads") threads = std::atoi(argv[i + 1]);+ else if (a == "--node") node = std::atoi(argv[i + 1]);+ else if (a == "--mode") mode = argv[i + 1];+ else if (a == "--experts") n_exp = std::atoi(argv[i + 1]);+ else if (a == "--per-call") per_call = std::atoi(argv[i + 1]);+ else if (a == "--iters") iters = std::atoi(argv[i + 1]);+ }+ const int E = 5120, FF = 2304, K = 3;+ const size_t tb = (size_t) E / 16 * FF / 16 * 32 * K; // one matrix's trellis+ std::vector<int> cpus;+ for (int c = 0; c < 44; ++c)+ if (node < 0 || c % 2 == node) cpus.push_back(c);+ strata::ds4::Pool pool(threads - 1, std::vector<int>(cpus.begin() + 1, cpus.end()));+ {+ cpu_set_t set;+ CPU_ZERO(&set);+ CPU_SET(cpus[0], &set);+ sched_setaffinity(0, sizeof set, &set);+ }+ uint8_t* mem = (uint8_t*) node_alloc(tb * n_exp, node);+ std::vector<uint16_t> suh(E), svh(FF);+ for (auto& v : suh) v = 0x3c00; // 1.0+ for (auto& v : svh) v = 0x3c00;+ pool.run(n_exp, [&](int e) {+ std::mt19937 r(e);+ uint32_t* w = (uint32_t*) (mem + tb * e);+ for (size_t i = 0; i < tb / 4; ++i) w[i] = r();+ });+ std::vector<float> x(E, 0.5f), xh(E), acc((size_t) 64 * FF), y((size_t) per_call * FF);+ const float xs = 0.0f;+ kc::Exl3Part p0;+ p0.suh = suh.data();+ p0.svh = svh.data();+ p0.k = E;+ p0.n = FF;+ p0.K = K;+ kc::exl3_cpu_prepare(p0, x.data(), xh.data());+ const int GQ = E / 256; // rows mode: pieces of 256 inputs over every column+ std::vector<float> accp((size_t) per_call * GQ * FF);+ auto t0 = std::chrono::steady_clock::now();+ long long bytes = 0;+ for (int it = 0; it < iters; ++it) {+ const int base = (it * per_call) % (n_exp - per_call + 1);+ if (mode == "band") {+ pool.run(per_call * (FF / 128), [&](int j) {+ kc::Exl3Part p = p0;+ p.trellis = (const uint32_t*) (mem + tb * (base + j / (FF / 128)));+ kc::exl3_cpu_band(p, xh.data(), xs, (j % (FF / 128)) * 128,+ y.data() + (size_t) (j / (FF / 128)) * FF + (j % (FF / 128)) * 128);+ });+ } else {+ pool.run(per_call * GQ, [&](int j) {+ kc::Exl3Part p = p0;+ p.trellis = (const uint32_t*) (mem + tb * (base + j / GQ));+ const int q = j % GQ;+ kc::exl3_cpu_rows_acc(p, xh.data(), q * 16, (q + 1) * 16, accp.data() + (size_t) j * FF);+ });+ }+ bytes += (long long) tb * per_call;+ }+ const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();+ std::printf("%s node %d threads %d per-call %d: %.3f ms per matrix, %.1f GB/s\n", mode.c_str(), node, threads,+ per_call, ms / (iters * per_call), bytes / 1e6 / ms);+ return 0;+}diff --git a/tests/dsv41/ds4_gguf_test.cu b/tests/dsv41/ds4_gguf_test.cunew file mode 100644index 000000000..a24059bcb--- /dev/null+++ b/tests/dsv41/ds4_gguf_test.cu@@ -0,0 +1,1616 @@+// tests/ds4_gguf_test.cu - maya-ds4's GGUF kernels (src/ds4/ds4_gguf.cu, ds4_gguf_cpu.cpp) against a plain+// double-precision reference of random BF16 matrices and MXFP4 experts: the GEMVs (BF16 / MXFP4, 1..16 activations,+// aligned or not), the token experts (gate / up / swiglu / down / combine, experts off the device, the CPU part, the+// shared expert), a window's grouped experts (bit for bit the token path's), the prompt path's expert sets and the CPU+// lane's rows; and the KV caches' packed rows (ds4_kernels.cu) against the BF16 rows bit for bit. Needs a GPU, no+// model. ds4_gguf_test [n_embd n_ff] | --kv (the caches alone)+#include "../src/ds4/ds4_gguf.cuh"+#include "../src/ds4/ds4_gguf_cpu.hpp"+#include "../src/ds4/ds4_iq.cuh"+#include "../src/ds4/ds4_kernels.cuh"+#include "../src/ds4/ds4_q6k.cuh"++#include "ggml.h" // (the low-bit types' reference values: ggml's own dequantize_row_*)++#include <cuda_fp16.h>++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <functional>+#include <thread>+#include <cstdlib>+#include <cstring>+#include <random>+#include <vector>++namespace gg = strata::ds4::gguf;+namespace x3 = strata::kernels::exl3;++namespace {++int g_fail = 0;+std::mt19937 rng(1234);++#define CU(x) \+ do { \+ cudaError_t e_ = (x); \+ if (e_ != cudaSuccess) { \+ std::printf("CUDA %s: %s (line %d)\n", #x, cudaGetErrorString(e_), __LINE__); \+ std::exit(1); \+ } \+ } while (0)++float frand(float a = 1.0f) { return std::uniform_real_distribution<float>(-a, a)(rng); }++float bf16f(uint16_t b) {+ uint32_t u = (uint32_t) b << 16;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}+uint16_t f2bf(float f) { // truncating is fine: the reference reads back the same bits+ uint32_t u;+ std::memcpy(&u, &f, 4);+ return (uint16_t) (u >> 16);+}+const float kVal[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};+double e8m0_half(int e) { return std::ldexp(1.0, e - 128); }++// random MXFP4 rows [n][k]: scales around 2^-6, every code+std::vector<uint8_t> mx_rand(int n, int k) {+ std::vector<uint8_t> w(gg::mx_bytes(n, k));+ for (size_t b = 0; b < w.size() / 17; ++b) {+ w[b * 17] = (uint8_t) (120 + rng() % 6);+ for (int j = 1; j < 17; ++j) w[b * 17 + j] = (uint8_t) (rng() & 0xff);+ }+ return w;+}+double mx_val(const uint8_t* row, int j) { // value j of a row+ const uint8_t* p = row + (size_t) (j / 32) * 17;+ const int jj = j % 32;+ const int code = jj < 16 ? (p[1 + jj] & 15) : (p[1 + jj - 16] >> 4);+ return kVal[code] * e8m0_half(p[0]);+}+// random Q8_0 rows [n][k], and a value of one+std::vector<uint8_t> q8_rand(int n, int k) {+ std::vector<uint8_t> w(gg::q8_row_bytes(k) * n);+ for (size_t b = 0; b < w.size() / 34; ++b) {+ const __half d = __float2half(0.001f + 0.002f * std::fabs(frand()));+ std::memcpy(&w[b * 34], &d, 2);+ for (int j = 2; j < 34; ++j) w[b * 34 + j] = (uint8_t) (rng() & 0xff);+ }+ return w;+}+double q8_val(const uint8_t* row, int j) {+ const uint8_t* p = row + (size_t) (j / 32) * 34;+ __half d;+ std::memcpy(&d, p, 2);+ return (double) __half2float(d) * (int8_t) p[2 + j % 32];+}+double mx_dot(const uint8_t* w, int k, int row, const float* x) {+ const uint8_t* r = w + (size_t) row * gg::mx_row_bytes(k);+ double s = 0;+ for (int j = 0; j < k; ++j) s += mx_val(r, j) * x[j];+ return s;+}+// x rounded as the kernels' q8_quant: per 32 values sc = max|x| / 127 (F32), q = round-half-even(x / sc)+std::vector<float> q8_round(const float* x, int k) {+ std::vector<float> v(k);+ for (int b = 0; b < k / 32; ++b) {+ float am = 0.0f;+ for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));+ const float sc = am / 127.0f;+ for (int j = 0; j < 32; ++j) {+ const float q = am > 0.0f ? std::nearbyint(x[b * 32 + j] / sc) : 0.0f;+ v[b * 32 + j] = (float) ((double) q * sc);+ }+ }+ return v;+}+double swiglu(double g, double u, double lim) {+ if (lim > 0) {+ g = std::min(g, lim);+ u = std::min(std::max(u, -lim), lim);+ }+ return g / (1.0 + std::exp(-g)) * u;+}++void expect(const char* what, const float* got, const double* want, size_t n, double rtol) {+ double worst = 0, scale = 1e-30;+ for (size_t i = 0; i < n; ++i) scale = std::max(scale, std::fabs(want[i]));+ size_t at = 0;+ for (size_t i = 0; i < n; ++i) {+ const double e = std::fabs(got[i] - want[i]) / scale;+ if (!(e <= worst)) {+ worst = e;+ at = i;+ }+ }+ const bool ok = worst <= rtol;+ if (!ok) ++g_fail;+ std::printf("%-44s %s max rel err %.2e (at %zu: %.6g vs %.6g)\n", what, ok ? "ok " : "FAIL", worst, at,+ (double) got[at], want[at]);+}++template <class T>+T* dev(const std::vector<T>& h) {+ T* p = nullptr;+ CU(cudaMalloc(&p, std::max<size_t>(1, h.size()) * sizeof(T)));+ if (!h.empty()) CU(cudaMemcpy(p, h.data(), h.size() * sizeof(T), cudaMemcpyHostToDevice));+ return p;+}+template <class T>+std::vector<T> host(const T* p, size_t n) {+ std::vector<T> h(n);+ CU(cudaMemcpy(h.data(), p, n * sizeof(T), cudaMemcpyDeviceToHost));+ return h;+}++void test_mv(cudaStream_t s) {+ for (int fmt : {gg::kFmtBF16, gg::kFmtMXFP4, gg::kFmtQ8_0})+ for (int nc : {1, 3, 16})+ for (int off : {0, 1}) { // off 1: activations not 16-byte aligned (the scalar kernels)+ const int k = 1280, n = 520;+ std::vector<uint16_t> wb;+ std::vector<uint8_t> wm;+ if (fmt == gg::kFmtBF16) {+ wb.resize((size_t) n * k);+ for (auto& v : wb) v = f2bf(frand(0.05f));+ } else {+ wm = fmt == gg::kFmtMXFP4 ? mx_rand(n, k) : q8_rand(n, k);+ }+ const size_t xcs = (size_t) k + 4 + off;+ std::vector<float> x(xcs * nc + 8), bias(n);+ for (auto& v : x) v = frand();+ for (auto& v : bias) v = frand(0.1f);+ float* dx = dev(x);+ float* db = dev(bias);+ std::vector<float> y0((size_t) n * nc, 0.0f);+ float* dy = dev(y0);+ void* dw = fmt == gg::kFmtBF16 ? (void*) dev(wb) : (void*) dev(wm);+ x3::Mat m;+ m.fmt = fmt;+ m.raw = dw;+ m.k = k;+ m.n = n;+ x3::MvJob j;+ j.m = &m;+ j.xf = dx + off;+ j.y = dy;+ j.bias = db;+ j.alpha = 0.5f;+ j.ncol = nc;+ j.xf_cs = xcs;+ j.y_cs = n;+ if (!gg::mv(&j, 1, s)) ++g_fail;+ CU(cudaStreamSynchronize(s));+ std::vector<float> y = host(dy, (size_t) n * nc);+ std::vector<double> want((size_t) n * nc);+ for (int c = 0; c < nc; ++c)+ for (int r = 0; r < n; ++r) {+ const float* xc = x.data() + off + c * xcs;+ double d = 0;+ if (fmt == gg::kFmtBF16)+ for (int i = 0; i < k; ++i) d += (double) bf16f(wb[(size_t) r * k + i]) * xc[i];+ else if (fmt == gg::kFmtMXFP4)+ d = mx_dot(wm.data(), k, r, xc);+ else {+ const std::vector<float> xr = q8_round(xc, k);+ for (int i = 0; i < k; ++i) d += q8_val(wm.data() + (size_t) r * gg::q8_row_bytes(k), i) * xr[i];+ }+ want[(size_t) c * n + r] = 0.5 * d + bias[r];+ }+ char what[96];+ std::snprintf(what, sizeof(what), "mv %s, %d activations%s", fmt == gg::kFmtBF16 ? "BF16" : fmt == gg::kFmtMXFP4 ? "MXFP4" : "Q8_0", nc,+ off ? ", unaligned" : "");+ expect(what, y.data(), want.data(), want.size(), 2e-5);+ // the first activation alone gives the same bits as in the batch+ if (nc > 1) {+ j.ncol = 1;+ std::vector<float> y1(n, 0.0f);+ float* dy1 = dev(y1);+ j.y = dy1;+ gg::mv(&j, 1, s);+ CU(cudaStreamSynchronize(s));+ y1 = host(dy1, n);+ const bool same = std::memcmp(y1.data(), y.data(), (size_t) n * 4) == 0;+ if (!same) ++g_fail;+ std::printf("%-44s %s\n", " ... one activation alone, bit for bit", same ? "ok" : "FAIL");+ cudaFree(dy1);+ }+ cudaFree(dx), cudaFree(db), cudaFree(dy), cudaFree(dw);+ }+ // rows(): BF16 -> F16 of a view+ {+ const int k = 256, n = 384;+ std::vector<uint16_t> wb((size_t) n * k);+ for (auto& v : wb) v = f2bf(frand(2.0f));+ uint16_t* dw = dev(wb);+ x3::Mat m;+ m.fmt = gg::kFmtBF16;+ m.raw = dw;+ m.k = k;+ m.n = n;+ const x3::Mat v = gg::view(m, 128, 128);+ std::vector<uint16_t> z((size_t) 128 * k);+ uint16_t* dd = dev(z);+ gg::rows(v, dd, false, s);+ CU(cudaStreamSynchronize(s));+ z = host(dd, z.size());+ std::vector<float> got(z.size());+ std::vector<double> want(z.size());+ for (size_t i = 0; i < z.size(); ++i) {+ got[i] = __half2float(*(const __half*) &z[i]);+ want[i] = bf16f(wb[(size_t) 128 * k + i]);+ }+ expect("rows BF16 -> F16 (a view)", got.data(), want.data(), want.size(), 1e-3);+ cudaFree(dw), cudaFree(dd);+ }+ { // rows(): Q8_0 -> F16 of a view+ const int k = 256, n = 384;+ std::vector<uint8_t> wq = q8_rand(n, k);+ uint8_t* dw = dev(wq);+ x3::Mat m;+ m.fmt = gg::kFmtQ8_0;+ m.raw = dw;+ m.k = k;+ m.n = n;+ const x3::Mat v = gg::view(m, 128, 128);+ std::vector<uint16_t> z((size_t) 128 * k);+ uint16_t* dd = dev(z);+ gg::rows(v, dd, false, s);+ CU(cudaStreamSynchronize(s));+ z = host(dd, z.size());+ std::vector<float> got(z.size());+ std::vector<double> want(z.size());+ for (size_t i = 0; i < z.size(); ++i) {+ got[i] = __half2float(*(const __half*) &z[i]);+ want[i] = q8_val(wq.data() + (128 + i / k) * gg::q8_row_bytes(k), (int) (i % k));+ }+ expect("rows Q8_0 -> F16 (a view)", got.data(), want.data(), want.size(), 1e-3);+ cudaFree(dw), cudaFree(dd);+ }+}++struct Experts {+ int D, F, n_exp;+ x3::ExpertLayout lay;+ size_t blob;+ std::vector<uint8_t> host; // n_exp blobs+ uint8_t* dev = nullptr;+};+Experts make_experts(int D, int F, int n_exp) {+ Experts E{D, F, n_exp, {}, 0, {}, nullptr};+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ E.lay.trellis[r] = off;+ off += (gg::mx_bytes(r < 2 ? F : D, r < 2 ? D : F) + 255) / 256 * 256;+ }+ E.lay.fmt = gg::kFmtMXFP4;+ E.blob = off;+ E.host.assign(E.blob * n_exp, 0);+ for (int e = 0; e < n_exp; ++e)+ for (int r = 0; r < 3; ++r) {+ const std::vector<uint8_t> m = mx_rand(r < 2 ? F : D, r < 2 ? D : F);+ std::memcpy(E.host.data() + e * E.blob + E.lay.trellis[r], m.data(), m.size());+ }+ E.dev = dev(E.host);+ return E;+}+const uint8_t* emat(const Experts& E, int e, int r) { return E.host.data() + e * E.blob + E.lay.trellis[r]; }+// reference of one token's expert e: h then the down output+// (x and h rounded to int8 per 32 first, as the dp4a kernels round them)+void ref_expert(const Experts& E, int e, const float* x, double lim, std::vector<float>& h, std::vector<double>& y) {+ const std::vector<float> xr = q8_round(x, E.D);+ h.resize(E.F);+ for (int f = 0; f < E.F; ++f)+ h[f] = (float) swiglu(mx_dot(emat(E, e, 0), E.D, f, xr.data()), mx_dot(emat(E, e, 1), E.D, f, xr.data()), lim);+ const std::vector<float> hr = q8_round(h.data(), E.F);+ y.resize(E.D);+ for (int d = 0; d < E.D; ++d) y[d] = mx_dot(emat(E, e, 2), E.F, d, hr.data());+}++void test_experts(int D, int F, cudaStream_t s) {+ const int n_exp = 12, k = 6, T = 5;+ const float lim = 10.0f;+ Experts E = make_experts(D, F, n_exp);+ // T tokens' plans [T][8]: picks, weights, some picks off the device (ptr 0), token 1 with a CPU part+ std::vector<int> pick((size_t) T * 8, 0);+ std::vector<float> pw((size_t) T * 8, 0.0f);+ std::vector<unsigned long long> pp((size_t) T * 8, 0ull);+ for (int t = 0; t < T; ++t) {+ std::vector<int> all(n_exp);+ for (int i = 0; i < n_exp; ++i) all[i] = i;+ std::shuffle(all.begin(), all.end(), rng);+ for (int i = 0; i < k; ++i) {+ pick[t * 8 + i] = all[i];+ pw[t * 8 + i] = 0.2f + 0.1f * frand();+ const bool on = !(t == 2 && i == 3) && !(t == 4 && i == 0);+ pp[t * 8 + i] = on ? (unsigned long long) (E.dev + (size_t) all[i] * E.blob) : 0ull;+ }+ }+ std::vector<float> x((size_t) T * D), sh((size_t) T * D), cpu((size_t) T * D);+ for (auto& v : x) v = frand();+ for (auto& v : sh) v = frand(0.1f);+ for (auto& v : cpu) v = frand(0.1f);+ std::vector<int> cflag(T, 0);+ cflag[1] = 1;+ float *dx = dev(x), *dsh = dev(sh), *dcpu = dev(cpu), *dpw = dev(pw);+ unsigned long long* dpp = dev(pp);+ int* dflag = dev(cflag);+ // reference outputs+ std::vector<double> want((size_t) T * D, 0.0);+ std::vector<float> hh;+ std::vector<double> yy;+ for (int t = 0; t < T; ++t) {+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] = 0;+ for (int i = 0; i < k; ++i) {+ if (pp[t * 8 + i] == 0ull) continue;+ ref_expert(E, pick[t * 8 + i], x.data() + (size_t) t * D, lim, hh, yy);+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] += pw[t * 8 + i] * yy[d];+ }+ for (int d = 0; d < D; ++d)+ want[(size_t) t * D + d] += (cflag[t] ? cpu[(size_t) t * D + d] : 0.0f) + sh[(size_t) t * D + d];+ }+ // the token path, token by token+ std::vector<float> tok((size_t) T * D);+ float* dh = dev(std::vector<float>((size_t) 8 * F));+ float* dout = dev(std::vector<float>(D));+ for (int t = 0; t < T; ++t) {+ x3::MoeArgs a;+ a.plan_ptr = dpp + t * 8;+ a.plan_w = dpw + t * 8;+ a.cpu_part = dcpu + (size_t) t * D;+ a.cpu_flag = dflag + t;+ a.lay = E.lay;+ a.k = k;+ a.n_embd = D;+ a.n_ff = F;+ a.limit = lim;+ gg::moe_gate_up(a, dx + (size_t) t * D, dh, s);+ gg::moe_down(a, dh, dsh + (size_t) t * D, dout, s);+ CU(cudaStreamSynchronize(s));+ CU(cudaMemcpy(tok.data() + (size_t) t * D, dout, (size_t) D * 4, cudaMemcpyDeviceToHost));+ }+ char what[96];+ std::snprintf(what, sizeof(what), "token experts (D %d, F %d)", D, F);+ expect(what, tok.data(), want.data(), want.size(), 1e-3); // (an h value can round to its other int8)+ // the window: groups by expert (the device ones), entries e = t * 8 + i+ std::vector<unsigned long long> gptr;+ std::vector<int> gstart, gent;+ for (int x0 = 0; x0 < n_exp; ++x0) {+ std::vector<int> ents;+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < k; ++i)+ if (pick[t * 8 + i] == x0 && pp[t * 8 + i] != 0ull) ents.push_back(t * 8 + i);+ if (ents.empty()) continue;+ gptr.push_back((unsigned long long) (E.dev + (size_t) x0 * E.blob));+ gstart.push_back((int) gent.size());+ for (int e : ents) gent.push_back(e);+ }+ gstart.push_back((int) gent.size());+ std::vector<int> ng(1, (int) gptr.size());+ unsigned long long* dgp = dev(gptr);+ int *dgs = dev(gstart), *dge = dev(gent), *dng = dev(ng);+ float* dwh = dev(std::vector<float>((size_t) T * 8 * F));+ float* dpart = dev(std::vector<float>((size_t) T * 8 * D));+ float* dwo = dev(std::vector<float>((size_t) T * D));+ x3::WinArgs w;+ w.grp_ptr = dgp;+ w.grp_start = dgs;+ w.grp_ent = dge;+ w.n_groups = dng;+ w.plan_ptr = dpp;+ w.plan_w = dpw;+ w.cpu_part = dcpu;+ w.cpu_stride = D;+ w.cpu_flag = dflag;+ w.x = dx;+ w.x_stride = D;+ w.h = dwh;+ w.part = dpart;+ w.sh_out = dsh;+ w.out = dwo;+ w.out_stride = D;+ w.lay = E.lay;+ w.T = T;+ w.k = k;+ w.n_embd = D;+ w.n_ff = F;+ w.limit = lim;+ if (!gg::win_experts(w, s)) ++g_fail;+ CU(cudaStreamSynchronize(s));+ std::vector<float> win = host(dwo, (size_t) T * D);+ std::snprintf(what, sizeof(what), "window experts (T %d)", T);+ expect(what, win.data(), want.data(), want.size(), 1e-3);+ const bool same = std::memcmp(win.data(), tok.data(), win.size() * 4) == 0;+ if (!same) ++g_fail;+ std::printf("%-44s %s\n", " ... bit for bit the token path's", same ? "ok" : "FAIL");++ // the prompt path's set: rows sorted by expert (bounds), each row a token index (row_idx) - experts of a few rows (the+ // light kernels), then of more (the tiles)+ for (int R : {100, 200, 400}) {+ std::vector<int> owner(R), idx(R);+ for (int r = 0; r < R; ++r) owner[r] = (int) (rng() % n_exp);+ std::sort(owner.begin(), owner.end());+ std::vector<int> bounds(n_exp + 1, 0);+ for (int e = 0, r = 0; e <= n_exp; ++e) {+ while (r < R && owner[r] < e) ++r;+ bounds[e] = r;+ }+ int max_rows = 0;+ for (int e = 0; e < n_exp; ++e) max_rows = std::max(max_rows, bounds[e + 1] - bounds[e]);+ const int NT = 40;+ std::vector<float> xs((size_t) NT * D);+ for (auto& v : xs) v = frand();+ for (int r = 0; r < R; ++r) idx[r] = (int) (rng() % NT);+ float* dxs = dev(xs);+ int *dbd = dev(bounds), *didx = dev(idx);+ uint16_t* dh16 = dev(std::vector<uint16_t>((size_t) R * F));+ float* dy = dev(std::vector<float>((size_t) R * D));+ x3::SetArgs sa;+ sa.wbase = E.dev;+ sa.stride = E.blob;+ sa.lay = E.lay;+ sa.n_exp = n_exp;+ sa.bounds = dbd;+ sa.row0 = 0;+ sa.max_rows = max_rows;+ sa.n_embd = D;+ sa.n_ff = F;+ sa.limit = lim;+ gg::set_gate_up(sa, dxs, D, didx, dh16, F, s);+ gg::set_down(sa, dh16, F, dy, D, s);+ CU(cudaStreamSynchronize(s));+ std::vector<uint16_t> h16 = host(dh16, (size_t) R * F);+ std::vector<float> y = host(dy, (size_t) R * D);+ std::vector<float> hgot((size_t) R * F);+ std::vector<double> hwant((size_t) R * F), ywant((size_t) R * D);+ // (an expert of at most 16 rows takes the light kernels: its inputs rounded to int8 per 32, as the token path's)+ int n_light = 0;+ for (int e = 0; e < n_exp; ++e) n_light += bounds[e + 1] - bounds[e] <= 16 ? bounds[e + 1] - bounds[e] : 0;+ for (int r = 0; r < R; ++r) {+ const bool light = bounds[owner[r] + 1] - bounds[owner[r]] <= 16;+ const std::vector<float> xq = light ? q8_round(xs.data() + (size_t) idx[r] * D, D) : std::vector<float>();+ const float* xr = light ? xq.data() : xs.data() + (size_t) idx[r] * D;+ std::vector<float> hr(F);+ for (int f = 0; f < F; ++f) {+ hwant[(size_t) r * F + f] =+ swiglu(mx_dot(emat(E, owner[r], 0), D, f, xr), mx_dot(emat(E, owner[r], 1), D, f, xr), lim);+ hgot[(size_t) r * F + f] = __half2float(*(const __half*) &h16[(size_t) r * F + f]);+ hr[f] = hgot[(size_t) r * F + f]; // the down from the F16 h the kernel kept+ }+ if (light) hr = q8_round(hr.data(), F);+ for (int d = 0; d < D; ++d) ywant[(size_t) r * D + d] = mx_dot(emat(E, owner[r], 2), F, d, hr.data());+ }+ char sw[64];+ std::snprintf(sw, sizeof(sw), "set gate/up (F16 h), %d rows, %d light", R, n_light);+ expect(sw, hgot.data(), hwant.data(), hwant.size(), 2e-3);+ std::snprintf(sw, sizeof(sw), "set down, %d rows, %d light", R, n_light);+ expect(sw, y.data(), ywant.data(), ywant.size(), n_light > 0 ? 1e-4 : 5e-5);+ cudaFree(dxs), cudaFree(dbd), cudaFree(didx), cudaFree(dh16), cudaFree(dy);+ }+ // the CPU lane's rows+ {+ const int n = 3;+ const float* xp[n] = {x.data(), x.data() + D, x.data() + 2 * D};+ std::vector<float> y((size_t) n * 100);+ float* yp[n] = {y.data(), y.data() + 100, y.data() + 200};+ gg::mx_cpu_rows(emat(E, 5, 0), gg::mx_row_bytes(D), D, 7, 107, xp, n, yp);+ std::vector<double> want2((size_t) n * 100);+ for (int i = 0; i < n; ++i)+ for (int r = 7; r < 107; ++r) want2[(size_t) i * 100 + r - 7] = mx_dot(emat(E, 5, 0), D, r, xp[i]);+ expect("CPU lane rows", y.data(), want2.data(), want2.size(), 5e-5);+ }+ // the CPU lane's int8 rows (gg::mx_rows), every tier this CPU runs: the gate and the down shapes, 1..9 tokens (past+ // a call's 8), against the double reference on the q8-rounded activations; the tiers against the scalar one; the+ // activation's bytes (vector and scalar quant, whole and by bands) against q8_round's values+ {+ const int NT = 9, R0 = 5, R1 = 69;+ for (int down = 0; down < 2; ++down) {+ const int k = down ? F : D;+ const uint8_t* w = emat(E, 7, down ? 2 : 0);+ const size_t rb = gg::mx_row_bytes(k), ab = gg::mx_act_bytes(k);+ std::vector<float> xs((size_t) NT * k);+ for (auto& v : xs) v = down ? frand(0.05f) : frand();+ for (int j = 0; j < 32; ++j) xs[(size_t) 2 * k + 32 + j] = 0.0f; // (an all-zero block: scale 0)+ std::vector<uint8_t> act((size_t) NT * ab), act_s(ab), act_b(ab);+ const void* ap[NT];+ bool q_ok = true;+ for (int i = 0; i < NT; ++i) {+ uint8_t* a = act.data() + (size_t) i * ab;+ gg::mx_quant(xs.data() + (size_t) i * k, k, a);+ gg::mx_quant_blocks_scalar(xs.data() + (size_t) i * k, 0, k / 32, act_s.data(), k);+ for (int b0 = 0; b0 < k / 32; b0 += 2) // (by bands of 2 blocks, as the lane's jobs write h)+ gg::mx_quant_blocks(xs.data() + (size_t) i * k + 32 * b0, b0, b0 + 2, act_b.data(), k);+ q_ok = q_ok && std::memcmp(a, act_s.data(), ab) == 0 && std::memcmp(a, act_b.data(), ab) == 0;+ const std::vector<float> xr = q8_round(xs.data() + (size_t) i * k, k);+ for (int b = 0; b < k / 32; ++b)+ for (int j = 0; j < 32; ++j) {+ const int q = gg::mx_act_value(a, b, j);+ const float sc = gg::mx_act_scale(a, k, b);+ q_ok = q_ok && (float) ((double) q * sc) == xr[(size_t) b * 32 + j];+ }+ ap[i] = a;+ }+ if (!q_ok) ++g_fail;+ std::printf("%-44s %s\n", down ? "CPU int8 quant (down's k)" : "CPU int8 quant (gate's k)",+ q_ok ? "ok vector = scalar = bands = q8_round" : "FAIL");+ std::vector<double> want3((size_t) NT * (R1 - R0));+ for (int i = 0; i < NT; ++i) {+ const std::vector<float> xr = q8_round(xs.data() + (size_t) i * k, k);+ for (int r = R0; r < R1; ++r) want3[(size_t) i * (R1 - R0) + r - R0] = mx_dot(w, k, r, xr.data());+ }+ std::vector<float> ref((size_t) NT * (R1 - R0));+ for (int t = 0; t < gg::kMxTiers; ++t) {+ if (!gg::mx_tier_ok(t)) continue;+ for (int n : {1, 2, 3, 8, 9}) {+ std::vector<float> y((size_t) n * (R1 - R0));+ std::vector<float*> yp((size_t) n);+ for (int i = 0; i < n; ++i) yp[(size_t) i] = y.data() + (size_t) i * (R1 - R0);+ gg::mx_rows_tier(t, w, rb, k, R0, R1, ap, n, yp.data(), t == gg::kMxScalar ? 0 : 256);+ char what[96];+ std::snprintf(what, sizeof what, "CPU int8 rows %s %s n=%d", down ? "down" : "gate",+ gg::mx_tier_name(t), n);+ expect(what, y.data(), want3.data(), y.size(), 5e-5);+ if (t == gg::kMxScalar && n == NT) ref = y;+ if (t != gg::kMxScalar && !ref.empty()) { // (every tier the same bits as the scalar one)+ const bool same = std::memcmp(y.data(), ref.data(), y.size() * sizeof(float)) == 0;+ if (!same) ++g_fail;+ std::printf(" %-42s %s\n", what, same ? "ok bit-identical to the scalar tier" : "FAIL differs from the scalar tier");+ }+ }+ }+ }+ std::printf("CPU int8 rows: this CPU's tier %s\n", gg::mx_tier_name(gg::mx_tier()));+ }+ cudaFree(E.dev), cudaFree(dx), cudaFree(dsh), cudaFree(dcpu), cudaFree(dpw), cudaFree(dpp), cudaFree(dflag);+ cudaFree(dh), cudaFree(dout), cudaFree(dgp), cudaFree(dgs), cudaFree(dge), cudaFree(dng), cudaFree(dwh);+ cudaFree(dpart), cudaFree(dwo);+}++// ---------------------------------------------------------------- the low-bit types (ds4_iq)+// random blocks of a type: random bytes, the block scales small normal values (every grid index, sign and code)+std::vector<uint8_t> q_rand(int type, int n, int k) {+ const size_t rb = gg::type_row_bytes(type, k), bb = (size_t) gg::iq::block_bytes(type);+ std::vector<uint8_t> w(rb * n);+ for (auto& v : w) v = (uint8_t) (rng() & 0xff);+ auto put = [&](size_t at, float f) {+ const __half h = __float2half(f);+ std::memcpy(&w[at], &h, 2);+ };+ for (size_t o = 0; o < w.size(); o += bb) {+ const float d = 0.001f + 0.002f * std::fabs(frand());+ switch (type) {+ case gg::iq::kQ4_K: put(o, d), put(o + 2, 0.5f * d); break;+ case gg::iq::kQ2_K: put(o + 80, d), put(o + 82, 0.5f * d); break;+ case gg::iq::kQ3_K: put(o + 108, d); break;+ default: put(o, d); break; // (the i-quants: d first)+ }+ }+ return w;+}+// a row's dot with x in double, the weights ggml's dequantize_row_* values+double q_dot(int type, const uint8_t* row, int k, const float* x) {+ static thread_local std::vector<float> w;+ w.resize(k);+ ggml_get_type_traits((ggml_type) type)->to_float(row, w.data(), k);+ double s = 0;+ for (int i = 0; i < k; ++i) s += (double) w[i] * x[i];+ return s;+}+const char* tn(int t) { return gg::type_name(t); }++void test_mv_q(int type, cudaStream_t s) {+ for (int nc : {1, 3, 16}) {+ const int k = 768, n = 136;+ std::vector<uint8_t> wq = q_rand(type, n, k);+ const size_t xcs = (size_t) k + 8;+ std::vector<float> x(xcs * nc), bias(n);+ for (auto& v : x) v = frand();+ for (auto& v : bias) v = frand(0.1f);+ float *dx = dev(x), *db = dev(bias), *dy = dev(std::vector<float>((size_t) n * nc));+ uint8_t* dw = dev(wq);+ x3::Mat m;+ m.fmt = gg::kFmtGG;+ m.K = type;+ m.raw = dw;+ m.k = k;+ m.n = n;+ x3::MvJob j;+ j.m = &m;+ j.xf = dx;+ j.y = dy;+ j.bias = db;+ j.alpha = 0.5f;+ j.ncol = nc;+ j.xf_cs = xcs;+ j.y_cs = n;+ if (!gg::mv(&j, 1, s)) ++g_fail;+ CU(cudaStreamSynchronize(s));+ std::vector<float> y = host(dy, (size_t) n * nc);+ std::vector<double> want((size_t) n * nc);+ for (int c = 0; c < nc; ++c) {+ const std::vector<float> xr = q8_round(x.data() + c * xcs, k);+ for (int r = 0; r < n; ++r)+ want[(size_t) c * n + r] = 0.5 * q_dot(type, wq.data() + r * gg::type_row_bytes(type, k), k, xr.data()) + bias[r];+ }+ char what[96];+ std::snprintf(what, sizeof(what), "mv %s, %d activations", tn(type), nc);+ expect(what, y.data(), want.data(), want.size(), 2e-5);+ cudaFree(dx), cudaFree(db), cudaFree(dy), cudaFree(dw);+ }+}++struct QExperts {+ int D, F, n_exp, tg, td;+ x3::ExpertLayout lay;+ size_t blob;+ std::vector<uint8_t> host;+ uint8_t* dev = nullptr;+ const uint8_t* mat(int e, int r) const { return host.data() + e * blob + lay.trellis[r]; }+ size_t rb(int r) const { return gg::type_row_bytes(r < 2 ? tg : td, r < 2 ? D : F); }+ double dot(int e, int r, int row, const float* x) const {+ return q_dot(r < 2 ? tg : td, mat(e, r) + row * rb(r), r < 2 ? D : F, x);+ }+};+QExperts make_q(int D, int F, int n_exp, int tg, int td) {+ QExperts E{D, F, n_exp, tg, td, {}, 0, {}, nullptr};+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ E.lay.trellis[r] = off;+ E.lay.K[r] = r < 2 ? tg : td;+ off += ((r < 2 ? F : D) * gg::type_row_bytes(E.lay.K[r], r < 2 ? D : F) + 255) / 256 * 256;+ }+ E.lay.fmt = gg::kFmtMXFP4;+ E.blob = off;+ E.host.assign(E.blob * n_exp, 0);+ for (int e = 0; e < n_exp; ++e)+ for (int r = 0; r < 3; ++r) {+ const std::vector<uint8_t> m = q_rand(E.lay.K[r], r < 2 ? F : D, r < 2 ? D : F);+ std::memcpy(E.host.data() + e * E.blob + E.lay.trellis[r], m.data(), m.size());+ }+ E.dev = dev(E.host);+ return E;+}++// test_experts' checks for low-bit experts: the token path, the window (bit for bit the token path's), the prompt sets+// and the CPU lane (ggml-cpu's arithmetic: Q8_K activations, so against a reference on those)+void test_experts_q(int D, int F, int tg, int td, cudaStream_t s) {+ const int n_exp = 10, k = 6, T = 4;+ const float lim = 10.0f;+ QExperts E = make_q(D, F, n_exp, tg, td);+ std::vector<int> pick((size_t) T * 8, 0);+ std::vector<float> pw((size_t) T * 8, 0.0f);+ std::vector<unsigned long long> pp((size_t) T * 8, 0ull);+ for (int t = 0; t < T; ++t) {+ std::vector<int> all(n_exp);+ for (int i = 0; i < n_exp; ++i) all[i] = i;+ std::shuffle(all.begin(), all.end(), rng);+ for (int i = 0; i < k; ++i) {+ pick[t * 8 + i] = all[i];+ pw[t * 8 + i] = 0.2f + 0.1f * frand();+ const bool on = !(t == 2 && i == 3);+ pp[t * 8 + i] = on ? (unsigned long long) (E.dev + (size_t) all[i] * E.blob) : 0ull;+ }+ }+ std::vector<float> x((size_t) T * D), sh((size_t) T * D), cpu((size_t) T * D);+ for (auto& v : x) v = frand();+ for (auto& v : sh) v = frand(0.1f);+ for (auto& v : cpu) v = frand(0.1f);+ std::vector<int> cflag(T, 0);+ cflag[1] = 1;+ float *dx = dev(x), *dsh = dev(sh), *dcpu = dev(cpu), *dpw = dev(pw);+ unsigned long long* dpp = dev(pp);+ int* dflag = dev(cflag);+ std::vector<double> want((size_t) T * D, 0.0);+ for (int t = 0; t < T; ++t) {+ const std::vector<float> xr = q8_round(x.data() + (size_t) t * D, D);+ for (int i = 0; i < k; ++i) {+ if (pp[t * 8 + i] == 0ull) continue;+ const int e = pick[t * 8 + i];+ std::vector<float> h(F);+ for (int f = 0; f < F; ++f) h[f] = (float) swiglu(E.dot(e, 0, f, xr.data()), E.dot(e, 1, f, xr.data()), lim);+ const std::vector<float> hr = q8_round(h.data(), F);+ for (int d = 0; d < D; ++d) want[(size_t) t * D + d] += pw[t * 8 + i] * E.dot(e, 2, d, hr.data());+ }+ for (int d = 0; d < D; ++d)+ want[(size_t) t * D + d] += (cflag[t] ? cpu[(size_t) t * D + d] : 0.0f) + sh[(size_t) t * D + d];+ }+ std::vector<float> tok((size_t) T * D);+ float* dh = dev(std::vector<float>((size_t) 8 * F));+ float* dout = dev(std::vector<float>(D));+ for (int t = 0; t < T; ++t) {+ x3::MoeArgs a;+ a.plan_ptr = dpp + t * 8;+ a.plan_w = dpw + t * 8;+ a.cpu_part = dcpu + (size_t) t * D;+ a.cpu_flag = dflag + t;+ a.lay = E.lay;+ a.k = k;+ a.n_embd = D;+ a.n_ff = F;+ a.limit = lim;+ gg::moe_gate_up(a, dx + (size_t) t * D, dh, s);+ gg::moe_down(a, dh, dsh + (size_t) t * D, dout, s);+ CU(cudaStreamSynchronize(s));+ CU(cudaMemcpy(tok.data() + (size_t) t * D, dout, (size_t) D * 4, cudaMemcpyDeviceToHost));+ }+ char what[96];+ std::snprintf(what, sizeof(what), "%s/%s token experts (D %d, F %d)", tn(tg), tn(td), D, F);+ expect(what, tok.data(), want.data(), want.size(), 1e-3);+ // the window+ std::vector<unsigned long long> gptr;+ std::vector<int> gstart, gent;+ for (int x0 = 0; x0 < n_exp; ++x0) {+ std::vector<int> ents;+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < k; ++i)+ if (pick[t * 8 + i] == x0 && pp[t * 8 + i] != 0ull) ents.push_back(t * 8 + i);+ if (ents.empty()) continue;+ gptr.push_back((unsigned long long) (E.dev + (size_t) x0 * E.blob));+ gstart.push_back((int) gent.size());+ for (int e : ents) gent.push_back(e);+ }+ gstart.push_back((int) gent.size());+ std::vector<int> ng(1, (int) gptr.size());+ unsigned long long* dgp = dev(gptr);+ int *dgs = dev(gstart), *dge = dev(gent), *dng = dev(ng);+ float* dwh = dev(std::vector<float>((size_t) T * 8 * F));+ float* dpart = dev(std::vector<float>((size_t) T * 8 * D));+ float* dwo = dev(std::vector<float>((size_t) T * D));+ x3::WinArgs w;+ w.grp_ptr = dgp;+ w.grp_start = dgs;+ w.grp_ent = dge;+ w.n_groups = dng;+ w.plan_ptr = dpp;+ w.plan_w = dpw;+ w.cpu_part = dcpu;+ w.cpu_stride = D;+ w.cpu_flag = dflag;+ w.x = dx;+ w.x_stride = D;+ w.h = dwh;+ w.part = dpart;+ w.sh_out = dsh;+ w.out = dwo;+ w.out_stride = D;+ w.lay = E.lay;+ w.T = T;+ w.k = k;+ w.n_embd = D;+ w.n_ff = F;+ w.limit = lim;+ if (!gg::win_experts(w, s)) ++g_fail;+ CU(cudaStreamSynchronize(s));+ std::vector<float> win = host(dwo, (size_t) T * D);+ std::snprintf(what, sizeof(what), " window experts (T %d)", T);+ expect(what, win.data(), want.data(), want.size(), 1e-3);+ const bool same = std::memcmp(win.data(), tok.data(), win.size() * 4) == 0;+ if (!same) ++g_fail;+ std::printf("%-44s %s\n", " ... bit for bit the token path's", same ? "ok" : "FAIL");+ for (int R : {90, 200, 400}) { // the prompt path's set: experts of a few rows (the light kernels), then of more+ const int NT = 30;+ std::vector<int> owner(R), idx(R);+ for (int r = 0; r < R; ++r) owner[r] = (int) (rng() % n_exp);+ std::sort(owner.begin(), owner.end());+ std::vector<int> bounds(n_exp + 1, 0);+ for (int e = 0, r = 0; e <= n_exp; ++e) {+ while (r < R && owner[r] < e) ++r;+ bounds[e] = r;+ }+ int max_rows = 0;+ for (int e = 0; e < n_exp; ++e) max_rows = std::max(max_rows, bounds[e + 1] - bounds[e]);+ std::vector<float> xs((size_t) NT * D);+ for (auto& v : xs) v = frand();+ for (int r = 0; r < R; ++r) idx[r] = (int) (rng() % NT);+ float* dxs = dev(xs);+ int *dbd = dev(bounds), *didx = dev(idx);+ uint16_t* dh16 = dev(std::vector<uint16_t>((size_t) R * F));+ float* dy = dev(std::vector<float>((size_t) R * D));+ x3::SetArgs sa;+ sa.wbase = E.dev;+ sa.stride = E.blob;+ sa.lay = E.lay;+ sa.n_exp = n_exp;+ sa.bounds = dbd;+ sa.row0 = 0;+ sa.max_rows = max_rows;+ sa.n_embd = D;+ sa.n_ff = F;+ sa.limit = lim;+ gg::set_gate_up(sa, dxs, D, didx, dh16, F, s);+ gg::set_down(sa, dh16, F, dy, D, s);+ CU(cudaStreamSynchronize(s));+ std::vector<uint16_t> h16 = host(dh16, (size_t) R * F);+ std::vector<float> y = host(dy, (size_t) R * D);+ std::vector<float> hgot((size_t) R * F);+ std::vector<double> hwant((size_t) R * F), ywant((size_t) R * D);+ // (the light kernels, experts of <= 16 rows: the inputs rounded to int8 per 32 as the token path rounds them; the+ // GEMM tiles: F16 inputs and weights - a low-bit weight is not always an F16 value)+ int n_light = 0;+ for (int e = 0; e < n_exp; ++e) n_light += bounds[e + 1] - bounds[e] <= 16 ? bounds[e + 1] - bounds[e] : 0;+ for (int r = 0; r < R; ++r) {+ const bool light = bounds[owner[r] + 1] - bounds[owner[r]] <= 16;+ const std::vector<float> xq = light ? q8_round(xs.data() + (size_t) idx[r] * D, D) : std::vector<float>();+ const float* xr = light ? xq.data() : xs.data() + (size_t) idx[r] * D;+ std::vector<float> hr(F);+ for (int f = 0; f < F; ++f) {+ hwant[(size_t) r * F + f] = swiglu(E.dot(owner[r], 0, f, xr), E.dot(owner[r], 1, f, xr), lim);+ hgot[(size_t) r * F + f] = __half2float(*(const __half*) &h16[(size_t) r * F + f]);+ hr[f] = hgot[(size_t) r * F + f];+ }+ if (light) hr = q8_round(hr.data(), F);+ for (int d = 0; d < D; ++d) ywant[(size_t) r * D + d] = E.dot(owner[r], 2, d, hr.data());+ }+ char sw[64];+ std::snprintf(sw, sizeof(sw), " set gate/up (F16 h), %d rows, %d light", R, n_light);+ expect(sw, hgot.data(), hwant.data(), hwant.size(), n_light == R ? 2e-3 : 8e-3); // (IQ4_XS: the widest weights in F16)+ std::snprintf(sw, sizeof(sw), " set down, %d rows, %d light", R, n_light);+ expect(sw, y.data(), ywant.data(), ywant.size(), n_light == R ? 1e-4 : 2e-3);+ cudaFree(dxs), cudaFree(dbd), cudaFree(didx), cudaFree(dh16), cudaFree(dy);+ }+ { // the CPU lane: 3 tokens of expert 4, gate / up rows [7, 107) then all its down rows from the CPU's own h+ gg::IqCpu fm;+ std::string err;+ if (!gg::iq_cpu_fmt(tg, td, D, F, fm, err)) {+ std::printf("%-44s FAIL %s\n", " CPU lane", err.c_str());+ ++g_fail;+ } else {+ const int n = 3, e = 4, r0 = 7, r1 = std::min(107, F);+ std::vector<std::vector<uint8_t>> aq(n, std::vector<uint8_t>(fm.act_bytes)),+ hq(n, std::vector<uint8_t>(fm.h_bytes));+ std::vector<std::vector<float>> ff(n, std::vector<float>(F, 0.0f)), yo(n, std::vector<float>(D, 0.0f));+ const void* ap[n];+ const void* hp[n];+ float* fp[n];+ float* yp[n];+ for (int i = 0; i < n; ++i) {+ gg::iq_cpu_quant(fm.gu_act, x.data() + (size_t) i * D, D, aq[i].data());+ ap[i] = aq[i].data();+ fp[i] = ff[i].data();+ }+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, r0, r1, lim);+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, 0, r0, lim);+ gg::iq_cpu_gate_up(fm, E.mat(e, 0), E.mat(e, 1), ap, n, fp, r1, F, lim);+ for (int i = 0; i < n; ++i) {+ gg::iq_cpu_quant(fm.d_act, ff[i].data(), F, hq[i].data());+ hp[i] = hq[i].data();+ yp[i] = yo[i].data();+ }+ gg::iq_cpu_down(fm, E.mat(e, 2), hp, n, yp, 0, D);+ // the reference on the Q8_K values the CPU took (block_q8_K: F32 d, int8 qs[256], int16 bsums[16])+ auto q8k = [](const std::vector<uint8_t>& q, int kk) {+ std::vector<float> v(kk);+ for (int b = 0; b < kk / 256; ++b) {+ const uint8_t* p = q.data() + (size_t) b * 292;+ float d;+ std::memcpy(&d, p, 4);+ for (int j = 0; j < 256; ++j) v[b * 256 + j] = d * (float) (int8_t) p[4 + j];+ }+ return v;+ };+ const bool k8 = fm.gu_act == GGML_TYPE_Q8_K && fm.d_act == GGML_TYPE_Q8_K;+ std::vector<float> got_h, got_y;+ std::vector<double> want_h, want_y;+ for (int i = 0; i < n && k8; ++i) {+ const std::vector<float> xa = q8k(aq[i], D), ha = q8k(hq[i], F);+ for (int f = 0; f < F; ++f) {+ got_h.push_back(ff[i][f]);+ want_h.push_back(swiglu(E.dot(e, 0, f, xa.data()), E.dot(e, 1, f, xa.data()), lim));+ }+ for (int d = 0; d < D; ++d) {+ got_y.push_back(yo[i][d]);+ want_y.push_back(E.dot(e, 2, d, ha.data()));+ }+ }+ if (!k8) {+ std::printf("%-44s FAIL activation types %d / %d\n", " CPU lane", fm.gu_act, fm.d_act);+ ++g_fail;+ } else {+ expect(" CPU lane gate/up (Q8_K)", got_h.data(), want_h.data(), want_h.size(), 1e-4);+ expect(" CPU lane down (Q8_K)", got_y.data(), want_y.data(), want_y.size(), 1e-4);+ }+ }+ }+ cudaFree(E.dev), cudaFree(dx), cudaFree(dsh), cudaFree(dcpu), cudaFree(dpw), cudaFree(dpp), cudaFree(dflag);+ cudaFree(dh), cudaFree(dout), cudaFree(dgp), cudaFree(dgs), cudaFree(dge), cudaFree(dng), cudaFree(dwh);+ cudaFree(dpart), cudaFree(dwo);+}++// the role pairs the tests take: every type as gate / up and as down+const std::pair<int, int> kQPairs[] = {+ {gg::iq::kIQ2_XXS, gg::iq::kIQ3_XXS}, {gg::iq::kIQ2_S, gg::iq::kIQ3_S}, {gg::iq::kIQ3_XXS, gg::iq::kIQ3_S},+ {gg::iq::kIQ4_XS, gg::iq::kQ4_K}, {gg::iq::kQ2_K, gg::iq::kQ3_K}, {gg::iq::kQ4_K, gg::iq::kIQ4_XS},+ {gg::iq::kQ3_K, gg::iq::kQ2_K}, {gg::iq::kIQ3_S, gg::iq::kIQ2_XXS}, {gg::iq::kIQ3_S, gg::iq::kIQ2_S}};++// a token's 6 low-bit experts at full size: GB/s of the weights read+void bench_q(cudaStream_t s) {+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ const int D = 5120, F = 2304;+ for (const auto& pr : kQPairs) {+ QExperts E = make_q(D, F, 6, pr.first, pr.second);+ std::vector<unsigned long long> pp(8, 0ull);+ std::vector<float> pw(8, 0.1f);+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);+ unsigned long long* dpp = dev(pp);+ float* dpw = dev(pw);+ float* dx = dev(std::vector<float>(D, 0.01f));+ float* dh = dev(std::vector<float>((size_t) 8 * F));+ float* dout = dev(std::vector<float>(D));+ x3::MoeArgs ma;+ ma.plan_ptr = dpp;+ ma.plan_w = dpw;+ ma.lay = E.lay;+ ma.k = 6;+ ma.n_embd = D;+ ma.n_ff = F;+ ma.limit = 10.0f;+ const double gu = 6.0 * 2 * F * E.rb(0), dn = 6.0 * D * E.rb(2);+ auto timed = [&](const char* what, double bytes, auto&& fn) {+ fn();+ CU(cudaStreamSynchronize(s));+ cudaEventRecord(a, s);+ for (int i = 0; i < 50; ++i) fn();+ cudaEventRecord(b, s);+ CU(cudaEventSynchronize(b));+ float ms = 0;+ cudaEventElapsedTime(&ms, a, b);+ std::printf("bench %-40s %8.1f us %6.0f GB/s\n", what, ms * 1000 / 50, bytes / (ms / 50 * 1e-3) / 1e9);+ };+ char what[96];+ std::snprintf(what, sizeof(what), "6 %s/%s experts (gate/up + down)", tn(pr.first), tn(pr.second));+ timed(what, gu + dn, [&] {+ gg::moe_gate_up(ma, dx, dh, s);+ gg::moe_down(ma, dh, nullptr, dout, s);+ });+ timed(" gate/up alone", gu, [&] { gg::moe_gate_up(ma, dx, dh, s); });+ timed(" down + combine alone", dn, [&] { gg::moe_down(ma, dh, nullptr, dout, s); });+ cudaFree(E.dev), cudaFree(dpp), cudaFree(dpw), cudaFree(dx), cudaFree(dh), cudaFree(dout);+ }+ // the prompt path's sets: 32 experts x R rows (128: an 8K chunk's share of 384 experts; 8: a short prompt's - the+ // weights' bandwidth), the MXFP4 ones first+ for (int R : {128, 8}) {+ const int NE = 32;+ for (int pi = -1; pi < (int) (sizeof(kQPairs) / sizeof(kQPairs[0])); ++pi) {+ x3::ExpertLayout lay;+ size_t blob;+ double wbytes;+ uint8_t* wdev;+ char what[96];+ if (pi < 0) {+ Experts M = make_experts(D, F, NE);+ lay = M.lay, blob = M.blob, wdev = M.dev;+ wbytes = 3.0 * gg::mx_bytes(F, D);+ std::snprintf(what, sizeof(what), "set MXFP4, %d experts x %d rows", NE, R);+ } else {+ QExperts E = make_q(D, F, NE, kQPairs[pi].first, kQPairs[pi].second);+ lay = E.lay, blob = E.blob, wdev = E.dev;+ wbytes = 2.0 * F * E.rb(0) + (double) D * E.rb(2);+ std::snprintf(what, sizeof(what), "set %s/%s, %d experts x %d rows", tn(kQPairs[pi].first),+ tn(kQPairs[pi].second), NE, R);+ }+ std::vector<int> bounds(NE + 1), idx(NE * R);+ for (int e = 0; e <= NE; ++e) bounds[e] = e * R;+ for (int i = 0; i < NE * R; ++i) idx[i] = i;+ int *db = dev(bounds), *di = dev(idx);+ float* xs = dev(std::vector<float>((size_t) NE * R * D, 0.01f));+ uint16_t* h16 = dev(std::vector<uint16_t>((size_t) NE * R * F));+ float* y = dev(std::vector<float>((size_t) NE * R * D));+ x3::SetArgs sa;+ sa.wbase = wdev;+ sa.stride = blob;+ sa.lay = lay;+ sa.n_exp = NE;+ sa.bounds = db;+ sa.max_rows = R;+ sa.n_embd = D;+ sa.n_ff = F;+ sa.limit = 10.0f;+ auto one = [&] {+ gg::set_gate_up(sa, xs, D, di, h16, F, s);+ gg::set_down(sa, h16, F, y, D, s);+ };+ one();+ CU(cudaStreamSynchronize(s));+ cudaEventRecord(a, s);+ for (int i = 0; i < 5; ++i) one();+ cudaEventRecord(b, s);+ CU(cudaEventSynchronize(b));+ float ms = 0;+ cudaEventElapsedTime(&ms, a, b);+ ms /= 5;+ std::printf("bench %-40s %8.2f ms %6.1f TFLOP/s %6.0f GB/s of weights\n", what, ms,+ 2.0 * NE * R * 3.0 * D * F / (ms * 1e-3) / 1e12, NE * wbytes / (ms * 1e-3) / 1e9);+ cudaFree(wdev), cudaFree(db), cudaFree(di), cudaFree(xs), cudaFree(h16), cudaFree(y);+ }+ }+}++// the CPU lane's speed: one full-size expert (gate / up rows, then down rows) for 1 and 4 tokens, on `nth` threads (rows+// split between them), GB/s of weights read - MXFP4 (mx_cpu_rows) and the low-bit pairs (ggml-cpu's rows)+void bench_cpu(int nth) {+ const int D = 5120, F = 2304, reps = 6;+ auto run = [&](const char* what, double bytes, const std::function<void(int, int)>& rows, int n_rows) {+ auto pass = [&] {+ std::vector<std::thread> th;+ for (int t = 0; t < nth; ++t)+ th.emplace_back([&, t] { rows(n_rows * t / nth, n_rows * (t + 1) / nth); });+ for (auto& x : th) x.join();+ };+ pass();+ const auto t0 = std::chrono::steady_clock::now();+ for (int i = 0; i < reps; ++i) pass();+ const double s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() / reps;+ std::printf("cpu %-46s %8.2f ms %6.1f GB/s (%d threads)\n", what, s * 1e3, bytes / s / 1e9, nth);+ };+ std::vector<float> x((size_t) 4 * D), h((size_t) 4 * F);+ for (auto& v : x) v = frand();+ for (auto& v : h) v = frand(0.2f);+ for (int n : {1, 4}) {+ Experts M = make_experts(D, F, 1);+ const float* xp[4] = {x.data(), x.data() + D, x.data() + 2 * D, x.data() + 3 * D};+ const float* hp[4] = {h.data(), h.data() + F, h.data() + 2 * F, h.data() + 3 * F};+ std::vector<float> yo((size_t) 4 * D), yg((size_t) 4 * F);+ char what[96];+ std::snprintf(what, sizeof(what), "MXFP4 expert, %d token%s", n, n > 1 ? "s" : "");+ run(what, 2.0 * gg::mx_bytes(F, D), [&](int r0, int r1) {+ float* yp[4] = {yg.data(), yg.data() + F, yg.data() + 2 * F, yg.data() + 3 * F};+ for (int i = 0; i < n; ++i) yp[i] += r0;+ gg::mx_cpu_rows(emat(M, 0, 0), gg::mx_row_bytes(D), D, r0, r1, xp, n, yp);+ gg::mx_cpu_rows(emat(M, 0, 1), gg::mx_row_bytes(D), D, r0, r1, xp, n, yp);+ }, F);+ run(" down", (double) gg::mx_bytes(D, F), [&](int r0, int r1) {+ float* yp[4] = {yo.data() + r0, yo.data() + D + r0, yo.data() + 2 * D + r0, yo.data() + 3 * D + r0};+ gg::mx_cpu_rows(emat(M, 0, 2), gg::mx_row_bytes(F), F, r0, r1, hp, n, yp);+ }, D);+ cudaFree(M.dev);+ for (const auto& pr : kQPairs) {+ QExperts E = make_q(D, F, 1, pr.first, pr.second);+ gg::IqCpu fm;+ std::string err;+ if (!gg::iq_cpu_fmt(pr.first, pr.second, D, F, fm, err)) continue;+ std::vector<uint8_t> aq((size_t) 4 * fm.act_bytes), hq((size_t) 4 * fm.h_bytes);+ const void* ap[4];+ const void* hqp[4];+ for (int i = 0; i < 4; ++i) {+ gg::iq_cpu_quant(fm.gu_act, xp[i], D, aq.data() + i * fm.act_bytes);+ gg::iq_cpu_quant(fm.d_act, hp[i], F, hq.data() + i * fm.h_bytes);+ ap[i] = aq.data() + i * fm.act_bytes;+ hqp[i] = hq.data() + i * fm.h_bytes;+ }+ std::snprintf(what, sizeof(what), "%s/%s expert, %d token%s", tn(pr.first), tn(pr.second), n, n > 1 ? "s" : "");+ run(what, 2.0 * F * E.rb(0), [&](int r0, int r1) {+ float* fp[4] = {yg.data(), yg.data() + F, yg.data() + 2 * F, yg.data() + 3 * F};+ gg::iq_cpu_gate_up(fm, E.mat(0, 0), E.mat(0, 1), ap, n, fp, r0, r1, 10.0f);+ }, F);+ run(" down", (double) D * E.rb(2), [&](int r0, int r1) {+ float* yp[4] = {yo.data(), yo.data() + D, yo.data() + 2 * D, yo.data() + 3 * D};+ gg::iq_cpu_down(fm, E.mat(0, 2), hqp, n, yp, r0, r1);+ }, D);+ cudaFree(E.dev);+ }+ }+}++// the decode kernels' speed: GB/s of weights read (a token's dense GEMVs and its experts)+void bench(cudaStream_t s) {+ cudaEvent_t a, b;+ cudaEventCreate(&a);+ cudaEventCreate(&b);+ auto timed = [&](const char* what, double bytes, auto&& fn) {+ fn();+ CU(cudaStreamSynchronize(s));+ cudaEventRecord(a, s);+ const int reps = 50;+ for (int i = 0; i < reps; ++i) fn();+ cudaEventRecord(b, s);+ CU(cudaEventSynchronize(b));+ float ms = 0;+ cudaEventElapsedTime(&ms, a, b);+ std::printf("bench %-40s %8.1f us %6.0f GB/s\n", what, ms * 1000 / reps, bytes / (ms / reps * 1e-3) / 1e9);+ };+ const std::pair<int, int> shapes[] = {{1280, 32768}, {5120, 1280}, {8192, 5120}, {5120, 129280}};+ for (int fmt : {gg::kFmtBF16, gg::kFmtQ8_0})+ for (const auto& kn : shapes) {+ const int k = kn.first, n = kn.second;+ const size_t bytes = fmt == gg::kFmtBF16 ? (size_t) n * k * 2 : gg::q8_row_bytes(k) * n;+ uint8_t* dw;+ CU(cudaMalloc(&dw, bytes));+ CU(cudaMemset(dw, 0, bytes));+ float *dx, *dy;+ CU(cudaMalloc(&dx, (size_t) k * 4));+ CU(cudaMemset(dx, 0, (size_t) k * 4));+ CU(cudaMalloc(&dy, (size_t) n * 4));+ x3::Mat m;+ m.fmt = fmt;+ m.raw = dw;+ m.k = k;+ m.n = n;+ x3::MvJob j;+ j.m = &m;+ j.xf = dx;+ j.y = dy;+ char what[64];+ std::snprintf(what, sizeof(what), "mv %s %dx%d", fmt == gg::kFmtBF16 ? "BF16" : "Q8_0", n, k);+ timed(what, (double) bytes, [&] { gg::mv(&j, 1, s); });+ cudaFree(dw), cudaFree(dx), cudaFree(dy);+ }+ { // a token's 6 experts at full size+ const int D = 5120, F = 2304;+ Experts E = make_experts(D, F, 6);+ std::vector<unsigned long long> pp(8, 0ull);+ std::vector<float> pw(8, 0.1f);+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);+ unsigned long long* dpp = dev(pp);+ float* dpw = dev(pw);+ float* dx = dev(std::vector<float>(D, 0.01f));+ float* dh = dev(std::vector<float>((size_t) 8 * F));+ float* dout = dev(std::vector<float>(D));+ x3::MoeArgs ma;+ ma.plan_ptr = dpp;+ ma.plan_w = dpw;+ ma.lay = E.lay;+ ma.k = 6;+ ma.n_embd = D;+ ma.n_ff = F;+ ma.limit = 10.0f;+ timed("6 MXFP4 experts (gate/up + down)", 6.0 * (2 * gg::mx_bytes(F, D) + gg::mx_bytes(D, F)), [&] {+ gg::moe_gate_up(ma, dx, dh, s);+ gg::moe_down(ma, dh, nullptr, dout, s);+ });+ cudaFree(E.dev);+ { // the prompt path: 32 experts x 128 rows each (an 8K chunk's share of 384 experts)+ const int NE = 32, R = 128;+ Experts S = make_experts(D, F, NE);+ std::vector<int> bounds(NE + 1), idx(NE * R);+ for (int e = 0; e <= NE; ++e) bounds[e] = e * R;+ for (int i = 0; i < NE * R; ++i) idx[i] = i;+ int *db = dev(bounds), *di = dev(idx);+ float* xs = dev(std::vector<float>((size_t) NE * R * D, 0.01f));+ uint16_t* h16 = dev(std::vector<uint16_t>((size_t) NE * R * F));+ float* y = dev(std::vector<float>((size_t) NE * R * D));+ x3::SetArgs sa;+ sa.wbase = S.dev;+ sa.stride = S.blob;+ sa.lay = S.lay;+ sa.n_exp = NE;+ sa.bounds = db;+ sa.max_rows = R;+ sa.n_embd = D;+ sa.n_ff = F;+ sa.limit = 10.0f;+ const double flops = 2.0 * NE * R * 3.0 * D * F;+ cudaEvent_t a0, a1;+ cudaEventCreate(&a0);+ cudaEventCreate(&a1);+ gg::set_gate_up(sa, xs, D, di, h16, F, s);+ gg::set_down(sa, h16, F, y, D, s);+ cudaEventRecord(a0, s);+ for (int i = 0; i < 5; ++i) {+ gg::set_gate_up(sa, xs, D, di, h16, F, s);+ gg::set_down(sa, h16, F, y, D, s);+ }+ cudaEventRecord(a1, s);+ CU(cudaEventSynchronize(a1));+ float ms = 0;+ cudaEventElapsedTime(&ms, a0, a1);+ std::printf("bench %-40s %8.1f ms %6.1f TFLOP/s\n", "prompt set: 32 experts x 128 rows", ms / 5,+ flops / (ms / 5 * 1e-3) / 1e12);+ cudaFree(S.dev), cudaFree(db), cudaFree(di), cudaFree(xs), cudaFree(h16), cudaFree(y);+ }+ E.dev = nullptr;+ E = make_experts(D, F, 6);+ for (int i = 0; i < 6; ++i) pp[i] = (unsigned long long) (E.dev + (size_t) i * E.blob);+ CU(cudaMemcpy(dpp, pp.data(), pp.size() * 8, cudaMemcpyHostToDevice));+ timed(" gate/up alone", 6.0 * 2 * gg::mx_bytes(F, D), [&] { gg::moe_gate_up(ma, dx, dh, s); });+ timed(" down + combine alone", 6.0 * gg::mx_bytes(D, F), [&] { gg::moe_down(ma, dh, nullptr, dout, s); });+ cudaFree(E.dev), cudaFree(dpp), cudaFree(dpw), cudaFree(dx), cudaFree(dh), cudaFree(dout);+ }+}++// ---- the caches' packed rows (ds4_kernels.cu): DeepSeek's FP8 / FP4 roundings kept as codes and scales. Each packed+// row against the BF16 row the same finish writes - a host decode of every value bit for bit - then the readers (the+// window ring and its rollback, the sparse attention over the compressed KV, the indexer's scores) the same bits from+// either, at full size (a 64K context's compressed rows)+namespace dk = strata::ds4;++float h_e2m1(uint32_t c) {+ static const float m[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f};+ const float v = m[c & 7u];+ return (c & 8u) ? -v : v;+}+float h_e4m3(uint32_t q) {+ const int e = (int) ((q >> 3) & 15u), m = (int) (q & 7u);+ const float v = e == 0 ? std::ldexp((float) m, -9) : std::ldexp(1.0f + (float) m / 8.0f, e - 7);+ return (q & 0x80u) ? -v : v;+}+float h_ue8m0(uint32_t b) {+ const uint32_t u = b << 23;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}+// value i of a packed row of d values+float h_unpack(int fmt, const uint8_t* row, int d, int i) {+ if (fmt == dk::kKvFp8) return h_e4m3(row[i]) * h_ue8m0(row[d + i / 32]);+ const uint32_t c = (uint32_t) row[i / 2] >> (4 * (i & 1));+ return h_e2m1(c) * (fmt == dk::kKvFp4 ? h_e4m3(row[d / 2 + i / 16]) : h_ue8m0(row[d / 2 + i / 32]));+}++// n rows of d values: groups of 16 at scales 2^-30 .. 2^10, some all zero, some with negative zeros, some tiny+std::vector<float> kv_rows(int n, int d) {+ std::vector<float> x((size_t) n * d);+ for (int r = 0; r < n; ++r)+ for (int g0 = 0; g0 < d; g0 += 16) {+ const int kind = (int) (rng() % 16u);+ const float sc = std::ldexp(1.0f, (int) (rng() % 41u) - 30);+ for (int e = g0; e < g0 + 16; ++e) {+ float v = frand(sc);+ if (kind == 0) v = 0.0f;+ else if (kind == 1 && e % 3 == 0) v = -0.0f;+ else if (kind == 2) v = frand(1e-7f);+ else if (kind == 3 && e == g0) v = sc * 1000.0f; // (one value far above the rest)+ x[(size_t) r * d + e] = v;+ }+ }+ return x;+}++// finish(quant) of n rows of d: the BF16 rows (dst) and the packed rows (pdst), both from the same input+void kv_finish(const float* x, int n, int d, int quant, uint16_t* bf, uint8_t* pk, cudaStream_t s) {+ dk::Finish f;+ f.quant = quant;+ f.dst = bf;+ f.dst_stride = d;+ dk::finish(const_cast<float*>(x), n, d, d, f, s);+ dk::Finish p;+ p.quant = quant;+ p.pdst = pk;+ p.pdst_stride = (long long) dk::kv_row_bytes(quant, d);+ dk::finish(const_cast<float*>(x), n, d, d, p, s);+}++// every packed value decoded on the host against the BF16 row's bits+void kv_check_rows(const char* what, int fmt, const std::vector<uint16_t>& bf, const std::vector<uint8_t>& pk, int n,+ int d) {+ const size_t rb = dk::kv_row_bytes(fmt, d);+ size_t bad = 0, at = 0;+ for (int r = 0; r < n; ++r)+ for (int i = 0; i < d; ++i) {+ const float v = h_unpack(fmt, pk.data() + (size_t) r * rb, d, i);+ uint32_t u;+ std::memcpy(&u, &v, 4);+ if ((u & 0xffffu) != 0u || (uint16_t) (u >> 16) != bf[(size_t) r * d + i]) {+ if (bad++ == 0) at = (size_t) r * d + i;+ }+ }+ if (bad) ++g_fail;+ std::printf("%-44s %s %d rows x %d, %zu of %zu values differ (first at %zu), %zu B a row vs %d\n", what,+ bad ? "FAIL" : "ok ", n, d, bad, (size_t) n * d, at, rb, d * 2);+}++void kv_same(const char* what, const void* a, const void* b, size_t bytes) {+ const bool ok = std::memcmp(a, b, bytes) == 0;+ if (!ok) ++g_fail;+ std::printf("%-44s %s %zu bytes %s\n", what, ok ? "ok " : "FAIL", bytes, ok ? "identical" : "DIFFER");+}++void test_kv_pack(cudaStream_t s) {+ const int HD = 512, IX = 128, NC = 65536 / 4 + 1, WIN = 128;+ // the compressed KV (FP4, an E4M3 scale per 16) and the index keys (FP4, a power-of-two scale per 32)+ std::vector<float> xc = kv_rows(NC, HD), xi = kv_rows(NC, IX);+ float *d_xc = dev(xc), *d_xi = dev(xi);+ uint16_t *c16 = nullptr, *i16 = nullptr;+ uint8_t *c4 = nullptr, *i4 = nullptr;+ CU(cudaMalloc(&c16, (size_t) NC * HD * 2));+ CU(cudaMalloc(&i16, (size_t) NC * IX * 2));+ CU(cudaMalloc(&c4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4, HD)));+ CU(cudaMalloc(&i4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4p, IX)));+ kv_finish(d_xc, NC, HD, dk::kKvFp4, c16, c4, s);+ kv_finish(d_xi, NC, IX, dk::kKvFp4p, i16, i4, s);+ CU(cudaStreamSynchronize(s));+ kv_check_rows("kv pack: compressed KV (FP4 + E4M3 / 16)", dk::kKvFp4, host(c16, (size_t) NC * HD),+ host(c4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4, HD)), NC, HD);+ kv_check_rows("kv pack: index keys (FP4 + 2^k / 32)", dk::kKvFp4p, host(i16, (size_t) NC * IX),+ host(i4, (size_t) NC * dk::kv_row_bytes(dk::kKvFp4p, IX)), NC, IX);+ kv_same("kv pack: finish leaves its input", xc.data(), host(d_xc, xc.size()).data(), xc.size() * 4);++ // the window ring (FP8, a power-of-two scale per 32): batches of 128 rows through the finish's BF16 rows, into the+ // ring (its scales derived from the BF16 values) and back out+ {+ const size_t rb = dk::kv_row_bytes(dk::kKvFp8, HD);+ std::vector<float> xw = kv_rows(WIN, HD);+ float* d_xw = dev(xw);+ uint16_t *ext = nullptr, *ext2 = nullptr;+ uint8_t *ring = nullptr, *pk = nullptr, *bak = nullptr;+ CU(cudaMalloc(&ext, (size_t) 2 * WIN * HD * 2));+ CU(cudaMalloc(&ext2, (size_t) WIN * HD * 2));+ CU(cudaMalloc(&ring, (size_t) WIN * rb));+ CU(cudaMalloc(&pk, (size_t) WIN * rb));+ CU(cudaMalloc(&bak, (size_t) dk::kWin * rb));+ size_t bad = 0, bad_pk = 0, n_vals = 0;+ for (int b = 0; b < 64; ++b) {+ if (b > 0) {+ xw = kv_rows(WIN, HD);+ CU(cudaMemcpy(d_xw, xw.data(), xw.size() * 4, cudaMemcpyHostToDevice));+ }+ kv_finish(d_xw, WIN, HD, dk::kKvFp8, ext + (size_t) WIN * HD, pk, s);+ dk::ext_to_ring(ext, ring, 0, WIN, WIN, HD, s);+ dk::ring_to_ext(ring, ext2, WIN, WIN, HD, s); // (p0 = n: the rows as they lie)+ CU(cudaStreamSynchronize(s));+ const std::vector<uint16_t> want = host(ext + (size_t) WIN * HD, (size_t) WIN * HD),+ got = host(ext2, (size_t) WIN * HD);+ for (size_t i = 0; i < want.size(); ++i) bad += want[i] != got[i];+ const std::vector<uint8_t> hr = host(ring, (size_t) WIN * rb), hp = host(pk, (size_t) WIN * rb);+ for (int r = 0; r < WIN; ++r) // the ring's codes against the finish's own packing: the same values+ for (int i = 0; i < HD; ++i)+ bad_pk += h_unpack(dk::kKvFp8, hr.data() + (size_t) r * rb, HD, i) !=+ h_unpack(dk::kKvFp8, hp.data() + (size_t) r * rb, HD, i) ||+ std::signbit(h_unpack(dk::kKvFp8, hr.data() + (size_t) r * rb, HD, i)) !=+ std::signbit(h_unpack(dk::kKvFp8, hp.data() + (size_t) r * rb, HD, i));+ n_vals += want.size();+ }+ if (bad || bad_pk) ++g_fail;+ std::printf("%-44s %s %zu values, %zu differ through the ring, %zu from the finish's packing; %zu B a row vs %d\n",+ "kv pack: window ring (FP8 + 2^k / 32)", bad || bad_pk ? "FAIL" : "ok ", n_vals, bad, bad_pk, rb,+ HD * 2);+ // a speculative window's rollback: 16 rows over the ring at position 1000 (backed up), the first 5 kept+ const std::vector<uint8_t> before = host(ring, (size_t) WIN * rb);+ xw = kv_rows(dk::kWin, HD);+ CU(cudaMemcpy(d_xw, xw.data(), (size_t) dk::kWin * HD * 4, cudaMemcpyHostToDevice));+ kv_finish(d_xw, dk::kWin, HD, dk::kKvFp8, ext + (size_t) WIN * HD, pk, s);+ dk::ext_to_ring(ext, ring, 1000, WIN, dk::kWin, HD, s, bak);+ CU(cudaStreamSynchronize(s));+ std::vector<uint8_t> want = before;+ const std::vector<uint8_t> after = host(ring, (size_t) WIN * rb);+ for (int t = 0; t < 5; ++t) {+ const size_t r = (size_t) ((1000 + t) % WIN);+ std::memcpy(want.data() + r * rb, after.data() + r * rb, rb);+ }+ dk::ring_restore(ring, bak, 1000, WIN, 5, dk::kWin, HD, s);+ CU(cudaStreamSynchronize(s));+ kv_same("kv pack: window ring rollback", want.data(), host(ring, (size_t) WIN * rb).data(), want.size());+ CU(cudaFree(d_xw));+ CU(cudaFree(ext));+ CU(cudaFree(ext2));+ CU(cudaFree(ring));+ CU(cudaFree(pk));+ CU(cudaFree(bak));+ }++ // the sparse attention: the window rows and 512 selected compressed rows each, 64 heads - the BF16 rows' kernel+ // against the packed rows' every way (decoded once a query into the scratch; staged, every number of heads a+ // block), and the time of each (a decode step's query, a few, a verification's 16, chunk slabs of 16 and 256)+ {+ const int TM = 256, NH = 64, SEL = 512, ST = 16;+ std::vector<float> q((size_t) TM * NH * HD), sink(NH);+ for (float& v : q) v = frand(0.2f);+ for (float& v : sink) v = frand(1.0f);+ std::vector<int> sel((size_t) TM * SEL);+ for (int& v : sel) v = (int) (rng() % (unsigned) NC);+ std::vector<float> xw = kv_rows(WIN + TM, HD);+ float *d_q = dev(q), *d_sink = dev(sink), *d_xw = dev(xw), *o1 = nullptr, *o2 = nullptr;+ int* d_sel = dev(sel);+ uint16_t* ext = nullptr;+ uint8_t* pk = nullptr;+ dk::AttnScratch scr;+ scr.max_t = ST;+ scr.cap = SEL;+ CU(cudaMalloc(&scr.rows, (size_t) ST * SEL * HD * 2));+ CU(cudaMalloc(&ext, (size_t) (WIN + TM) * HD * 2));+ CU(cudaMalloc(&pk, (size_t) (WIN + TM) * dk::kv_row_bytes(dk::kKvFp8, HD)));+ CU(cudaMalloc(&o1, q.size() * 4));+ CU(cudaMalloc(&o2, q.size() * 4));+ kv_finish(d_xw, WIN + TM, HD, dk::kKvFp8, ext, pk, s);+ cudaEvent_t e0, e1;+ CU(cudaEventCreate(&e0));+ CU(cudaEventCreate(&e1));+ const float scale = 1.0f / std::sqrt((float) HD);+ const struct {+ int T, gen, iters;+ const char* what;+ } shapes[] = {{1, 0, 400, "1 query"}, {2, 0, 400, "2 queries"}, {4, 0, 200, "4 queries"},+ {8, 0, 200, "8 queries"}, {16, 0, 100, "16 queries"}, {16, 1, 100, "a chunk slab of 16"},+ {256, 1, 10, "a chunk slab of 256"}};+ for (const auto& sh : shapes) {+ const int T = sh.T;+ dk::AttnRows ar;+ if (sh.gen) { // a chunk's rows at 40000.. (ratio 4: every query its 512)+ ar.gen = 1;+ ar.p0 = 40000;+ ar.t0 = 0;+ ar.win = WIN;+ ar.R = 4;+ ar.kcap = SEL;+ } else+ for (int t = 0; t < T; ++t) {+ ar.win0[t] = t + 1;+ ar.nwin[t] = WIN;+ ar.nsel[t] = SEL - 31 * t;+ }+ const size_t n = (size_t) T * NH * HD;+ dk::sparse_attn(d_q, ext, c16, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvBf16);+ CU(cudaStreamSynchronize(s));+ const std::vector<float> want = host(o1, n);+ bool same = true;+ for (int g : {-1, 0, 1, 2, 4, 8}) {+ CU(cudaMemset(o2, 0xff, n * 4));+ dk::sparse_attn(d_q, ext, c4, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o2, s, dk::kKvFp4, g, &scr);+ CU(cudaStreamSynchronize(s));+ same = same && std::memcmp(want.data(), host(o2, n).data(), n * 4) == 0;+ }+ // the time: BF16 rows; packed as the engine runs it (0), through the scratch (-1), staged (its own G)+ float us[4];+ const int ways[4] = {99, 0, -1, -2};+ for (int w = 0; w < 4; ++w) {+ CU(cudaEventRecord(e0, s));+ for (int it = 0; it < sh.iters; ++it) {+ if (ways[w] == 99)+ dk::sparse_attn(d_q, ext, c16, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvBf16);+ else+ dk::sparse_attn(d_q, ext, c4, d_sel, SEL, ar, d_sink, NH, HD, T, scale, o1, s, dk::kKvFp4,+ ways[w] == -2 ? 0 : ways[w], ways[w] == -2 ? nullptr : &scr);+ }+ CU(cudaEventRecord(e1, s));+ CU(cudaEventSynchronize(e1));+ float ms = 0.0f;+ CU(cudaEventElapsedTime(&ms, e0, e1));+ us[w] = ms * 1000.0f / sh.iters;+ }+ if (!same) ++g_fail;+ char what[96];+ std::snprintf(what, sizeof what, "kv pack: sparse attention, %s", sh.what);+ std::printf("%-44s %s %s every way; BF16 rows %.1f us a call, packed %.1f (decoded once a query %.1f, "+ "staged %.1f)\n",+ what, same ? "ok " : "FAIL", same ? "identical" : "DIFFER", us[0], us[1], us[2], us[3]);+ }+ CU(cudaEventDestroy(e0));+ CU(cudaEventDestroy(e1));+ CU(cudaFree(d_q));+ CU(cudaFree(d_sink));+ CU(cudaFree(d_xw));+ CU(cudaFree(d_sel));+ CU(cudaFree(ext));+ CU(cudaFree(pk));+ CU(cudaFree(o1));+ CU(cudaFree(o2));+ CU(cudaFree(scr.rows));+ }++ // the indexer's scores over every key (32 heads), BF16 vs packed keys+ {+ const int T = 4, NH = 32;+ std::vector<float> q((size_t) T * NH * IX), w((size_t) T * 64);+ for (float& v : q) v = frand(1.0f);+ for (float& v : w) v = frand(0.3f);+ float *d_q = dev(q), *d_w = dev(w), *o1 = nullptr, *o2 = nullptr;+ const long long ss = NC + 64;+ CU(cudaMalloc(&o1, (size_t) T * ss * 4));+ CU(cudaMalloc(&o2, (size_t) T * ss * 4));+ CU(cudaMemset(o1, 0, (size_t) T * ss * 4));+ CU(cudaMemset(o2, 0, (size_t) T * ss * 4));+ dk::Rows lim;+ for (int t = 0; t < T; ++t) {+ lim.n[t] = NC - 3 * t;+ lim.k[t] = 512;+ }+ dk::index_score(d_q, d_w, i16, NH, IX, T, NC, nullptr, 0, 1, lim, o1, ss, s, dk::kKvBf16);+ dk::index_score(d_q, d_w, i4, NH, IX, T, NC, nullptr, 0, 1, lim, o2, ss, s, dk::kKvFp4p);+ CU(cudaStreamSynchronize(s));+ kv_same("kv pack: index scores", host(o1, (size_t) T * ss).data(), host(o2, (size_t) T * ss).data(),+ (size_t) T * ss * 4);+ { // (the time of each over the 16385 keys, 50 times)+ cudaEvent_t e0, e1;+ CU(cudaEventCreate(&e0));+ CU(cudaEventCreate(&e1));+ float ms[2];+ for (int f = 0; f < 2; ++f) {+ CU(cudaEventRecord(e0, s));+ for (int it = 0; it < 50; ++it)+ dk::index_score(d_q, d_w, f ? (const void*) i4 : (const void*) i16, NH, IX, T, NC, nullptr, 0, 1, lim,+ o1, ss, s, f ? dk::kKvFp4p : dk::kKvBf16);+ CU(cudaEventRecord(e1, s));+ CU(cudaEventSynchronize(e1));+ CU(cudaEventElapsedTime(&ms[f], e0, e1));+ }+ std::printf("%-44s BF16 keys %.1f us a call, packed %.1f us\n", "kv pack: index score time", ms[0] * 20.0f,+ ms[1] * 20.0f);+ CU(cudaEventDestroy(e0));+ CU(cudaEventDestroy(e1));+ }+ CU(cudaFree(d_q));+ CU(cudaFree(d_w));+ CU(cudaFree(o1));+ CU(cudaFree(o2));+ }+ CU(cudaFree(d_xc));+ CU(cudaFree(d_xi));+ CU(cudaFree(c16));+ CU(cudaFree(i16));+ CU(cudaFree(c4));+ CU(cudaFree(i4));+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess) {+ std::printf("kv pack: %s\n", cudaGetErrorString(e));+ ++g_fail;+ }+}++} // namespace++// ---------------------------------------------------------------- Engram rows in Q6_K+__global__ void q6k_rows_kernel(const uint8_t* rows, int hd, float* out) { // (engram_dequant's Q6_K branch)+ const int r = blockIdx.x;+ for (int e = threadIdx.x; e < hd; e += blockDim.x)+ out[(size_t) r * hd + e] =+ strata::ds4::q6k_value(rows + (size_t) r * hd + (size_t) (e / 256) * strata::ds4::kQ6kBytes, e % 256);+}+void test_engram_q6k(cudaStream_t s) {+ const int n = 64, hd = 512, sb = hd / 256; // rows of two superblocks, each row hd bytes apart (as the engine keeps)+ std::vector<uint8_t> rows((size_t) n * hd, 0), packed((size_t) n * sb * 210);+ for (auto& v : packed) v = (uint8_t) (rng() & 0xff);+ for (size_t b = 0; b < (size_t) n * sb; ++b) { // d: a small normal F16+ const __half h = __float2half(0.0005f + 0.002f * std::fabs(frand()));+ std::memcpy(&packed[b * 210 + 208], &h, 2);+ }+ for (int r = 0; r < n; ++r) std::memcpy(&rows[(size_t) r * hd], &packed[(size_t) r * sb * 210], (size_t) sb * 210);+ std::vector<float> want((size_t) n * hd);+ ggml_get_type_traits(GGML_TYPE_Q6_K)->to_float(packed.data(), want.data(), (int64_t) n * hd);+ uint8_t* d_rows = dev(rows);+ float* d_out = dev(std::vector<float>((size_t) n * hd));+ q6k_rows_kernel<<<n, 256, 0, s>>>(d_rows, hd, d_out);+ CU(cudaStreamSynchronize(s));+ const std::vector<float> got = host(d_out, (size_t) n * hd);+ const bool same = std::memcmp(got.data(), want.data(), got.size() * 4) == 0;+ if (!same) ++g_fail;+ std::printf("%-44s %s\n", "Engram Q6_K rows = ggml's, bit for bit", same ? "ok" : "FAIL");+ cudaFree(d_rows), cudaFree(d_out);+}++int main(int argc, char** argv) {+ cudaStream_t s;+ CU(cudaStreamCreate(&s));+ if (!gg::prepare(s)) return 1;+ if (argc >= 2 && std::strcmp(argv[1], "--bench") == 0) {+ bench(s);+ bench_q(s);+ return 0;+ }+ if (argc >= 2 && std::strcmp(argv[1], "--bench-q") == 0) {+ bench_q(s);+ return 0;+ }+ if (argc >= 2 && std::strcmp(argv[1], "--bench-cpu") == 0) { // [threads]+ bench_cpu(argc >= 3 ? std::atoi(argv[2]) : 1);+ return 0;+ }+ if (argc >= 2 && std::strcmp(argv[1], "--kv") == 0) { // the caches' packing alone+ test_kv_pack(s);+ std::printf("%s (%d failed)\n", g_fail ? "FAILED" : "ALL OK", g_fail);+ return g_fail ? 1 : 0;+ }+ test_kv_pack(s);+ test_mv(s);+ test_engram_q6k(s);+ test_experts(512, 256, s);+ for (int t : {gg::iq::kIQ2_XXS, gg::iq::kIQ2_S, gg::iq::kIQ3_XXS, gg::iq::kIQ3_S, gg::iq::kIQ4_XS, gg::iq::kQ2_K,+ gg::iq::kQ3_K, gg::iq::kQ4_K})+ test_mv_q(t, s);+ for (const auto& pr : kQPairs) test_experts_q(512, 256, pr.first, pr.second, s);+ if (argc >= 3) {+ test_experts(std::atoi(argv[1]), std::atoi(argv[2]), s);+ for (int i : {0, 4}) test_experts_q(std::atoi(argv[1]), std::atoi(argv[2]), kQPairs[i].first, kQPairs[i].second, s);+ }+ if (gg::launch_errors() != 0) {+ std::printf("launch errors: %d\n", gg::launch_errors());+ ++g_fail;+ }+ std::printf("%s (%d failed)\n", g_fail ? "FAILED" : "ALL OK", g_fail);+ return g_fail ? 1 : 0;+}diff --git a/tests/dsv41/ds4_mx_bench.cpp b/tests/dsv41/ds4_mx_bench.cppnew file mode 100644index 000000000..af6b77650--- /dev/null+++ b/tests/dsv41/ds4_mx_bench.cpp@@ -0,0 +1,381 @@+// tests/ds4_mx_bench.cpp - how fast does this CPU run DeepSeek-V4.1-Flash's MXFP4 experts on the CPU lane?+//+// ds4_mx_bench [--threads 1,6,12] [--T 1,2,4,8,16] [--experts 16] [--per-layer 3] [--secs 0.4] [--pf 0]+// [--kernels float,avx2,avxvnni,avx512,avx512vnni,ggml,scalar] [--file PATH] [--make-file PATH]+//+// Experts of the real shape (gate / up 2304 rows x 5120, down 5120 x 2304, MXFP4: 18.8 MB each) with random codes in+// anonymous memory (huge pages where the OS gives them), or in a file's mapping (--file: 4 KiB page-cache pages, as the+// engine reads the GGUF; --make-file writes one). A "layer" is --per-layer experts, as the lane runs them: the gate /+// up rows of every expert in 64-row jobs on the pool, then (a barrier) their down rows; the layers walk through the+// experts, so the weights come from RAM, not the cache. Prints the weights' GB/s for each kernel x tokens x threads+// (T tokens a call: each row read once for up to 8 of them), the int8 tiers' agreement with the scalar tier, and the+// cost of the activation quant and of madvise(WILLNEED) on cached pages.+#include "../src/ds4/ds4_gguf_cpu.hpp"+#include "../src/ds4/ds4_pool.hpp"++#if defined(STRATA_NATIVE_EXPERTS)+#include "ggml-cpu.h"+#include "ggml.h"+#endif++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++namespace gg = strata::ds4::gguf;+namespace os = strata::ds4::os;+using strata::ds4::Pool;++namespace {++constexpr int E = 5120, FF = 2304, RB = 64;+const size_t kGr = (size_t) E / 32 * 17, kDr = (size_t) FF / 32 * 17; // row bytes+const size_t kMat = kGr * FF; // gate (= up = down) bytes+const size_t kExpert = 3 * kMat;++double now_s() {+ return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();+}+std::vector<int> ints(const std::string& s) {+ std::vector<int> v;+ size_t a = 0;+ while (a < s.size()) {+ size_t b = s.find(',', a);+ if (b == std::string::npos) b = s.size();+ v.push_back(std::atoi(s.substr(a, b - a).c_str()));+ a = b + 1;+ }+ return v;+}+std::vector<std::string> words(const std::string& s) {+ std::vector<std::string> v;+ size_t a = 0;+ while (a < s.size()) {+ size_t b = s.find(',', a);+ if (b == std::string::npos) b = s.size();+ v.push_back(s.substr(a, b - a));+ a = b + 1;+ }+ return v;+}++// the logical CPUs, one of each core first, then their second threads (pinning order)+std::vector<int> cpu_order() {+ std::vector<int> first, rest;+#if defined(_WIN32)+ DWORD len = 0;+ GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len);+ std::vector<uint8_t> buf(len);+ if (len > 0 && GetLogicalProcessorInformationEx(RelationProcessorCore,+ (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len)) {+ for (size_t o = 0; o < buf.size();) {+ auto* e = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (buf.data() + o);+ if (e->Processor.GroupCount > 0 && e->Processor.GroupMask[0].Group == 0) {+ const KAFFINITY m = e->Processor.GroupMask[0].Mask;+ bool f = true;+ for (int b = 0; b < 64; ++b)+ if ((m >> b) & 1) {+ (f ? first : rest).push_back(b);+ f = false;+ }+ }+ o += e->Size;+ }+ }+#else+ for (int c = 0; c < 1024; ++c) {+ std::FILE* f = std::fopen(("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list").c_str(), "r");+ if (f == nullptr) break;+ int s0 = -1;+ if (std::fscanf(f, "%d", &s0) != 1) s0 = c;+ std::fclose(f);+ (s0 == c ? first : rest).push_back(c);+ }+#endif+ if (first.empty())+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency()); ++c) first.push_back((int) c);+ std::sort(first.begin(), first.end());+ std::sort(rest.begin(), rest.end());+ first.insert(first.end(), rest.begin(), rest.end());+ return first;+}++void fill_mx(uint8_t* p, size_t bytes, uint64_t seed) { // random MXFP4 blocks, scales 2^-14 .. 2^-6 as weights'+ std::mt19937_64 rng(seed);+ for (size_t b = 0; b + 17 <= bytes; b += 17) {+ const uint64_t r = rng(), s = rng();+ p[b] = (uint8_t) (114 + r % 9);+ std::memcpy(p + b + 1, &r, 8);+ std::memcpy(p + b + 9, &s, 8);+ }+}++enum Kern { kFloat = 100, kGgml = 101 };+int kern_id(const std::string& s) {+ if (s == "float") return kFloat;+ if (s == "ggml") return kGgml;+ for (int t = 0; t < gg::kMxTiers; ++t)+ if (s == gg::mx_tier_name(t)) return t;+ return -1;+}++struct Acts { // T tokens' activations of E and FF values in every form a kernel takes+ int T = 0;+ std::vector<float> xf, hf; // F32 (rounded as q8_round) [T][E], [T][FF]+ std::vector<uint8_t> xq, hq; // mx_quant's [T][2E], [T][2FF]+ std::vector<uint8_t> x8, h8; // ggml's block_q8_0 [T][E / 32 * 34]+ void make(int t, uint64_t seed) {+ T = t;+ std::mt19937 rng((unsigned) seed);+ std::normal_distribution<float> nd(0.0f, 1.0f);+ xf.resize((size_t) T * E);+ hf.resize((size_t) T * FF);+ for (auto& v : xf) v = nd(rng);+ for (auto& v : hf) v = 0.05f * nd(rng);+ xq.resize((size_t) T * gg::mx_act_bytes(E));+ hq.resize((size_t) T * gg::mx_act_bytes(FF));+ x8.resize((size_t) T * E / 32 * 34);+ h8.resize((size_t) T * FF / 32 * 34);+ for (int i = 0; i < T; ++i) {+ gg::mx_quant(&xf[(size_t) i * E], E, &xq[(size_t) i * gg::mx_act_bytes(E)]);+ gg::mx_quant(&hf[(size_t) i * FF], FF, &hq[(size_t) i * gg::mx_act_bytes(FF)]);+#if defined(STRATA_NATIVE_EXPERTS)+ const auto* q8 = ggml_get_type_traits_cpu(GGML_TYPE_Q8_0);+ q8->from_float(&xf[(size_t) i * E], &x8[(size_t) i * E / 32 * 34], E);+ q8->from_float(&hf[(size_t) i * FF], &h8[(size_t) i * FF / 32 * 34], FF);+#endif+ gg::q8_round(&xf[(size_t) i * E], E, &xf[(size_t) i * E]);+ gg::q8_round(&hf[(size_t) i * FF], FF, &hf[(size_t) i * FF]);+ }+ }+};++// rows [r0, r1) of one matrix (k values a row) against the T activations, kernel `kern`+void rows(int kern, const uint8_t* w, size_t rb, int k, int r0, int r1, const Acts& a, float* const* y, int pf) {+ const bool down = k == FF;+ if (kern == kFloat) {+ for (int i0 = 0; i0 < a.T; i0 += 8) {+ const float* xp[8];+ const int n = std::min(8, a.T - i0);+ for (int i = 0; i < n; ++i) xp[i] = (down ? a.hf.data() : a.xf.data()) + (size_t) (i0 + i) * k;+ gg::mx_cpu_rows(w, rb, k, r0, r1, xp, n, y + i0);+ }+ return;+ }+ if (kern == kGgml) {+#if defined(STRATA_NATIVE_EXPERTS)+ static const ggml_vec_dot_t dot = ggml_get_type_traits_cpu(GGML_TYPE_MXFP4)->vec_dot;+ const uint8_t* x8 = down ? a.h8.data() : a.x8.data();+ for (int r = r0; r < r1; ++r)+ for (int i = 0; i < a.T; ++i) dot(k, &y[i][r - r0], 0, w + (size_t) r * rb, 0, x8 + (size_t) i * k / 32 * 34, 0, 1);+#endif+ return;+ }+ const void* ap[16];+ for (int i = 0; i < a.T; ++i) ap[i] = (down ? a.hq.data() : a.xq.data()) + (size_t) i * gg::mx_act_bytes(k);+ gg::mx_rows_tier(kern, w, rb, k, r0, r1, ap, a.T, y, pf);+}++struct Bench {+ const uint8_t* mem = nullptr;+ int n_exp = 16, per_layer = 3;+ double secs = 0.4;+ // GB/s of weights with `pool` (its size the threads), kernel kern, the T tokens of a+ double run(Pool& pool, int kern, const Acts& a, int pf) {+ const int GB = FF / RB, DB = E / RB;+ std::vector<std::vector<float>> ys((size_t) pool.size() * 64);+ int layer = 0;+ size_t bytes = 0;+ auto one_layer = [&] {+ const int e0 = (layer * per_layer) % n_exp;+ std::function<void(int)> fgu = [&](int j) {+ const int e = (e0 + j / (2 * GB)) % n_exp, m = (j / GB) % 2, b = j % GB;+ float yb[16][RB];+ float* yp[16];+ for (int i = 0; i < a.T; ++i) yp[i] = yb[i];+ rows(kern, mem + (size_t) e * kExpert + (size_t) m * kMat, kGr, E, b * RB, b * RB + RB, a, yp, pf);+ };+ pool.run(per_layer * 2 * GB, fgu);+ std::function<void(int)> fd = [&](int j) {+ const int e = (e0 + j / DB) % n_exp, b = j % DB;+ float yb[16][RB];+ float* yp[16];+ for (int i = 0; i < a.T; ++i) yp[i] = yb[i];+ rows(kern, mem + (size_t) e * kExpert + 2 * kMat, kDr, FF, b * RB, b * RB + RB, a, yp, pf);+ };+ pool.run(per_layer * DB, fd);+ bytes += (size_t) per_layer * kExpert;+ ++layer;+ };+ for (int w = 0; w < 2; ++w) one_layer(); // (warm: the threads, the code)+ bytes = 0;+ const double t0 = now_s();+ double t = t0;+ while (t - t0 < secs) {+ one_layer();+ t = now_s();+ }+ return bytes / (t - t0) / 1e9;+ }+};+++} // namespace++int main(int argc, char** argv) {+ std::vector<int> threads = {1}, Ts = {1, 2, 4, 8, 16}, pfs = {0};+ std::vector<std::string> kerns = {"float", "avx2", "avxvnni", "avx512", "avx512vnni", "ggml"};+ std::string file, make_file;+ Bench B;+ for (int i = 1; i + 1 < argc; i += 2) {+ const std::string a = argv[i], v = argv[i + 1];+ if (a == "--threads") threads = ints(v);+ else if (a == "--T") Ts = ints(v);+ else if (a == "--pf") pfs = ints(v);+ else if (a == "--experts") B.n_exp = std::atoi(v.c_str());+ else if (a == "--per-layer") B.per_layer = std::atoi(v.c_str());+ else if (a == "--secs") B.secs = std::atof(v.c_str());+ else if (a == "--kernels") kerns = words(v);+ else if (a == "--file") file = v;+ else if (a == "--make-file") make_file = v;+ }+#if defined(STRATA_NATIVE_EXPERTS)+ ggml_cpu_init();+#endif+ const size_t total = (size_t) B.n_exp * kExpert;+ if (!make_file.empty()) {+ std::FILE* f = std::fopen(make_file.c_str(), "wb");+ if (f == nullptr) return 1;+ std::vector<uint8_t> buf(kExpert);+ for (int e = 0; e < B.n_exp; ++e) {+ fill_mx(buf.data(), buf.size(), 1000 + (uint64_t) e);+ std::fwrite(buf.data(), 1, buf.size(), f);+ }+ std::fclose(f);+ std::printf("wrote %s: %d experts, %.0f MB\n", make_file.c_str(), B.n_exp, total / 1e6);+ return 0;+ }+ os::fd_t fd = os::kBadFd;+ if (!file.empty()) {+ fd = os::open_ro(file);+ if (fd == os::kBadFd || os::file_size(fd) < total) {+ std::fprintf(stderr, "%s: cannot open, or smaller than %d experts\n", file.c_str(), B.n_exp);+ return 1;+ }+ B.mem = (const uint8_t*) os::map_ro(fd, total);+ volatile uint64_t s = 0; // (into the page cache and the mapping)+ for (size_t o = 0; o < total; o += 4096) s += B.mem[o];+ } else {+ uint8_t* m = (uint8_t*) os::anon_alloc(total, -1, 1);+ if (m == nullptr) return 1;+ for (int e = 0; e < B.n_exp; ++e) fill_mx(m + (size_t) e * kExpert, kExpert, 1000 + (uint64_t) e);+ B.mem = m;+ }+ std::printf("CPU tiers:");+ for (int t = 0; t < gg::kMxTiers; ++t)+ if (gg::mx_tier_ok(t)) std::printf(" %s", gg::mx_tier_name(t));+ std::printf(" (mx_rows takes %s: %s); weights %s, %d experts (%.0f MB), %d a layer\n", gg::mx_tier_name(gg::mx_tier()),+ gg::mx_tier_report(), file.empty() ? "anonymous memory" : file.c_str(), B.n_exp, total / 1e6, B.per_layer);++ // ---- the int8 tiers against the scalar tier, and the old F32 path against both (T = 3, 64 rows of each kind)+ {+ Acts a;+ a.make(3, 7);+ for (int down = 0; down < 2; ++down) {+ const int k = down ? FF : E;+ const size_t rb = down ? kDr : kGr;+ const uint8_t* w = B.mem + (down ? 2 * kMat : 0);+ float ref[3][RB], got[3][RB];+ float* rp[3] = {ref[0], ref[1], ref[2]};+ float* gp[3] = {got[0], got[1], got[2]};+ rows(gg::kMxScalar, w, rb, k, 0, RB, a, rp, 0);+ for (int t = 0; t < gg::kMxTiers + 1; ++t) {+ const int kern = t < gg::kMxTiers ? t : kFloat;+ if (kern != kFloat && (kern == gg::kMxScalar || !gg::mx_tier_ok(kern))) continue;+ rows(kern, w, rb, k, 0, RB, a, gp, 0);+ double md = 0, mx = 0;+ for (int i = 0; i < 3; ++i)+ for (int r = 0; r < RB; ++r) {+ md = std::max(md, (double) std::fabs(got[i][r] - ref[i][r]));+ mx = std::max(mx, (double) std::fabs(ref[i][r]));+ }+ std::printf(" %-6s %-10s vs scalar: max |diff| %.3g (max |y| %.3g, rel %.2g)%s\n", down ? "down" : "gate",+ kern == kFloat ? "float" : gg::mx_tier_name(kern), md, mx, md / std::max(mx, 1e-30),+ kern != kFloat && std::memcmp(got, ref, sizeof got) == 0 ? " bit-identical" : "");+ }+ }+ // the quant's two forms: the same bytes+ std::vector<uint8_t> q1(gg::mx_act_bytes(E)), q2(gg::mx_act_bytes(E));+ gg::mx_quant_blocks(a.xf.data(), 0, E / 32, q1.data(), E);+ gg::mx_quant_blocks_scalar(a.xf.data(), 0, E / 32, q2.data(), E);+ std::printf(" quant: vector vs scalar bytes %s\n", q1 == q2 ? "identical" : "DIFFER");+ }++ // ---- the quant and madvise costs+ {+ std::vector<float> x(E), r(E);+ std::mt19937 rng(3);+ std::normal_distribution<float> nd;+ for (auto& v : x) v = nd(rng);+ std::vector<uint8_t> q(gg::mx_act_bytes(E));+ const int N = 20000;+ double t0 = now_s();+ for (int i = 0; i < N; ++i) gg::mx_quant(x.data(), E, q.data());+ const double tq = (now_s() - t0) / N;+ t0 = now_s();+ for (int i = 0; i < N / 10; ++i) gg::q8_round(x.data(), E, r.data());+ const double tr = (now_s() - t0) / (N / 10);+ std::printf("activation of %d: mx_quant %.2f us, q8_round (old, scalar) %.2f us\n", E, tq * 1e6, tr * 1e6);+ if (!file.empty()) {+ t0 = now_s();+ int n = 0;+ for (int e = 0; e < B.n_exp; ++e)+ for (int m = 0; m < 3; ++m, ++n)+ os::prefetch({{B.mem + (size_t) e * kExpert + (size_t) m * kMat, kMat}});+ std::printf("madvise(WILLNEED) of a cached expert matrix (%.1f MB): %.1f us\n", kMat / 1e6,+ (now_s() - t0) / n * 1e6);+ t0 = now_s();+ int nres = 0;+ for (int rep = 0; rep < 10; ++rep)+ for (int e = 0; e < B.n_exp; ++e)+ for (int m = 0; m < 3; ++m)+ nres += os::resident_sampled(B.mem + (size_t) e * kExpert + (size_t) m * kMat, kMat);+ std::printf("os::resident_sampled (8 pages) of an expert matrix: %.2f us (%d of %d resident)\n",+ (now_s() - t0) / (10 * n) * 1e6, nres / 10, n);+ }+ }++ // ---- the table+ const std::vector<int> order = cpu_order();+ std::printf("\n%-11s %3s %4s %5s %9s %10s\n", "kernel", "T", "thr", "pf", "GB/s", "GB/s/thr");+ for (int nt : threads) {+ nt = std::max(1, std::min(nt, (int) order.size()));+ Pool pool(nt - 1, std::vector<int>(order.begin() + 1, order.begin() + nt), 20000);+ os::pin_self(order[0]);+ for (const std::string& ks : kerns) {+ const int kern = kern_id(ks);+ if (kern < 0 || (kern < gg::kMxTiers && !gg::mx_tier_ok(kern))) continue;+#if !defined(STRATA_NATIVE_EXPERTS)+ if (kern == kGgml) continue;+#endif+ for (int T : Ts) {+ if (T < 1 || T > 16) continue;+ Acts a;+ a.make(T, 11 + (uint64_t) T);+ for (int pf : pfs) {+ if (pf != 0 && (kern == kFloat || kern == kGgml || kern == gg::kMxScalar)) continue;+ const double g = B.run(pool, kern, a, pf);+ std::printf("%-11s %3d %4d %5d %9.2f %10.2f\n", ks.c_str(), T, nt, pf, g, g / nt);+ std::fflush(stdout);+ }+ }+ }+ }+ return 0;+}diff --git a/third_party/project-maya/LICENSE b/third_party/project-maya/LICENSEnew file mode 100644index 000000000..7cf2d3348--- /dev/null+++ b/third_party/project-maya/LICENSE@@ -0,0 +1,22 @@+MIT License++Copyright (c) 2026 Peasant Smith and the Project Maya contributors+Copyright (c) 2026 Niko1221 and the Strata contributors++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/third_party/project-maya/NOTICE.md b/third_party/project-maya/NOTICE.mdnew file mode 100644index 000000000..0b3bf289f--- /dev/null+++ b/third_party/project-maya/NOTICE.md@@ -0,0 +1,11 @@+# Project Maya (maya-ds4), DeepSeek-V4.1 engine++The files under `src/dsv41/`, `include/strata/dsv41/`, `tests/dsv41/` and `tools/dsv41/` come from+https://github.com/mw00/project-maya (release v1.1.17, commit 8d9a9b5d), its DeepSeek engine `src/ds4`, `tests/ds4_*`,+`tools/ds4` and the kernel sources it builds on (`exl3`, `sampler`). Project Maya is itself built on Strata.++Copyright (c) 2026 Peasant Smith (mw00) and the Project Maya contributors, including needmorevram.+MIT License, the full text is in `LICENSE` beside this file. The notice stays with every copy.++Changes made while porting into Strata are listed in the git history of the `src/dsv41` directory+(the first commit of each file is the unmodified upstream copy).diff --git a/tools/dsv41/bytecheck.sh b/tools/dsv41/bytecheck.shnew file mode 100644index 000000000..18f606f2d--- /dev/null+++ b/tools/dsv41/bytecheck.sh@@ -0,0 +1,66 @@+#!/bin/bash+# tools/ds4/bytecheck.sh - maya-ds4's byte-identity check of two builds (a change that must not move a bit: the same+# prompt through each, every logit row dumped, the files compared byte for byte, the greedy tokens compared).+#+# tools/ds4/bytecheck.sh <base maya-ds4> <new maya-ds4> <pack> <out dir> <ids file> [<check> ...]+# checks: windows (DS4_CHUNK_MIN=100000: windows of 16), chunked (the prompt as one chunked run), drafter+# (--dspark on); default: windows chunked. 64 greedy tokens after the prompt.+# env: NEW_RESERVE_MIB (default 600, the engine's --reserve-mib default; the packed KV caches: 624, 626 with the+# drafter - see "placement" below); GPU (default 0); CTX (default 8192)+#+# Placement. --no-expert-cache --expert-profile none fix WHICH experts VRAM holds (by index) and keep them there, but+# not HOW MANY: that follows the free VRAM at load. A change that frees or uses VRAM (smaller caches, a new buffer)+# moves an expert or more between VRAM and the CPU lane / the prompt path's streamed sets, and the prompt path sums a+# token's experts batch by batch - another order, other bits (the windows path only differs when a moved expert is+# picked). So whenever a change frees or uses VRAM, give the build with more free VRAM that much more reserve:+# NEW_RESERVE_MIB = 600 + what it frees, measured, not added up from the allocations (the driver rounds them): load+# both once (the check's own first pair will do), and with E experts in VRAM each and F MiB free after the load,+# freed = (F_new - F_base) + (E_new - E_base) * the MiB of an expert (the load line: experts GB / E). Then check that+# the load lines' "(N of M in VRAM)" agree - this script prints them. The packed KV caches on a 32 GB V100 at 8K:+# 24 MiB (626 with the drafter's 26; the allocations' sum said 18-19, and 618 / 619 left one expert more).+#+# A prompt that a checkpoint splits (600 tokens, then 400 more from the checkpoint) is not the same bits as the 1000 in+# one chunked run: chunk boundaries change the arithmetic. Compare a split run with the same split in the other build.+set -u+base=$1+new=$2+pack=$3+out=$4+ids=$(cat "$5")+shift 5+checks=${*:-windows chunked}+mkdir -p "$out"+common="--pack $pack --max-context ${CTX:-8192} --vision off --verbose --no-expert-cache --expert-profile none --n 64"+run() { # <build> <check> <exe> [env ...] -- <args ...>+ local b=$1 c=$2 exe=$3+ shift 3+ local envs=()+ while [ "$1" != "--" ]; do+ envs+=("$1")+ shift+ done+ shift+ local reserve=600+ [ "$b" = new ] && reserve=${NEW_RESERVE_MIB:-600}+ env CUDA_VISIBLE_DEVICES=${GPU:-0} ${envs[@]+"${envs[@]}"} "$exe" $common --reserve-mib $reserve "$@" --ids "$ids" \+ --dump-logits "$out/${b}_$c.f32" > "$out/${b}_$c.out" 2> "$out/${b}_$c.err"+ echo "$b $c rc=$?"+}+for c in $checks; do+ for b in base new; do+ exe=$base+ [ $b = new ] && exe=$new+ case $c in+ windows) run $b $c "$exe" DS4_CHUNK_MIN=100000 -- --dspark none ;;+ chunked) run $b $c "$exe" -- --dspark none --window 1048576 ;;+ drafter) run $b $c "$exe" -- --dspark on ;;+ *) echo "unknown check $c" && exit 2 ;;+ esac+ done+ echo "== $c"+ for b in base new; do+ echo " $b: $(grep -h 'in VRAM' "$out/${b}_$c.err" | head -1 | sed 's/.*experts/experts/')"+ done+ echo " logits: $(cmp -s "$out/base_$c.f32" "$out/new_$c.f32" && echo IDENTICAL || echo DIFFER);" \+ "tokens: $(cmp -s <(grep -h '^[0-9]' "$out/base_$c.out") <(grep -h '^[0-9]' "$out/new_$c.out") && echo IDENTICAL || echo DIFFER)"+donediff --git a/tools/dsv41/chat_template.jinja b/tools/dsv41/chat_template.jinjanew file mode 100644index 000000000..7b983a45f--- /dev/null+++ b/tools/dsv41/chat_template.jinja@@ -0,0 +1,93 @@+{#- DeepSeek-V4.1's prompt format (the model repo's encoding/encoding.py), as a chat template for Maya's server.+ Turns: <|User|> ... <|Assistant|> (reasoning</think>)answer<|end▁of▁sentence|>. Thinking mode ends the prompt+ with <think> and starts with a reasoning-effort line (low 50, high 75, max 100); chat mode ends it with </think>.+ Tool results (role "tool") are <tool_result> blocks of a user turn, merged with the user messages around them;+ tool calls are DSML blocks; the tools are listed in the system turn. Earlier turns keep their reasoning only when+ tools are given (encoding.py's drop_thinking). A picture in a message's content list is its placeholder+ <|deepseek_image|>, a block of its own between the text blocks (the server expands it into the picture's span). -#}+{%- set thinking = enable_thinking is not defined or enable_thinking -%}+{%- set efforts = {'low': 50, 'medium': 75, 'high': 75, 'xhigh': 100, 'max': 100} -%}+{%- if reasoning_effort is defined and reasoning_effort is number -%}+{%- set budget = reasoning_effort -%}+{%- elif reasoning_effort is defined and reasoning_effort in efforts -%}+{%- set budget = efforts[reasoning_effort] -%}+{%- else -%}+{%- set budget = 75 -%}+{%- endif -%}+{%- set has_tools = tools is defined and tools is not none and tools | length > 0 -%}+{%- set keep = has_tools -%}+{%- set ns = namespace(last_user=-1, schemas='') -%}+{%- if has_tools -%}{%- for t in tools -%}{%- set ns.schemas = ns.schemas ~ (t | tojson) ~ ('' if loop.last else '\n') -%}{%- endfor -%}{%- endif -%}+{%- for m in messages -%}{%- if m.role == 'user' -%}{%- set ns.last_user = loop.index0 -%}{%- endif -%}{%- endfor -%}+{%- macro text(c) -%}+{%- if c is string -%}{{ c }}{%- elif c is iterable -%}+{%- for p in c if p.type in ['text', 'image'] -%}{%- if not loop.first -%}{{ '\n\n' }}{%- endif -%}{{ p.text if p.type == 'text' else '<|deepseek_image|>' }}{%- endfor -%}+{%- endif -%}+{%- endmacro -%}+{%- macro tools_block() -%}+## Tools++You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following:++<|DSML| calls>+<|DSML| invoke name="$TOOL_NAME">+<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML| parameter>+...+</|DSML| invoke>+<|DSML| invoke name="$TOOL_NAME2">+...+</|DSML| invoke>+</|DSML| calls>++String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.++If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.++Otherwise, output directly after </think> with tool calls or final response.++### Available Tool Schemas++{{ ns.schemas }}++You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.+{% endmacro -%}+{{- '<|begin▁of▁sentence|>' -}}+{%- set effort_line = ('Reasoning Effort: ' ~ budget ~ ' (range 1-100, the higher the value, the more thorough the reasoning)\n\n') if thinking else '' -%}+{%- if messages and messages[0].role == 'system' -%}+{{- '<|System|>' ~ effort_line ~ text(messages[0].content) -}}+{%- if has_tools -%}{{- '\n\n' ~ tools_block() -}}{%- endif -%}+{%- elif has_tools -%}+{{- '<|System|>' ~ effort_line ~ '\n\n' ~ tools_block() -}}+{%- elif thinking -%}+{{- '<|System|>' ~ effort_line -}}+{%- endif -%}+{%- for m in messages -%}+{%- set i = loop.index0 -%}+{%- set prev = messages[i - 1].role if i > 0 else '' -%}+{%- set next = messages[i + 1].role if i + 1 < messages | length else '' -%}+{%- if m.role == 'system' and i == 0 -%}+{%- elif m.role in ['user', 'tool', 'system'] -%}+{%- if prev in ['user', 'tool'] or (prev == 'system' and i - 1 > 0) -%}{{- '\n\n' -}}{%- else -%}{{- '<|User|>' -}}{%- endif -%}+{%- if m.role == 'tool' -%}{{- '<tool_result>' ~ text(m.content) ~ '</tool_result>' -}}{%- else -%}{{- text(m.content) -}}{%- endif -%}+{%- if next not in ['user', 'tool', 'system'] -%}+{%- if thinking and (keep or i >= ns.last_user) -%}{{- '<|Assistant|><think>' -}}{%- else -%}{{- '<|Assistant|></think>' -}}{%- endif -%}+{%- endif -%}+{%- elif m.role == 'assistant' -%}+{%- if thinking and (keep or i > ns.last_user) -%}{{- (m.reasoning_content or '') ~ '</think>' -}}{%- endif -%}+{{- text(m.content) -}}+{%- if m.tool_calls -%}+{{- '\n\n<|DSML| calls>\n' -}}+{%- for c in m.tool_calls -%}+{%- set f = c.function if c.function is defined else c -%}+{{- '<|DSML| invoke name="' ~ f.name ~ '">\n' -}}+{%- for k, v in (f.arguments or {}).items() -%}+{{- '<|DSML| parameter name="' ~ k ~ '" string="' ~ ('true' if v is string else 'false') ~ '">' ~ (v if v is string else v | tojson) ~ '</|DSML| parameter>' -}}+{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}+{%- endfor -%}+{{- '\n</|DSML| invoke>' -}}{%- if not loop.last -%}{{- '\n' -}}{%- endif -%}+{%- endfor -%}+{{- '\n</|DSML| calls>' -}}+{%- endif -%}+{{- '<|end▁of▁sentence|>' -}}+{%- endif -%}+{%- endfor -%}diff --git a/tools/dsv41/compare.py b/tools/dsv41/compare.pynew file mode 100644index 000000000..9dfd91b43--- /dev/null+++ b/tools/dsv41/compare.py@@ -0,0 +1,60 @@+"""tools/ds4/compare.py - the engine's logits (maya-ds4 --dump-logits) against the reference's (tools/ds4/ref.py --dump).++ python tools/ds4/compare.py --ref ref.npz --engine engine.f32 [--vocab 129280]++Per position: the cosine of the two logit rows, the largest difference, whether the top token agrees, and the KL+divergence of the engine's distribution from the reference's.+"""+from __future__ import annotations++import argparse+import sys++import numpy as np+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--ref", required=True)+ ap.add_argument("--engine", required=True)+ ap.add_argument("--vocab", type=int, default=129280)+ ap.add_argument("--layers", help="the engine's DS4_DUMP_H file: each layer's residual stream against the ref's")+ a = ap.parse_args()+ r = np.load(a.ref)+ ref = r["logits"].astype(np.float64)+ eng = np.fromfile(a.engine, dtype=np.float32).reshape(-1, a.vocab).astype(np.float64)+ n = min(len(ref), len(eng))+ agree = 0+ kls = []+ for i in range(n):+ x, y = ref[i], eng[i]+ cos = float(x @ y / (np.linalg.norm(x) * np.linalg.norm(y)))+ px = np.exp(x - x.max())+ px /= px.sum()+ py = np.exp(y - y.max())+ py /= py.sum()+ kl = float(np.sum(px * (np.log(px + 1e-30) - np.log(py + 1e-30))))+ kls.append(kl)+ top = int(x.argmax()) == int(y.argmax())+ agree += top+ print("pos %3d cos %.6f max|d| %8.4f top %s (ref %6d eng %6d) KL %.2e" %+ (i, cos, np.abs(x - y).max(), "same" if top else "DIFF", x.argmax(), y.argmax(), kl))+ print("%d positions: top-1 agrees at %d, mean KL %.3e, max KL %.3e" % (n, agree, np.mean(kls), np.max(kls)))+ if a.layers:+ hs = r["hs"] # [layers, T, hc, dim]+ nl, T = hs.shape[0], hs.shape[1]+ eh = np.fromfile(a.layers, dtype=np.float32).reshape(-1, nl, *hs.shape[2:]) # [T, layers, hc, dim]+ T = min(T, len(eh))+ print("relative error of each layer's residual stream (rows: layers, columns: positions)")+ print(" " + " ".join("%6d" % t for t in range(T)))+ for li in range(nl):+ row = []+ for t in range(T):+ x, y = hs[li, t].astype(np.float64), eh[t, li].astype(np.float64)+ row.append(np.linalg.norm(x - y) / np.linalg.norm(x))+ print("L%-3d " % li + " ".join("%6.4f" % v for v in row))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/download-3.5bpw.json b/tools/dsv41/download-3.5bpw.jsonnew file mode 100644index 000000000..f9f45dcc2--- /dev/null+++ b/tools/dsv41/download-3.5bpw.json@@ -0,0 +1,239 @@+{+ "_": "DeepSeek-V4.1-Flash-1017 at 3.5 bits for maya-ds4 (maya.py's DeepSeek setup): every file pinned to a revision, with its size and sha256. The EXL3 quant's repo has no Engram tables: they come from DeepSeek's base repo (they changed in the 1017 release - the older V4.1's must not be used). The repo's assets/ and encoding/ folders (its README's picture, DeepSeek's prompt-format reference) are not needed and not listed.",+ "name": "DeepSeek-V4.1-Flash-1017",+ "repo": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-3.5bpw",+ "revision": "04a492fc3b4d25ddbab8f2d2f07f235ec77c9010",+ "license": "MIT",+ "files": {+ ".gitattributes": [+ 1697,+ "5c43150d6197107ff8d2b87f76f9f88b7a1a6ab1af968d7828ab33a62e9d06f4"+ ],+ "LICENSE": [+ 1084,+ "f2c6c602815669d292889e5be8c802f2ed950653b77999b1584e8e6aed25d040"+ ],+ "README.md": [+ 12779,+ "3dfe8dd8e4861a3e2a95d72962d2c323fef9d663e829f1c1c134893ff23964d4"+ ],+ "chat_template.jinja": [+ 34813,+ "2ed8bc3372c9c68577f232229e78ebfe37a53ae7964694d845533e485ed738e8"+ ],+ "config.json": [+ 3628,+ "2cbc57cd18428f62aba490870ceda44d2b0639649c05754d6040a5be8e976cf4"+ ],+ "exl3_k_map.json": [+ 1323,+ "efc15305bfe0cc0a728d89943c542bd885fc6cbdc8863f892ae9a8330e5c172c"+ ],+ "model-00001-of-00044.safetensors": [+ 1323827293,+ "ba5ccb4d39882d08800e856150bce42b23584f3d9943520ae4cff5647d5a3045"+ ],+ "model-00002-of-00044.safetensors": [+ 7060206910,+ "012504ba47677b9bb8727bb89cb8c6327a066ef10d2ade179f7ffd5c8151ff84"+ ],+ "model-00003-of-00044.safetensors": [+ 6942014187,+ "9f9d5228106bd5c8064d672f5ae24cce6005be8c2cf33592ef0478655c9e9b80"+ ],+ "model-00004-of-00044.safetensors": [+ 6950309596,+ "8cae1bf802a2e10c59e1f3e057b58f183a0f64e6bf4a1c76107168a9cbb9d532"+ ],+ "model-00005-of-00044.safetensors": [+ 6942014187,+ "b583b45c6a5848dc33a7219d2c34149663779059b11b1bba5a9e76951b333af7"+ ],+ "model-00006-of-00044.safetensors": [+ 6942014187,+ "c4d2e5290f5253262adb8b83b91f37406f24d0a1ce14c7c6844734aa2cbe064a"+ ],+ "model-00007-of-00044.safetensors": [+ 6942014187,+ "65a639c07a74c59da458298adc99bca87036f8316f9c09ac409e2c9b68d414ba"+ ],+ "model-00008-of-00044.safetensors": [+ 6942014187,+ "90f9448977e45f2db5859ca7c9d33ea34a6cdaffc4c28c0d8bb030938db4d614"+ ],+ "model-00009-of-00044.safetensors": [+ 6942014187,+ "06cd3e7672a90c4edfb9ff98b77d11261e39a21558af314f2b235ea5b90b7518"+ ],+ "model-00010-of-00044.safetensors": [+ 6950309596,+ "286c1c0597f29d154391d9f8252cabc560a89be68b50c1f227d0b078200336b2"+ ],+ "model-00011-of-00044.safetensors": [+ 6942014187,+ "b644d168923b319816185fc99b77478db7698067e5e8c67fae619f56a6b18405"+ ],+ "model-00012-of-00044.safetensors": [+ 5243325245,+ "e8e278e5d6a44c81d6fe16c9231aa2cb3823cb8d26e55bdb491345f4e0797f28"+ ],+ "model-00013-of-00044.safetensors": [+ 5243325245,+ "39e25072f8f8de381b3176cb56eb0ec7caa8ab70e85956b57d7f9452291cd386"+ ],+ "model-00014-of-00044.safetensors": [+ 5243325245,+ "3e6e26fad54c7cbc90fbda7209a2f389ff73e58160fb60326bcaa5635e52d811"+ ],+ "model-00015-of-00044.safetensors": [+ 5361517974,+ "e34e397fb634b06a8558643d6b621a2b560b68c0766aac9c0252451ea3daffe2"+ ],+ "model-00016-of-00044.safetensors": [+ 5251620673,+ "db9782ab6346e2dbcf259b79289e6dbc6b4fe183c4936bc1c1c892e2604fd6c7"+ ],+ "model-00017-of-00044.safetensors": [+ 5243325245,+ "711c747176c8494d0cef290e98de5026658bd79a3ef51ed4d43ce403acbfa481"+ ],+ "model-00018-of-00044.safetensors": [+ 5243325245,+ "111ce8d8496d6265098337f5b5ff79e71a31ba340eb2b682d0042dbca4dc954a"+ ],+ "model-00019-of-00044.safetensors": [+ 5243325245,+ "6feedcd839b5593b275761107593ad9f9a0244098dffed66ae6f88ead8844c0d"+ ],+ "model-00020-of-00044.safetensors": [+ 5243325245,+ "bf22fe666d473899b7d6f7bfa1709ad3cffe93f69d39a3da272eac854b39a882"+ ],+ "model-00021-of-00044.safetensors": [+ 5243325245,+ "afef0df9f451e9392cbf4704c9f8ee9183695e543b669d811b6eca88c21a7fe2"+ ],+ "model-00022-of-00044.safetensors": [+ 5249642888,+ "987c51d505f4b49442481ad8596256d846067e8c1fa8edb6c90bd6fc3f85d454"+ ],+ "model-00023-of-00044.safetensors": [+ 5243325245,+ "38cfb33a3a47483a1b1f7b3eebd22ffc9087437d45c8561062a72a8798fa48ef"+ ],+ "model-00024-of-00044.safetensors": [+ 5243325245,+ "aec09849c4916f7254129324bb6aa73eb3685448709534f66fbb31009e0edaf1"+ ],+ "model-00025-of-00044.safetensors": [+ 5243325245,+ "6e0730da79761933690a243e2dd7dd8ecada43c95d9ec96fcc3a520b9f3194a2"+ ],+ "model-00026-of-00044.safetensors": [+ 5247596381,+ "d9a15b99de6190d10099c62383e283c32ef42fdab8ec37ebf355448669eeb027"+ ],+ "model-00027-of-00044.safetensors": [+ 5243325245,+ "c673fa72f6aca977106267afb63f218f241f49a57eb833311ec8affe16009560"+ ],+ "model-00028-of-00044.safetensors": [+ 5243325245,+ "e54fe28e42bd8eec92fa07ac0e4be0850149b20b74e15d7af37d5314a9842deb"+ ],+ "model-00029-of-00044.safetensors": [+ 5243325245,+ "f69616591acdc45b7760db5ef9fab39dfb66558a908778aab230bd8c0d9faa3d"+ ],+ "model-00030-of-00044.safetensors": [+ 5247596381,+ "ac4862535de5828c521e7e5e13906f993395940693cb66fbd323ce9360cd1917"+ ],+ "model-00031-of-00044.safetensors": [+ 5243325245,+ "0beae804fe362965413ba7d26c9c902cf44dd77824838239878332c95170dc7b"+ ],+ "model-00032-of-00044.safetensors": [+ 6942018869,+ "48df9384d595f94b0093b9182b63b4c48216c3c1e486f7aa43a894e93521785d"+ ],+ "model-00033-of-00044.safetensors": [+ 6942018869,+ "a2cd7f6f24043d39ad247a24dd642ce78cb75a2f332fd17779aa012817354c28"+ ],+ "model-00034-of-00044.safetensors": [+ 6946290005,+ "e2f1620bc902230c92361b8689d70f7a72e88283da24a8202561782390c91f70"+ ],+ "model-00035-of-00044.safetensors": [+ 6942018869,+ "c9fc5076793da76174d3d92ec780fedc73cd2f31a79d26a3ef6f26f8f23c0f8a"+ ],+ "model-00036-of-00044.safetensors": [+ 6942018869,+ "905176e9573296c98eb4636c8c1c1559291ac6bdf04c7be5ec97f11eddd30a52"+ ],+ "model-00037-of-00044.safetensors": [+ 6942018869,+ "3f146f50cd59ce9d656f729260d56e5fce4df9e3fcb625102764800abb74ed0e"+ ],+ "model-00038-of-00044.safetensors": [+ 6946290005,+ "eb83d2598c932d5f367606615a9a92e9b45f958e928850eee45ce53bb01db05a"+ ],+ "model-00039-of-00044.safetensors": [+ 6942018869,+ "be69239f0303332cabbf3e0d390d32a45e165ebb8bc514aa186896e792bd17bc"+ ],+ "model-00040-of-00044.safetensors": [+ 6942018869,+ "4ad6d074dcdec1da6425abd105ec105bdcfd594ea7847396cb6d949e86c34df1"+ ],+ "model-00041-of-00044.safetensors": [+ 6942029194,+ "2900611e1b5e83d8ebeefbd4e0956ddb83356e360edde30f0fd076f48ac86d38"+ ],+ "model-00042-of-00044.safetensors": [+ 4086008391,+ "f84c4f2f383ec95f46beef3acc71b73809c9e0b7d54d483db27524ecbe70bea0"+ ],+ "model-00043-of-00044.safetensors": [+ 7329857517,+ "d673275ee4833a78ba76e3a537c50290d0e0ed419b35f68c598b23168f081ee4"+ ],+ "model-00044-of-00044.safetensors": [+ 833651138,+ "4ddb11bc29f6728e95258e2b52faeb91471d3ae145b8f8bfa3e267dd47d3ce19"+ ],+ "model.safetensors.index.json": [+ 15504210,+ "002568f81f402b357a92e2601e6eb664216ab8b27f02c6053d4233a9fdbc72e4"+ ],+ "quantization_config.json": [+ 54653015,+ "493549dee7a0c686cef48bd298bf4ec4f89f4445da2784123726f41854484c5f"+ ],+ "tokenizer.json": [+ 6367257,+ "c90dfa01249db1be4245780a052ede752e1361c612ac6d08e2bdada7d599476b"+ ],+ "tokenizer_config.json": [+ 801,+ "6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547"+ ]+ },+ "engram": {+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",+ "files": {+ "model-00047-of-00048.safetensors": [+ 101535150936,+ "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5"+ ],+ "model-00048-of-00048.safetensors": [+ 101537926640,+ "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd"+ ]+ }+ }+}\ No newline at end of filediff --git a/tools/dsv41/download.json b/tools/dsv41/download.jsonnew file mode 100644index 000000000..7af42fc54--- /dev/null+++ b/tools/dsv41/download.json@@ -0,0 +1,231 @@+{+ "_": "DeepSeek-V4.1-Flash-1017 for maya-ds4 (maya.py's DeepSeek setup): every file pinned to a revision, with its size and sha256. The EXL3 quant's repo has no Engram tables: they come from DeepSeek's base repo (they changed in the 1017 release - the older V4.1's must not be used). The repo's assets/ and encoding/ folders (its README's picture, DeepSeek's prompt-format reference) are not needed and not listed.",+ "name": "DeepSeek-V4.1-Flash-1017",+ "repo": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw",+ "revision": "c178a01284fc5c0e859d8668cffaac81ca8f4042",+ "license": "MIT",+ "files": {+ ".gitattributes": [+ 1697,+ "5c43150d6197107ff8d2b87f76f9f88b7a1a6ab1af968d7828ab33a62e9d06f4"+ ],+ "LICENSE": [+ 1084,+ "f2c6c602815669d292889e5be8c802f2ed950653b77999b1584e8e6aed25d040"+ ],+ "README.md": [+ 11290,+ "80e9744ad84e77c5668b895f61086712f494da946212a5a9b9da334ee4a03cc3"+ ],+ "config.json": [+ 3628,+ "eb48f2216645b36f400036fa2d67d8ad6b48233b0f4a1bcb7e99fa6450dad253"+ ],+ "exl3_k_map.json": [+ 1409,+ "4359af687bdce7fccd4fc74a57c2f9154b42053447cdc6e0d1768e68fffbec0f"+ ],+ "model-00001-of-00043.safetensors": [+ 1323827293,+ "ba5ccb4d39882d08800e856150bce42b23584f3d9943520ae4cff5647d5a3045"+ ],+ "model-00002-of-00043.safetensors": [+ 5322748742,+ "11ed8a0cb267169a5f3ea0ceebb24ab09fb5b84b98e4d4c86dc2f06f1f5e0d40"+ ],+ "model-00003-of-00043.safetensors": [+ 5223069939,+ "1de02ee6b4d909eb5936e08e7c3825cbf2fa65693c5674deade2e70bcb773462"+ ],+ "model-00004-of-00043.safetensors": [+ 5230054628,+ "8e68a9e396d958d6645378b44ef3419a4c0b743da0e92b354d2163a4c8bf85ce"+ ],+ "model-00005-of-00043.safetensors": [+ 5223069939,+ "c14a66ba26f57e9577e1a656764b3e5828e09c7630ad7404b80c6e265d03a37d"+ ],+ "model-00006-of-00043.safetensors": [+ 5223069939,+ "2866602c20b535eec3bb1a737f62d5d6d2dfecbf362977d3a4f5f8e701580b6d"+ ],+ "model-00007-of-00043.safetensors": [+ 5223069939,+ "4d12814cd25cf07fe9c6791f0050a8afbc1038bae02fb201243f9c56de8171cd"+ ],+ "model-00008-of-00043.safetensors": [+ 5223069939,+ "cb88f50d0ddd81b98db00a6475c356511c3c21f843b2ccd9845fb4017c3f5d15"+ ],+ "model-00009-of-00043.safetensors": [+ 5223069939,+ "dbebd82073a27f79dd1353d3c309255f46e577eb1c2f9d8a10a52bc38f6d8bc2"+ ],+ "model-00010-of-00043.safetensors": [+ 5230054628,+ "8cde4e4124614502c2950e9f3a643027eceeeedbdf3c54cf786dbf125d3d536e"+ ],+ "model-00011-of-00043.safetensors": [+ 5223069939,+ "c96372d73271d2c59e5a959bb81551aaacb2abe8445f5b9ea9391e2b4a033601"+ ],+ "model-00012-of-00043.safetensors": [+ 5223074621,+ "337345b1ca174aebcff619bbc2c2a6e33881a42ad8f638956e69b5de4fe21d41"+ ],+ "model-00013-of-00043.safetensors": [+ 5218650941,+ "042b4274c48e55d09bfe51ac099915f8bc3bb0c7be9b0f9d5e75a9f5c2045497"+ ],+ "model-00014-of-00043.safetensors": [+ 5218650941,+ "bc27a27e60bebd88ad36f4896aa027911bc5f7fbb30ad9a25f95a59a6374d957"+ ],+ "model-00015-of-00043.safetensors": [+ 5297522070,+ "74d3f0d12e32a658d8f6c5f9a104826e473c53299267519b46d594619756ae66"+ ],+ "model-00016-of-00043.safetensors": [+ 5225635649,+ "0a1b0b3ecb817f448d9fb34ebf0757d4b00042220515ecb13f1b86908cb092f3"+ ],+ "model-00017-of-00043.safetensors": [+ 5218650941,+ "e4b4278a51ce9daf61290d4caba5b953f5a243832ffb5c8d29a0c259807facea"+ ],+ "model-00018-of-00043.safetensors": [+ 5218650941,+ "07e9d048f9cefb94b9a30f41e39b9390b6bf54a763c0b7e1dcd5d81144fc0849"+ ],+ "model-00019-of-00043.safetensors": [+ 5218650941,+ "2c6116ee6f584af8c4027cee8f5069d0d51a4e9b487dd73027127d29f4caf274"+ ],+ "model-00020-of-00043.safetensors": [+ 3519956829,+ "817d948752825a739073b95bc6171a35e4677c095ac617bd6772560ae0e4b5e4"+ ],+ "model-00021-of-00043.safetensors": [+ 3519956829,+ "4bdda16b5a00ac5ee0e39bc81271af6155caacb7c03928876b07344e20a4ba5f"+ ],+ "model-00022-of-00043.safetensors": [+ 3525291432,+ "00afb7bd831cd50added9b809cc2e9cec57ab44963f152f8bc42cd6d22f0a4e4"+ ],+ "model-00023-of-00043.safetensors": [+ 3519956829,+ "28ba9d401e2cd556d0145b31c03ed341f009e2849162e6d9f68438a067d74a59"+ ],+ "model-00024-of-00043.safetensors": [+ 3519956829,+ "1240d8744902914287da94cd8e2a5d9f17e7c39167bfab20c737a34d7605e50c"+ ],+ "model-00025-of-00043.safetensors": [+ 5218650941,+ "a053c3dc54c4ecfe295cfa4bd7fb44f1054ade1d596e46496f6791c87ffb60df"+ ],+ "model-00026-of-00043.safetensors": [+ 5222266717,+ "028c44abbea066c94da7ed0e64f07f8d36ae925a4ff437ef999f5a2b5ec6c9d4"+ ],+ "model-00027-of-00043.safetensors": [+ 5218650941,+ "69cbb8f2f00ddcb7a83b04d34ca9151c61f3319ec742d0ac5f06abf295a91eef"+ ],+ "model-00028-of-00043.safetensors": [+ 5218650941,+ "451c69700eb30a9e47ec80fe219aa537985c6afa3f7391db5699fba50d93e05b"+ ],+ "model-00029-of-00043.safetensors": [+ 5218650941,+ "906a9525cde8176d18066d27ae9d457ff55ee3134438dadea4ee63af8dce45ad"+ ],+ "model-00030-of-00043.safetensors": [+ 5222266717,+ "420ad929c1efa3f90ba4a39d28159d9a29175089cd9bd4fe32f25c6fb7e473f6"+ ],+ "model-00031-of-00043.safetensors": [+ 5220125501,+ "0312eb9c04dbe5f538efd94a21d1f76266a8cac9ea01250084aa6513657fc5df"+ ],+ "model-00032-of-00043.safetensors": [+ 5223074621,+ "4653d193e28120319fe5b3508ffed537d23f57b8d31b20f3f69371fd569e59b2"+ ],+ "model-00033-of-00043.safetensors": [+ 5223074621,+ "3a6bfc60065324e606082ae8ad68ff403285b24e7a720b3b2274e516a247252a"+ ],+ "model-00034-of-00043.safetensors": [+ 5226690397,+ "b1825ddd7016d3cefea0f13ec78aff33637c68e0eb721cdb12f83104c03b3760"+ ],+ "model-00035-of-00043.safetensors": [+ 5223074621,+ "9b513ca99a0e8403d172df623a4c0fe7ecaaa51e09015d9574efa90bd6414a32"+ ],+ "model-00036-of-00043.safetensors": [+ 5223074621,+ "206ca962082e2c4de968ba4cc82cc1837dccb0b8960f493e04891b281bbfe497"+ ],+ "model-00037-of-00043.safetensors": [+ 5223074621,+ "6ed69fdcd3bd32762ddefae671c7fce7f9b5a212cdbe1aaee64adaa5ef0eef2c"+ ],+ "model-00038-of-00043.safetensors": [+ 5226690397,+ "24a61b540656dc210b3bc3215b1713f0b4bee8b724176a4f5968e24b9540035c"+ ],+ "model-00039-of-00043.safetensors": [+ 5223074621,+ "29e5b40a97edcb91ed6a382b39bbb4d16ce40de2d1ddce2f75a8fd75e0ad609f"+ ],+ "model-00040-of-00043.safetensors": [+ 5223074621,+ "5c850a3950ce68a93a24336ccc417ac660fdbdb84bcc494aa6d57ea5c28246be"+ ],+ "model-00041-of-00043.safetensors": [+ 5719789284,+ "5a6f5791d3f5f9635e58db5a693f68c64c6c9e14357e8386a21e5fd232af032d"+ ],+ "model-00042-of-00043.safetensors": [+ 4854649316,+ "2f80f4a44e4608c6da139eccf2483122ac88a240046872965d546b4fa26ded06"+ ],+ "model-00043-of-00043.safetensors": [+ 3500776125,+ "c5b94d519d323921baacb34b6111e518ce2c36309d06c80fe4e42e3aa9662bc5"+ ],+ "model.safetensors.index.json": [+ 15504210,+ "778baeaf9fbcd351800536dc090ff1edddb72d102472b08de0edd9a52cb96656"+ ],+ "quantization_config.json": [+ 54653012,+ "696cdad521cdebea0642a3c22d01c9152dca4e5ccbf209b5e7f2b808f359bd27"+ ],+ "tokenizer.json": [+ 6367257,+ "c90dfa01249db1be4245780a052ede752e1361c612ac6d08e2bdada7d599476b"+ ],+ "tokenizer_config.json": [+ 801,+ "6ac8c8dc065ed118161d02dd532749ae3f52c243deac27872134fae2f50d8547"+ ]+ },+ "engram": {+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",+ "files": {+ "model-00047-of-00048.safetensors": [+ 101535150936,+ "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5"+ ],+ "model-00048-of-00048.safetensors": [+ 101537926640,+ "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd"+ ]+ }+ }+}diff --git a/tools/dsv41/download_gguf.json b/tools/dsv41/download_gguf.jsonnew file mode 100644index 000000000..72b911fc5--- /dev/null+++ b/tools/dsv41/download_gguf.json@@ -0,0 +1,337 @@+{+ "_": "DeepSeek-V4.1-Flash-1017 as unsloth's GGUFs for maya-ds4 (maya.py's DeepSeek setup, tools/ds4/pack_gguf.py): dense weights BF16 (UD-Q8_K_XL) or Q8_0 (UD-Q4_K_XL), routed experts MXFP4, the Engram tables MXFP8 - every file pinned to a revision, with its size and sha256. Neither has the DSpark drafter or the vision tower (\"drafter\" and \"vision\" below: from DeepSeek's own checkpoint).",+ "name": "DeepSeek-V4.1-Flash-1017",+ "repo": "unsloth/DeepSeek-V4.1-Flash-1017-GGUF",+ "revision": "c991b28e7c109eed10a55075ddb46f830b5b3059",+ "license": "MIT",+ "quants": {+ "UD-Q8_K_XL": {+ "files": {+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00009.gguf": [+ 5780589,+ "cc15d470afcb030558bbc4d8e2ccd840182e2e95de1a95198138efe63a5fd99f"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00002-of-00009.gguf": [+ 50536121984,+ "9cb2047676bd20e448be2377b480080c97fb38fae117f4846e07de2c65bb8522"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00003-of-00009.gguf": [+ 50536121984,+ "17d398dd327e7ff1991b382677814db75d970b074661757564b6f2ac3d0c8d95"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00004-of-00009.gguf": [+ 50536122016,+ "7ea8da1143b9a6bf1424a361eb1b0d6f66c7ce79eac9711162f212cc4f39c319"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00005-of-00009.gguf": [+ 50536122016,+ "1952da19a841176b46f9681c99ea57c674f8a8861e16d76fb35cec4befe039b2"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00006-of-00009.gguf": [+ 50536122016,+ "d6711781d8effab79b410af9c40920bffcc0c28976e509bc1227ff0c6669c33a"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00007-of-00009.gguf": [+ 36097230048,+ "2ebc8d198d03a14865e8a85df637285ceb5dd79b1ffc78424bd3c3e06df57823"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00009.gguf": [+ 202758032592,+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00009.gguf": [+ 16835109952,+ "df33722d1dcf5ffe0347ed8406f7acb3b18061f4ded2c13c2b0a184b8d41c9ea"+ ]+ },+ "bytes": 508376763197,+ "earlier": [+ {+ "_": "unsloth's layout before it re-sharded the GGUFs on 2026-10-19 (10 shards, the Engram tables in 00008 and 00009): used as it is when the folder holds it whole - nothing downloaded again",+ "revision": "dc788bdb683b0af7a6e5315a7d98e9d082faf5e5",+ "files": {+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00001-of-00010.gguf": [+ 5781234,+ "e88691700c4778890cfa845bc21242cf101faa014c8a44d1315402e3f64e701a"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00002-of-00010.gguf": [+ 48129640000,+ "8b9e24ae7322af960c6cceb3cc79623ac79c173fba860da2b10fe5de04e763e2"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00003-of-00010.gguf": [+ 48129640000,+ "9b5047e0f84d64f673d638a6d85282bae16b53f39cf12559c4b8605b127baeca"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00004-of-00010.gguf": [+ 48129640000,+ "83a630d0dd9f2dc6e1d1223b265e78523a51ba6483ab70e3ae3f188978df2491"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00005-of-00010.gguf": [+ 48129640000,+ "90ecd6d773dd0eab5ce1a7bdff626617f8b59360ae53a2cd5d763bde0088b2e8"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00006-of-00010.gguf": [+ 48129640000,+ "e688bfa9a31f00ac33ac7cd739fbc0a903ae0d1601e85c485c23147c57a282fd"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00007-of-00010.gguf": [+ 48129640000,+ "daf02b75c0ad3562c8ccc505fcd94ce08584f967f485b9e9e3ebdbfcafcaf286"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00010.gguf": [+ 101377628544,+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00010.gguf": [+ 101380404256,+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00010-of-00010.gguf": [+ 16835109952,+ "93749d36aa1d2a73dba2bbaf903cc366e694769bc16c216ec2e0dff846e74cac"+ ]+ }+ }+ ]+ },+ "UD-Q4_K_XL": {+ "files": {+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00001-of-00009.gguf": [+ 5780589,+ "cc15d470afcb030558bbc4d8e2ccd840182e2e95de1a95198138efe63a5fd99f"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00002-of-00009.gguf": [+ 48129640000,+ "17df217d8fc26a39f21ea40805dc6ab788c7ce8623a59a95c431051df7eafc94"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00003-of-00009.gguf": [+ 48129640000,+ "0c8065f620a049cb437024c54ff12c314c94603a1cc683f5191be4689d6d6d3b"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00004-of-00009.gguf": [+ 48129640000,+ "3bd8b2ebf8f51d8708a73ae0895f8d4acc7e4c983ba4e89d038492b203cc89ac"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00005-of-00009.gguf": [+ 48129640000,+ "929054a6bbf3701765779b58c37988b6c6284a3b8413901858b59c0e95f37cc3"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00006-of-00009.gguf": [+ 48129640000,+ "49d3de04bd6a09de786c9ef7db900eae02ee9fd391573f91a923013d7d5b24d3"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00007-of-00009.gguf": [+ 48129640000,+ "f18feae5878dd680f1ea87ecd1d84f4fb098480298b25a3162dcbdab8dc2ca5b"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00008-of-00009.gguf": [+ 202758032592,+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00009-of-00009.gguf": [+ 9165923392,+ "c21ad6105a780bc2f44eb7dd41f9b1a1efb4e8ec87618f60b733bc2652bd2ffe"+ ]+ },+ "bytes": 500707576573,+ "earlier": [+ {+ "_": "unsloth's layout before it re-sharded the GGUFs on 2026-10-19 (10 shards, the Engram tables in 00008 and 00009): used as it is when the folder holds it whole - nothing downloaded again",+ "revision": "dc788bdb683b0af7a6e5315a7d98e9d082faf5e5",+ "files": {+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00001-of-00010.gguf": [+ 5781234,+ "e88691700c4778890cfa845bc21242cf101faa014c8a44d1315402e3f64e701a"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00002-of-00010.gguf": [+ 48129640000,+ "8b9e24ae7322af960c6cceb3cc79623ac79c173fba860da2b10fe5de04e763e2"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00003-of-00010.gguf": [+ 48129640000,+ "9b5047e0f84d64f673d638a6d85282bae16b53f39cf12559c4b8605b127baeca"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00004-of-00010.gguf": [+ 48129640000,+ "83a630d0dd9f2dc6e1d1223b265e78523a51ba6483ab70e3ae3f188978df2491"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00005-of-00010.gguf": [+ 48129640000,+ "90ecd6d773dd0eab5ce1a7bdff626617f8b59360ae53a2cd5d763bde0088b2e8"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00006-of-00010.gguf": [+ 48129640000,+ "e688bfa9a31f00ac33ac7cd739fbc0a903ae0d1601e85c485c23147c57a282fd"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00007-of-00010.gguf": [+ 48129640000,+ "daf02b75c0ad3562c8ccc505fcd94ce08584f967f485b9e9e3ebdbfcafcaf286"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00008-of-00010.gguf": [+ 101377628544,+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00009-of-00010.gguf": [+ 101380404256,+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"+ ],+ "UD-Q4_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q4_K_XL-00010-of-00010.gguf": [+ 9165923392,+ "82a3958d47e2c220f49d7235256a54c3a93a08f9c1bbf90c7a0c552a64c90d9a"+ ]+ }+ }+ ]+ }+ },+ "maya": {+ "_": "Maya's quants of DeepSeek-V4.1-Flash-1017 (tools/maya_quant: per-expert statistics from the model itself, error-feedback rounding of the experts), each one GGUF split with everything in it - the Engram tables too, in Q6_K (made from DeepSeek's released weights). The files are listed here once they are uploaded, with the commit they were uploaded at.",+ "repo": "peasantsmith/DeepSeek-V4.1-Flash-1017-Maya-GGUF",+ "revision": "276e0badcfb4a6d8f6c2b9d4da1b468cd339c221",+ "quants": {+ "Maya-S": {+ "about": "experts IQ2_XXS / IQ3_XXS, the first and last four layers IQ2_S / IQ3_S, dense weights Q8_0, the Engram tables Q6_K",+ "files": {+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00001-of-00006.gguf": [+ 47887697312,+ "1f3437f57042e69f94c141ffbcba0d5610dab63feb07244fb468be4940d6763f"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00002-of-00006.gguf": [+ 47669578400,+ "66501fee8702ffd54e6f065323ab54bb3150d20b10b01922c589e7462c7b5ba6"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00003-of-00006.gguf": [+ 47103347360,+ "3195b23f33337c46a07c48583946d99147c3635412bc4adc6acdc4cf42bf30ad"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00004-of-00006.gguf": [+ 35532532032,+ "d60dba35fac9601cd5c6026439dd980c457a5bb1c009192ea539b220108ea0db"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00005-of-00006.gguf": [+ 80641295472,+ "c98d46f96a25d17c4c54da5a60782c130255c364a22e621ea28d5c98a692ceb2"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00006-of-00006.gguf": [+ 80643503412,+ "98d65ef4f36e0b193935d44c3586241ca09b1b47e90f118e85332cfdb5860263"+ ]+ },+ "earlier": [+ {+ "_": "v1.1.16's Maya-S: 4 shards without the Engram tables, which came from unsloth's MXFP8 shard of them beside it in <...-GGUF>/Engram (\"engram\": either of its layouts) - used as they are when both are whole, nothing downloaded again",+ "revision": "26c6aeaf5ecce88fe47103f780279bcc6f1f0d9b",+ "files": {+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00001-of-00004.gguf": [+ 47887697280,+ "b33d91333b463414fd91fbeb14a0939e3064a1abd820fa06310992da611eeb2e"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00002-of-00004.gguf": [+ 47669578400,+ "8198c1a5b9e2f964affacc1141627e2f0715711571c1fe04cb81702f232c5c39"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00003-of-00004.gguf": [+ 47103347360,+ "96b9b5e22ed60825794c377301cf2671a7a2ec463c7580b57e101f222c8ee516"+ ],+ "Maya-S/DeepSeek-V4.1-Flash-1017-Maya-S-IQ2_XXS-00004-of-00004.gguf": [+ 35532532032,+ "506dd95a2ea4f17c933534d730ec72b2ef1b82eec4847770f4f23c38344f96ca"+ ]+ },+ "engram": [+ {+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00009.gguf": [+ 202758032592,+ "12df293355efbd104b088a77e13ee22410fb1ec3f7ab3863947f09896e64bec7"+ ]+ },+ {+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00008-of-00010.gguf": [+ 101377628544,+ "b9a5d8344f60c87bf8e8b2f56f89074e22c3a148342455129fd2a6f65bc6a612"+ ],+ "UD-Q8_K_XL/DeepSeek-V4.1-Flash-1017-UD-Q8_K_XL-00009-of-00010.gguf": [+ 101380404256,+ "0dc8cda974060d492b147a0e3fa074a892bddb0c8b8d6322dc2367c6c6a72fbf"+ ]+ }+ ]+ }+ ]+ },+ "Maya-M": {+ "about": "experts IQ3_S, dense weights Q8_0",+ "files": {}+ }+ },+ "drafter": {+ "_": "The DSpark drafter Maya ships: DeepSeek's FP4 experts as released, its FP8 dense weights in Q8_0 (the engine's formats: used as it is, beside the quants' folders as dspark/); the shards + drafter_pack.py path (\"drafter\" below) is the fallback while this is not pinned.",+ "files": {+ "dspark/dspark.bin": [+ 7967616768,+ "42df65159f980a9189d80597e42b30012c7b10e0dd385a47051f3b2fd659bb7b"+ ],+ "dspark/dspark.json": [+ 96965,+ "639c3144bca8cfe4db59057047dca40411f0cc732796e2a9321feb98fee2295f"+ ]+ }+ }+ },+ "drafter": {+ "_": "DeepSeek's DSpark drafter (the speculative decoder the GGUFs leave out): the three shards of DeepSeek's own checkpoint that hold only its blocks (mtp.0-2), at the revision the Engram tables come from; tools/ds4/drafter_pack.py makes them the GGUF path's formats (dspark.bin, beside the quants' folders).",+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",+ "files": {+ "model-00044-of-00048.safetensors": [+ 2652728736,+ "3e8ac315c2bcca3e83c5e6e25c5758a75c03732063a6ff0c1b0e24b9ff49f6e6"+ ],+ "model-00045-of-00048.safetensors": [+ 2573998176,+ "9c73b27c4343236790008979d5e0b565f687aa64440031b9e096fed1ab8039c5"+ ],+ "model-00046-of-00048.safetensors": [+ 2706402896,+ "1dccc19a3741972c661de734572dc7d4b899e6e99bfbdf3ae4e057c707e5b339"+ ]+ }+ },+ "vision": {+ "_": "DeepSeek's vision tower (the GGUFs leave it out; every GGUF of the model shares it, the Maya quants too): shard 1 of DeepSeek's own checkpoint holds only the tower (vision.*) and the aligner (aligner.*), BF16, and config.json their geometry; the picture span's three marker rows sit beside the embedding in shard 2, so only their 30 KB are read from it (\"marks\": an HTTP range request, each row [offset in the file, bytes, sha256]). tools/ds4/vision_pack.py makes them vision.bin + vision.json, beside the quants' folders.",+ "repo": "deepseek-ai/DeepSeek-V4.1-Flash-1017",+ "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170",+ "files": {+ "model-00001-of-00048.safetensors": [+ 970533624,+ "13bd121075f0b68cb7e8e8021a1011d42a81a83f557a2602fd9c07d52fddb0ec"+ ],+ "config.json": [+ 3311,+ "8be45ce0476004a3f529fd896115a4a2e800a129ad2d3ec05b16050f52e21879"+ ]+ },+ "marks": {+ "file": "model-00002-of-00048.safetensors",+ "bytes": 1323858272,+ "rows": {+ "image_start": [+ 1323848032,+ 10240,+ "5f86461a3a0565f77468091e13c0c81db83614295ad004136b987cbc430470fd"+ ],+ "image_newline": [+ 1323837792,+ 10240,+ "50e8ecbca5409f87546c3d9f78b49b1653204cfb2a8f5068dc738cd02b3df69f"+ ],+ "image_end": [+ 1323827552,+ 10240,+ "f55bfbcb7e06217c4822d27cf7a6696966c099f2805dbf47fc1a6d6f56a885b6"+ ]+ }+ }+ }+}diff --git a/tools/dsv41/drafter_pack.py b/tools/dsv41/drafter_pack.pynew file mode 100644index 000000000..0b4394f58--- /dev/null+++ b/tools/dsv41/drafter_pack.py@@ -0,0 +1,216 @@+"""tools/ds4/drafter_pack.py - DeepSeek-V4.1's DSpark drafter (the speculative decoder the GGUFs leave out) for a GGUF+install: its three blocks from DeepSeek's own checkpoint (the shards that hold only them, model-00044..46-of-00048),+in the formats the GGUF path runs, as one file beside them that pack_gguf.py adds to the pack.++ python tools/ds4/drafter_pack.py --dir <folder with the three shards> -> dspark.bin + dspark.json there++Every value is kept exactly: the FP8 (E4M3) matrices with their E8M0 scale a 32 x 32 block become BF16 (an E4M3 value+times a power of two has at most four significant bits); the routed experts (FP4 E2M1 codes two to a byte, the first in+the low nibble, an E8M0 scale a 32) become ggml's MXFP4 blocks (the scale byte, then value j's code in byte j's low+nibble and value j + 16's in its high one); BF16 and F32 tensors are copied. dspark.json lists every tensor (name,+dtype, offset, bytes, shape) under the checkpoint's names, as tensors.txt does.+"""+from __future__ import annotations++import argparse+import json+import pathlib+import re+import struct+import sys++import numpy as np++SHARDS = ("model-00044-of-00048.safetensors", "model-00045-of-00048.safetensors", "model-00046-of-00048.safetensors")+ALIGN = 256+++def e4m3_table() -> np.ndarray:+ t = np.zeros(256, np.float32)+ for c in range(256):+ s, e, m = c >> 7, (c >> 3) & 15, c & 7+ v = np.nan if (e == 15 and m == 7) else (m / 8.0) * 2.0 ** -6 if e == 0 else (1 + m / 8.0) * 2.0 ** (e - 7)+ t[c] = -v if s else v+ return t+++E4M3 = e4m3_table()+++def shard(path: pathlib.Path):+ raw = np.memmap(path, np.uint8, mode="r")+ n = struct.unpack("<Q", bytes(raw[:8]))[0]+ head = json.loads(bytes(raw[8:8 + n]))+ head.pop("__metadata__", None)+ return raw, head, 8 + n+++def data(raw, base: int, e: dict) -> np.ndarray:+ a, b = e["data_offsets"]+ return raw[base + a:base + b]+++def bf16(x: np.ndarray) -> np.ndarray:+ """float32 -> BF16 bits, rounded to nearest even (exact here)."""+ u = x.astype(np.float32).view(np.uint32)+ return ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16)+++def fp8_to_bf16(w: np.ndarray, s: np.ndarray, rows: int, cols: int) -> np.ndarray:+ w = w.reshape(rows, cols)+ scale = np.exp2(s.reshape((rows + 31) // 32, (cols + 31) // 32).astype(np.float32) - 127.0)+ v = E4M3[w] * np.repeat(np.repeat(scale, 32, 0), 32, 1)[:rows, :cols]+ return bf16(v)+++def fp4_to_mxfp4(w: np.ndarray, s: np.ndarray, rows: int, k: int) -> np.ndarray:+ nb = k // 32+ p = w.reshape(rows, nb, 16)+ codes = np.empty((rows, nb, 32), np.uint8)+ codes[..., 0::2] = p & 15+ codes[..., 1::2] = p >> 4+ out = np.empty((rows, nb, 17), np.uint8)+ out[..., 0] = s.reshape(rows, nb)+ out[..., 1:] = codes[..., :16] | (codes[..., 16:] << 4)+ return out+++QTYPES = {"Q8_0": (8, 32, 34), "Q2_K": (10, 256, 84), "Q3_K": (11, 256, 110), "Q4_K": (12, 256, 144),+ "Q5_K": (13, 256, 176), "Q6_K": (14, 256, 210), "IQ2_XXS": (16, 256, 66), "IQ3_XXS": (18, 256, 98),+ "IQ3_S": (21, 256, 110), "IQ2_S": (22, 256, 82), "IQ4_XS": (23, 256, 136)}+DENSE_KEEP = ("markov_head", "confidence_head", "norm", "hc_", "ffn.gate.", "attn_sink")+E2M1 = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)+++def requant(d: pathlib.Path, name: str, experts: str, dense: str, lib: str, jobs: int, out=None) -> int:+ """dspark.bin's tensors again with the routed experts in `experts` and the dense matrices (the ones the engine reads+ as matrices) in `dense`, through ggml's quantizers (libggml-base): dspark-<name>.bin / .json beside it - a smaller+ drafter (pack_gguf.py --drafter dspark-<name>)."""+ import ctypes+ from concurrent.futures import ThreadPoolExecutor+ g = ctypes.CDLL(lib)+ g.ggml_quantize_chunk.restype = ctypes.c_size_t+ g.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,+ ctypes.c_int64, ctypes.c_void_p]+ src = json.loads((d / "dspark.json").read_text())+ raw = np.memmap(d / "dspark.bin", np.uint8, mode="r")++ def f32(dt, b):+ if dt == "BF16":+ return (np.frombuffer(b, np.uint16).astype(np.uint32) << 16).view(np.float32)+ if dt == "MXFP4":+ blk = np.frombuffer(b, np.uint8).reshape(-1, 17)+ v = np.empty((blk.shape[0], 32), np.float32)+ v[:, :16] = E2M1[blk[:, 1:] & 15]+ v[:, 16:] = E2M1[blk[:, 1:] >> 4]+ return (v * np.ldexp(np.float32(1), blk[:, :1].astype(np.int32) - 128)).reshape(-1)+ raise ValueError(dt)++ def quant(t, x, rows, ne0): # (one tensor on the calling thread; the pool runs tensors side by side)+ tid, per, bpb = QTYPES[t]+ q = np.empty(rows * ne0 // per * bpb, np.uint8)+ ones = np.ones(ne0, np.float32)+ g.ggml_quantize_chunk(tid, x.ctypes.data, q.ctypes.data, 0, rows, ne0, ones.ctypes.data)+ return q++ role_re = re.compile(r"(mtp\.\d+\.ffn\.experts\.)\d+\.(w[123])\.weight$")++ def prep(item):+ t, dt, so, nb, shape = item+ b = raw[so:so + nb]+ m = role_re.match(t)+ role = (m.group(1), m.group(2)) if m else None+ want = experts if role else dense+ if (not role and (dt != "BF16" or len(shape) != 2 or shape[1] % 32 or any(k in t for k in DENSE_KEEP))) \+ or want in ("keep", dt):+ return t, dt, np.asarray(b), shape, role+ if shape[1] % QTYPES[want][1]:+ raise SystemExit("%s: rows of %d are not whole %s blocks" % (t, shape[1], want))+ return t, want, quant(want, np.ascontiguousarray(f32(dt, b)), shape[0], shape[1]), shape, role++ for t in {experts, dense} - {"keep"}:+ g.ggml_quantize_init(QTYPES[t][0])+ entries, off, prev = [], 0, None+ o = pathlib.Path(out) if out else d+ o.mkdir(parents=True, exist_ok=True)+ items = list(src["tensors"])+ with (o / ("dspark-%s.bin.part" % name)).open("wb") as f, ThreadPoolExecutor(jobs) as ex:+ for i0 in range(0, len(items), 2 * jobs):+ for t, odt, q, shape, role in ex.map(prep, items[i0:i0 + 2 * jobs]):+ if role is None or role != prev: # (a role's experts back to back: expert e at the first's + e x size)+ pad = (-off) % ALIGN+ f.write(b"\0" * pad)+ off += pad+ f.write(q.tobytes())+ entries.append([t, odt, off, q.nbytes, shape])+ off += q.nbytes+ prev = role+ (o / ("dspark-%s.bin.part" % name)).replace(o / ("dspark-%s.bin" % name))+ (o / ("dspark-%s.json" % name)).write_text(json.dumps({"from": "dspark.bin", "experts": experts, "dense": dense,+ "tensors": entries}))+ print("dspark-%s.bin: %d tensors, %.2f GB (experts %s, dense %s)" % (name, len(entries), off / 1e9, experts, dense))+ return 0+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--dir", required=True, help="the folder with the drafter's three shards")+ ap.add_argument("--requant", metavar="NAME",+ help="from dspark.bin there: dspark-NAME.bin with --experts / --dense types (ggml's quantizers)")+ ap.add_argument("--experts", default="IQ4_XS", help="(--requant) the routed experts' type, or keep")+ ap.add_argument("--dense", default="Q8_0", help="(--requant) the dense matrices' type, or keep")+ ap.add_argument("--lib", default=str(pathlib.Path("~/llama.cpp/build/bin/libggml-base.so").expanduser()),+ help="(--requant) libggml-base")+ ap.add_argument("--jobs", type=int, default=8)+ ap.add_argument("--out", help="(--requant) the folder to write to (default: --dir)")+ a = ap.parse_args()+ d = pathlib.Path(a.dir)+ if a.requant:+ return requant(d, a.requant, a.experts, a.dense, a.lib, a.jobs, a.out)+ entries, off = [], 0+ exp = re.compile(r"(mtp\.\d+\.ffn\.experts\.)(\d+)\.(w[123])\.weight$")++ def order(head):+ """The block's tensors, then each role's experts (w1, w3, w2) back to back in expert order - the engine reads+ expert e of a role at the first one's offset + e x its size, as the GGUF stacks them."""+ plain = sorted(t for t in head if t.startswith("mtp.") and not t.endswith(".scale") and not exp.match(t))+ ex = [exp.match(t) for t in head if exp.match(t)]+ roles = sorted({(m.group(1), m.group(3)) for m in ex}, key=lambda pr: (pr[0], "w1w3w2".index(pr[1])))+ n = {pr: 1 + max(int(m.group(2)) for m in ex if (m.group(1), m.group(3)) == pr) for pr in roles}+ return [(t, False) for t in plain] + [("%s%d.%s.weight" % (p, i, r), i < n[(p, r)] - 1)+ for p, r in roles for i in range(n[(p, r)])]++ with (d / "dspark.bin.part").open("wb") as f:+ for name in SHARDS:+ raw, head, base = shard(d / name)+ for t, packed in order(head):+ e, shape = head[t], list(head[t]["shape"])+ if e["dtype"] == "F8_E4M3":+ s = head[t[:-len("weight")] + "scale"]+ out, dt = fp8_to_bf16(data(raw, base, e), data(raw, base, s), shape[0], shape[1]), "BF16"+ elif e["dtype"] == "I8":+ s = head[t[:-len("weight")] + "scale"]+ shape = [shape[0], shape[1] * 2]+ out, dt = fp4_to_mxfp4(data(raw, base, e), data(raw, base, s), shape[0], shape[1]), "MXFP4"+ elif e["dtype"] in ("BF16", "F32"):+ out, dt = data(raw, base, e), e["dtype"]+ else:+ raise SystemExit(f"{name}: {t} is {e['dtype']}, not a type the drafter's pack covers")+ b = out.tobytes()+ f.write(b)+ entries.append([t, dt, off, len(b), shape])+ off += len(b)+ if packed: # (the next expert of the role follows at once)+ continue+ pad = (-off) % ALIGN+ f.write(b"\0" * pad)+ off += pad+ del raw+ (d / "dspark.bin.part").replace(d / "dspark.bin")+ (d / "dspark.json").write_text(json.dumps({"from": list(SHARDS), "tensors": entries}))+ print(f"dspark.bin: {len(entries)} tensors, {off / 1e9:.2f} GB")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/dspark_ref.py b/tools/dsv41/dspark_ref.pynew file mode 100644index 000000000..e27485813--- /dev/null+++ b/tools/dsv41/dspark_ref.py@@ -0,0 +1,387 @@+"""tools/ds4/dspark_ref.py - DeepSeek-V4.1's DSpark drafter in torch, straight from DeepSeek's checkpoint, against what+the engine's drafter computed (the engine's oracle for the drafter; ref.py / ref_gguf.py cover the trunk only).++ DS4_DSPARK_DUMP=d.bin maya-ds4 --pack <pack> ... --n 64 (the engine: its drafter's inputs and outputs)+ python tools/ds4/dspark_ref.py --dump d.bin --shards <dir with model-00044..46-of-00048> --pack <pack>+ [--rounds 4] [--head]++It follows DeepSeek's inference/model.py (MIT) - DSparkBlock / DSparkAttention / forward_spec: the taps of target+layers 37..39 (the streams' mean entering them) through main_proj / main_norm into every block's window ring (wkv,+kv_norm, rope, FP8 rounding), the block [y, noise x 4] at the positions after the ring's through the three blocks+(Hyper-Connections, window attention over the ring and the whole block, the 128-expert MoE with 3 picks), the final+collapse and, with --head, the trunk's head plus the markov bias (the drafts) - with every weight decoded here from+the checkpoint's own FP8 / FP4 / BF16 tensors (not from drafter_pack.py's dspark.bin), activations F32. Per round+it prints how far the engine's hidden rows and confidences are from it, and with --head whether the drafts agree.+"""+from __future__ import annotations++import argparse+import json+import math+import pathlib+import struct+import sys++import numpy as np+import torch+import torch.nn.functional as F++SHARDS = ("model-00044-of-00048.safetensors", "model-00045-of-00048.safetensors", "model-00046-of-00048.safetensors")+++def e4m3_table() -> np.ndarray:+ t = np.zeros(256, np.float32)+ for c in range(256):+ s, e, m = c >> 7, (c >> 3) & 15, c & 7+ v = np.nan if (e == 15 and m == 7) else (m / 8.0) * 2.0 ** -6 if e == 0 else (1 + m / 8.0) * 2.0 ** (e - 7)+ t[c] = -v if s else v+ return t+++E4M3 = e4m3_table()+E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], np.float32)+++class Ckpt:+ """The drafter's shards: each tensor decoded to F32 on request (FP8 with its 32 x 32 E8M0 block scales; FP4 codes+ two to a byte, the first in the low nibble, an E8M0 scale per 32 along K; BF16; F32)."""++ def __init__(self, d: pathlib.Path):+ self.t = {}+ for name in SHARDS:+ raw = np.memmap(d / name, np.uint8, mode="r")+ n = struct.unpack("<Q", bytes(raw[:8]))[0]+ head = json.loads(bytes(raw[8:8 + n]))+ head.pop("__metadata__", None)+ for k, e in head.items():+ self.t[k] = (raw, 8 + n, e)++ def raw(self, k):+ raw, base, e = self.t[k]+ a, b = e["data_offsets"]+ return raw[base + a:base + b], e["dtype"], e["shape"]++ def get(self, k) -> torch.Tensor:+ w, dt, sh = self.raw(k)+ if dt == "F32":+ v = np.frombuffer(w, np.float32).reshape(sh)+ elif dt == "BF16":+ v = (np.frombuffer(w, np.uint16).astype(np.uint32) << 16).view(np.float32).reshape(sh)+ elif dt == "F8_E4M3":+ s, _, ssh = self.raw(k[:-len("weight")] + "scale")+ r, c = sh+ sc = np.exp2(np.asarray(s, np.float32).reshape(ssh) - 127.0)+ v = E4M3[np.asarray(w).reshape(r, c)] * np.repeat(np.repeat(sc, 32, 0), 32, 1)[:r, :c]+ elif dt == "I8":+ s, _, ssh = self.raw(k[:-len("weight")] + "scale")+ r, c2 = sh+ p = np.asarray(w).reshape(r, c2)+ codes = np.empty((r, c2 * 2), np.uint8)+ codes[:, 0::2] = p & 15+ codes[:, 1::2] = p >> 4+ sc = np.exp2(np.asarray(s, np.float32).reshape(ssh) - 127.0)+ v = E2M1[codes] * np.repeat(sc, 32, 1)+ else:+ raise SystemExit("%s: dtype %s" % (k, dt))+ return torch.from_numpy(np.ascontiguousarray(v, np.float32))+++def fp8_round(x: torch.Tensor, block: int = 32) -> torch.Tensor:+ sh = x.shape+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)+ amax = v.abs().amax(-1, keepdim=True).clamp_min(1e-4)+ s = torch.exp2(torch.ceil(torch.log2(amax / 448.0)))+ y = (v / s).clamp(-448.0, 448.0).to(torch.float8_e4m3fn).float() * s+ return y.reshape(sh)+++def rms(x, w, eps):+ x = x.float()+ y = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps)+ return y * w if w is not None else y+++def freqs(dim, n, base):+ f = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ a = torch.outer(torch.arange(n, dtype=torch.float32), f)+ return torch.polar(torch.ones_like(a), a)+++def rope(x, fc, inverse=False):+ c = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2).contiguous())+ if inverse:+ fc = fc.conj()+ if x.ndim == 3:+ fc = fc[:, None, :]+ return torch.view_as_real(c * fc).flatten(-2)+++def pack_conf(pack: pathlib.Path):+ cfg = {}+ for line in (pack / "ds4.txt").read_text().splitlines():+ k, _, v = line.partition(" ")+ cfg[k] = v+ files, tens = {}, {}+ for line in (pack / "tensors.txt").read_text().splitlines():+ p = line.split()+ if p[0] == "file":+ files[int(p[1])] = p[2]+ elif p[0] == "t":+ tens[p[1]] = (p[2], int(p[3]), int(p[4]), int(p[5]), [int(x) for x in p[6:]])+ return cfg, files, tens+++def gguf_rows(pack, cfg, files, tens, name):+ dt, fi, off, nb, sh = tens[name]+ if dt != "BF16":+ raise SystemExit("%s: %s, the oracle reads a BF16 one" % (name, dt))+ path = pathlib.Path(cfg["model_dir"]) / files[fi]+ return np.memmap(path, np.uint16, mode="r", offset=off, shape=tuple(sh))+++def bf16_rows(m, idx):+ return torch.from_numpy((np.asarray(m[idx]).astype(np.uint32) << 16).view(np.float32))+++class Drafter:+ def __init__(self, ck: Ckpt, cfg: dict):+ self.ck, self.cfg = ck, cfg+ self.D = int(cfg["hidden_size"])+ self.hc = int(cfg["hc_mult"])+ self.eps = float(cfg["rms_norm_eps"])+ self.hc_eps = float(cfg["hc_eps"])+ self.sink_iters = int(cfg["hc_sinkhorn_iters"])+ self.H, self.HD, self.rd = int(cfg["num_attention_heads"]), int(cfg["head_dim"]), int(cfg["qk_rope_head_dim"])+ self.G = int(cfg["o_groups"])+ self.win = int(cfg["sliding_window"])+ self.k = int(cfg["dspark_num_experts_per_tok"])+ self.scale = float(cfg["routed_scaling_factor"])+ self.lim = float(cfg["swiglu_limit"])+ self.fc = freqs(self.rd, 1 << 16, float(cfg["rope_theta"]))+ self.nb = 3+ self.cache = {}+ self.ring = [dict() for _ in range(self.nb)] # position -> main KV row, per block++ def W(self, k):+ if k not in self.cache:+ self.cache[k] = self.ck.get(k)+ return self.cache[k]++ def seed(self, p0, taps): # taps [n, 3 * D]+ main_x = rms(taps @ self.W("mtp.0.main_proj.weight").T, self.W("mtp.0.main_norm.weight"), self.eps)+ n = taps.shape[0]+ fc = self.fc[p0:p0 + n]+ for b in range(self.nb):+ A = "mtp.%d.attn." % b+ kv = rms(main_x @ self.W(A + "wkv.weight").T, self.W(A + "kv_norm.weight"), self.eps)+ kv = torch.cat([kv[:, :-self.rd], rope(kv[:, -self.rd:], fc)], -1)+ kv = fp8_round(kv)+ for i in range(n):+ self.ring[b][p0 + i] = kv[i]++ def hc_mixes(self, x, P):+ hc = self.hc+ fn, scale, base = self.W(P + "_fn"), self.W(P + "_scale"), self.W(P + "_base")+ xf = x.flatten(1).float()+ r = torch.rsqrt(xf.square().mean(-1, keepdim=True) + self.eps)+ mixes = (xf @ fn.T) * r+ pre = torch.sigmoid(mixes[:, :hc] * scale[0] + base[:hc]) + self.hc_eps+ post = 2 * torch.sigmoid(mixes[:, hc:2 * hc] * scale[1] + base[hc:2 * hc])+ comb = (mixes[:, 2 * hc:] * scale[2] + base[2 * hc:]).view(-1, hc, hc)+ eps = self.hc_eps+ comb = comb.softmax(-1) + eps+ comb = comb / (comb.sum(-2, keepdim=True) + eps)+ for _ in range(self.sink_iters - 1):+ comb = comb / (comb.sum(-1, keepdim=True) + eps)+ comb = comb / (comb.sum(-2, keepdim=True) + eps)+ return pre, post, comb++ def attention(self, b, x, p0):+ A = "mtp.%d.attn." % b+ T = x.shape[0]+ H, D, rd = self.H, self.HD, self.rd+ fc = self.fc[p0:p0 + T]+ qr = rms(x @ self.W(A + "wq_a.weight").T, self.W(A + "q_norm.weight"), self.eps)+ q = (qr @ self.W(A + "wq_b.weight").T).view(T, H, D)+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)+ kv = rms(x @ self.W(A + "wkv.weight").T, self.W(A + "kv_norm.weight"), self.eps)+ kv = torch.cat([kv[:, :-rd], rope(kv[:, -rd:], fc)], -1)+ kv = fp8_round(kv)+ ring = [self.ring[b][p] for p in range(max(0, p0 - self.win), p0) if p in self.ring[b]]+ allkv = torch.cat([torch.stack(ring), kv], 0) if ring else kv+ sink = self.W(A + "attn_sink")+ s = torch.einsum("thd,kd->thk", q, allkv) * D ** -0.5+ m = torch.maximum(s.amax(-1), sink[None, :])+ pr = torch.exp(s - m.unsqueeze(-1))+ den = pr.sum(-1) + torch.exp(sink[None, :] - m)+ o = torch.einsum("thk,kd->thd", pr, allkv) / den.unsqueeze(-1)+ o = torch.cat([o[..., :-rd], rope(o[..., -rd:], fc, inverse=True)], -1)+ o = fp8_round(o.reshape(T, self.G, -1))+ wo_a = self.W(A + "wo_a.weight").view(self.G, -1, o.shape[-1])+ oa = torch.einsum("tgd,grd->tgr", o, wo_a)+ return oa.flatten(1) @ self.W(A + "wo_b.weight").T++ def moe(self, b, x):+ M = "mtp.%d.ffn." % b+ T = x.shape[0]+ scores = F.softplus(x.float() @ self.W(M + "gate.weight").T).sqrt()+ idx = (scores + self.W(M + "gate.bias")).topk(self.k, dim=-1)[1]+ wts = scores.gather(1, idx)+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * self.scale+ lim = self.lim+ y = torch.zeros(T, self.D)+ for e in torch.unique(idx).tolist():+ P = M + "experts.%d." % e+ rows, slot = torch.where(idx == e)+ xs = x[rows]+ w1, w3 = self.ck.get(P + "w1.weight"), self.ck.get(P + "w3.weight")+ h = F.silu((xs @ w1.T).clamp(max=lim)) * (xs @ w3.T).clamp(-lim, lim)+ del w1, w3+ w2 = self.ck.get(P + "w2.weight")+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ w2.T)+ del w2+ S = M + "shared_experts."+ h = F.silu((x @ self.W(S + "w1.weight").T).clamp(max=lim)) * (x @ self.W(S + "w3.weight").T).clamp(-lim, lim)+ return y + h @ self.W(S + "w2.weight").T, idx++ def block_forward(self, emb, p0): # emb [T, D] -> the collapsed hidden rows [T, D], each block's stream+ T = emb.shape[0]+ h = emb.unsqueeze(1).repeat(1, self.hc, 1)+ pre = torch.zeros(T, self.hc)+ pre[:, 0] = 1.0+ routes = []+ streams = []+ for b in range(self.nb):+ B = "mtp.%d." % b+ res = h+ a_pre, a_post, a_comb = self.hc_mixes(h, B + "hc_attn")+ x = rms((pre.unsqueeze(-1) * h).sum(1), self.W(B + "attn_norm.weight"), self.eps)+ y = self.attention(b, x, p0)+ h = a_post.unsqueeze(-1) * y.unsqueeze(1) + (a_comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)+ res = h+ f_pre, f_post, f_comb = self.hc_mixes(h, B + "hc_ffn")+ x = rms((a_pre.unsqueeze(-1) * h).sum(1), self.W(B + "ffn_norm.weight"), self.eps)+ y, idx = self.moe(b, x)+ routes.append(idx)+ h = f_post.unsqueeze(-1) * y.unsqueeze(1) + (f_comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)+ pre = f_pre+ streams.append(h)+ return (pre.unsqueeze(-1) * h).sum(1), routes, streams+++def records(path, nb, hc, win, hd):+ """The engine's dump: 'S' seeds (first position, the taps), 'D' draft rounds (the block's position, y, the drafts,+ their confidences, the collapsed hidden rows, then per block the ring it read (BF16) and the stream it left)."""+ with open(path, "rb") as f:+ while True:+ hb = f.read(32)+ if len(hb) < 32:+ return+ kind, a, b, c = struct.unpack("<4q", hb)+ if kind == ord("S"):+ n, w = b, c+ taps = np.frombuffer(f.read(n * w * 4), np.float32).reshape(n, w)+ yield "S", a, taps+ elif kind == ord("D"):+ T = c+ D = int(DIM[0])+ toks = np.frombuffer(f.read(T * 4), np.int32)+ conf = np.frombuffer(f.read(T * 4), np.float32)+ x = np.frombuffer(f.read(T * 4 * D), np.float32).reshape(T, D)+ rings, streams = [], []+ for _ in range(nb):+ r = np.frombuffer(f.read(win * hd * 2), np.uint16).reshape(win, hd)+ rings.append((r.astype(np.uint32) << 16).view(np.float32))+ streams.append(np.frombuffer(f.read(T * hc * D * 4), np.float32).reshape(T, hc, D))+ yield "D", (a, b, T, toks, conf, x, rings, streams), None+ else:+ raise SystemExit("dump: record %d" % kind)+++DIM = [5120]+++def rel(a: torch.Tensor, b: torch.Tensor) -> float:+ return float((a - b).norm() / b.norm().clamp_min(1e-30))+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--dump", required=True)+ ap.add_argument("--shards", required=True, help="the folder with DeepSeek's model-00044..46-of-00048")+ ap.add_argument("--pack", required=True, help="the maya-ds4 pack (ds4.txt, tensors.txt: the embedding, the head)")+ ap.add_argument("--rounds", type=int, default=4, help="draft rounds to check (from --skip)")+ ap.add_argument("--skip", type=int, default=0, help="draft rounds to pass over first")+ ap.add_argument("--head", action="store_true", help="also the drafts: the trunk's head + the markov bias")+ ap.add_argument("--threads", type=int, default=4)+ a = ap.parse_args()+ torch.set_num_threads(a.threads)+ pack = pathlib.Path(a.pack)+ cfg, files, tens = pack_conf(pack)+ DIM[0] = int(cfg["hidden_size"])+ ck = Ckpt(pathlib.Path(a.shards))+ dr = Drafter(ck, cfg)+ emb = gguf_rows(pack, cfg, files, tens, "embed.weight")+ head = gguf_rows(pack, cfg, files, tens, "head.weight") if a.head else None+ noise = int(cfg["dspark_noise_token_id"])+ seen = done = 0+ worst_x = worst_c = 0.0+ agree = total = 0+ for kind, v, taps in records(a.dump, dr.nb, dr.hc, dr.win, dr.HD):+ if kind == "S":+ dr.seed(v, torch.from_numpy(taps.copy()))+ continue+ p0, y, T, toks, conf, xd, rings, streams = v+ seen += 1+ if seen <= a.skip:+ continue+ ex = torch.from_numpy(xd.copy())+ ids = [y] + [noise] * (T - 1)+ xh, routes, ref_streams = dr.block_forward(bf16_rows(emb, ids), p0)+ for b in range(dr.nb): # the ring each block read (the positions the reference holds), the stream it left+ ps = [p for p in range(max(0, p0 - dr.win), p0) if p in dr.ring[b]]+ er = torch.from_numpy(rings[b][[p % dr.win for p in ps]].copy())+ rr = torch.stack([dr.ring[b][p] for p in ps])+ print(" block %d: ring rel err %.4f (%d rows), stream rel err %.4f, routes %s" % (+ b, rel(er, rr), len(ps), rel(torch.from_numpy(streams[b].copy()), ref_streams[b]),+ routes[b].tolist()), flush=True)+ prev = [y] + [int(t) for t in toks[:-1]]+ me = dr.W("mtp.2.markov_head.embed.weight")[prev]+ cw = dr.W("mtp.2.confidence_head.proj.weight")[0]+ cf = torch.sigmoid(torch.cat([xh, me], -1) @ cw)+ rerr = ((ex - xh).norm(dim=-1) / xh.norm(dim=-1)).tolist()+ cos = F.cosine_similarity(ex, xh, dim=-1).tolist()+ dc = (torch.from_numpy(conf.copy()) - cf).abs().tolist()+ worst_x = max(worst_x, max(rerr))+ worst_c = max(worst_c, max(dc))+ line = "round %d (pos %d, y %d): hidden rel err %s, cos %s; conf engine %s ref %s" % (+ seen, p0, y, " ".join("%.4f" % r for r in rerr), " ".join("%.5f" % c for c in cos),+ " ".join("%.3f" % c for c in conf.tolist()), " ".join("%.3f" % c for c in cf.tolist()))+ if head is not None:+ xn = rms(xh, dr.W("mtp.2.norm.weight"), dr.eps)+ lg = torch.zeros(T, head.shape[0])+ for c0 in range(0, head.shape[0], 16384):+ c1 = min(head.shape[0], c0 + 16384)+ lg[:, c0:c1] = xn @ bf16_rows(head, slice(c0, c1)).T+ mh = dr.W("mtp.2.markov_head.head.weight")+ ref_t = []+ p = y+ for i in range(T): # greedy chain on the reference's own drafts+ row = lg[i] + mh @ dr.W("mtp.2.markov_head.embed.weight")[p]+ p = int(row.argmax())+ ref_t.append(p)+ same = sum(int(r == int(t)) for r, t in zip(ref_t, toks))+ agree += same+ total += T+ line += "; drafts engine %s ref %s" % (list(map(int, toks)), ref_t)+ print(line, flush=True)+ done += 1+ if done >= a.rounds:+ break+ print("checked %d rounds: worst hidden rel err %.4f, worst conf diff %.4f%s" % (+ done, worst_x, worst_c, (", drafts agree %d/%d" % (agree, total)) if head is not None else ""))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/dumpcmp.py b/tools/dsv41/dumpcmp.pynew file mode 100644index 000000000..1af24c526--- /dev/null+++ b/tools/dsv41/dumpcmp.py@@ -0,0 +1,184 @@+"""dumpcmp.py - two maya-ds4 runs' debug dumps side by side (the prompt path's numerics: where two runs part).++ layers A B [--pos P] DS4_DUMP_L files: each layer record's relative error between the runs - the median over+ the positions, the worst position, and position P's+ route R --pos P a DS4_DUMP_R file: position P's experts a layer (virtual slot, the expert's rows, P's row)+ gate R... --pos P DS4_DUMP_R files: position P's router a layer - its picks and the margin between the k-th+ pick's score and the next expert's (sqrt(softplus(logit)) + bias)+ sets A B --pos P --route RA RB+ DS4_DUMP_Y files (and each run's DS4_DUMP_R): each layer's set rows' h / y error between+ the runs (rows matched by token and expert), the worst row and position P's, with their+ expert's rows and the piece they were computed in+"""+import argparse+import sys++import numpy as np+++def read_l(path, dim_hc):+ out = []+ with open(path, "rb") as f:+ while True:+ h = f.read(24)+ if len(h) < 24:+ break+ il, p0, t = (int(v) for v in np.frombuffer(h, dtype=np.int64))+ v = np.frombuffer(f.read(t * dim_hc * 4), dtype=np.float32).reshape(t, dim_hc)+ out.append(((il, p0), v))+ return out+++def rel(a, b):+ a = a.astype(np.float64)+ b = b.astype(np.float64)+ return np.linalg.norm(a - b, axis=-1) / np.maximum(np.linalg.norm(b, axis=-1), 1e-30)+++def read_r(path):+ out = {}+ with open(path, "rb") as f:+ while True:+ h = f.read(64)+ if len(h) < 64:+ break+ il, p0, t, k, ne, nvs, nsl, kp = (int(v) for v in np.frombuffer(h, dtype=np.int64))+ ri = np.frombuffer(f.read(t * kp * 4), dtype=np.int32).reshape(t, kp)[:, :k]+ vs = np.frombuffer(f.read(ne * 4), dtype=np.int32)+ b = np.frombuffer(f.read((nvs + 1) * 4), dtype=np.int32)+ rt = np.frombuffer(f.read(t * k * 4), dtype=np.int32)+ gl = np.frombuffer(f.read(t * ne * 4), dtype=np.float32).reshape(t, ne)+ gb = np.frombuffer(f.read(ne * 4), dtype=np.float32)+ out[(il, p0)] = dict(t=t, k=k, ri=ri, vs=vs, b=b, rt=rt, nsl=nsl, glog=gl, bias=gb)+ return out+++def read_y(path, ff, dim): # a layer's pieces at a time (the dump holds the layers in order)+ cur, il0 = [], None+ with open(path, "rb") as f:+ while True:+ h = f.read(48)+ if len(h) < 48:+ break+ il, a, n, ne, ring, mr = (int(v) for v in np.frombuffer(h, dtype=np.int64))+ b = np.frombuffer(f.read((ne + 1) * 4), dtype=np.int32)+ hv = np.frombuffer(f.read(n * ff * 2), dtype=np.float16).reshape(n, ff)+ yv = np.frombuffer(f.read(n * dim * 4), dtype=np.float32).reshape(n, dim)+ if il0 is not None and il != il0:+ yield il0, cur+ cur = []+ il0 = il+ cur.append(dict(il=il, a=a, n=n, ne=ne, ring=ring, mr=mr, b=b, h=hv, y=yv))+ if cur:+ yield il0, cur+++def row_keys(r):+ """A layer's sorted rows -> (token, expert, the expert's rows, the row's place among them)."""+ keys = {}+ for e, v in enumerate(r["vs"]):+ lo, hi = int(r["b"][v]), int(r["b"][v + 1])+ for g in range(lo, hi):+ keys[g] = (int(r["rt"][g]), e, hi - lo, g - lo)+ return keys+++def sets(a):+ """Each layer: the set rows' y / h error between run A and run B, a row matched to the other run's by its (token,+ expert) - DS4_DUMP_R of each run (--route RA RB: the routing may differ) - the median, the worst row, position P's+ rows, with the expert's rows, the row's place in them and the piece it was computed in (row0+rows/experts x max)."""+ RA, RB = (read_r(f) for f in a.route)+ YB = read_y(a.files[1], a.ff, a.dim)+ print("%-3s %8s | %-52s | %s" % ("L", "median y", "worst: y err, h err, token, expert rows+place, ring, piece",+ "pos %d: y err, h err, expert rows+place, ring, piece" % a.pos))+ for il, pa in read_y(a.files[0], a.ff, a.dim):+ ilb, pb = next(YB)+ assert ilb == il, (il, ilb)+ ka = [row_keys(r) for (l2, _), r in RA.items() if l2 == il]+ kb = [row_keys(r) for (l2, _), r in RB.items() if l2 == il]+ if len(ka) != 1 or len(kb) != 1: # (one chunk: p0 0)+ continue+ ka, kb = ka[0], kb[0]+ rows = {}+ for d in pb:+ for i in range(d["n"]):+ t, e, _, _ = kb[d["a"] + i]+ rows[(t, e)] = (d, i)+ errs = []+ for d in pa:+ for i in range(d["n"]):+ t, e, n, pl = ka[d["a"] + i]+ if (t, e) not in rows:+ continue+ db, ib = rows[(t, e)]+ eh = float(rel(d["h"][i:i + 1].astype(np.float32), db["h"][ib:ib + 1].astype(np.float32))[0])+ ey = float(rel(d["y"][i:i + 1], db["y"][ib:ib + 1])[0])+ errs.append((ey, eh, t, e, n, pl, d["ring"], "%d+%d/%dx%d" % (d["a"], d["n"], d["ne"], d["mr"])))+ if not errs:+ continue+ w = max(errs)+ mine = sorted((x for x in errs if x[2] == a.pos), reverse=True)+ print("%-3d %8.1e | %7.1e %7.1e t%-4d e%-3d %3d+%-3d %d %-16s | %s" % (+ il, float(np.median([x[0] for x in errs])), w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7],+ "; ".join("%.1e %.1e e%d %d+%d %d %s" % (x[0], x[1], x[3], x[4], x[5], x[6], x[7]) for x in mine)))+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("mode", choices=["layers", "route", "sets", "gate"])+ ap.add_argument("files", nargs="+")+ ap.add_argument("--pos", type=int, default=-1)+ ap.add_argument("--dim", type=int, default=5120)+ ap.add_argument("--hc", type=int, default=4)+ ap.add_argument("--ff", type=int, default=2304)+ ap.add_argument("--route", nargs=2, metavar=("RA", "RB"))+ a = ap.parse_args()+ if a.mode == "layers":+ A = read_l(a.files[0], a.dim * a.hc)+ B = dict(read_l(a.files[1], a.dim * a.hc))+ print("%-12s %10s %10s %6s %10s" % ("record", "median", "worst", "at", "pos %d" % a.pos))+ for key, va in A:+ if key not in B:+ continue+ e = rel(va, B[key])+ w = int(np.argmax(e))+ p = a.pos - key[1]+ ep = "%.3e" % e[p] if 0 <= p < len(e) else "-"+ print("%-12s %10.3e %10.3e %6d %10s" % ("L%d@%d" % key, float(np.median(e)), float(e[w]), w + key[1], ep))+ return+ if a.mode == "gate":+ for f in a.files: # each run's router at position P: the picks and the margin past the k-th+ print("== %s" % f)+ for (il, p0), r in sorted(read_r(f).items()):+ p = a.pos - p0+ if not 0 <= p < r["t"]:+ continue+ lg = r["glog"][p].astype(np.float64)+ sc = np.sqrt(np.where(lg > 20, lg, np.log1p(np.exp(np.minimum(lg, 20)))))+ sel = sc + r["bias"]+ o = np.argsort(-sel, kind="stable")+ k = r["k"]+ print("L%-3d picks %s | k-th %.5f, next e%d %.5f: margin %.2e" % (+ il, " ".join("e%d" % e for e in o[:k]), sel[o[k - 1]], o[k], sel[o[k]], sel[o[k - 1]] - sel[o[k]]))+ return+ if a.mode == "route":+ R = read_r(a.files[0])+ for (il, p0), r in sorted(R.items()):+ p = a.pos - p0+ if not 0 <= p < r["t"]:+ continue+ rows = np.nonzero(r["rt"] == p)[0]+ parts = []+ for e in r["ri"][p]:+ v = int(r["vs"][e])+ lo, hi = int(r["b"][v]), int(r["b"][v + 1])+ mine = [int(x) for x in rows if lo <= x < hi]+ parts.append("e%d v%d%s rows %d (row %s, %d into it)" % (e, v, "" if v < r["nsl"] else "*", hi - lo,+ mine[0] if mine else "?", (mine[0] - lo) if mine else -1))+ print("L%d@%d: %s" % (il, p0, "; ".join(parts)))+ return+ sets(a)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/dumpscan.py b/tools/dsv41/dumpscan.pynew file mode 100644index 000000000..2c799cbd7--- /dev/null+++ b/tools/dsv41/dumpscan.py@@ -0,0 +1,42 @@+"""dumpscan.py DUMP_L [--pos P] [--dim 5120] [--hc 4] - one run's DS4_DUMP_L records: each layer's residual stream at+position P against the other positions (its norm over the median norm, its largest |value| and where, non-finite+values) - where a position's stream blows up or goes non-finite."""+import argparse+import sys++import numpy as np+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("dump")+ ap.add_argument("--pos", type=int, default=0)+ ap.add_argument("--dim", type=int, default=5120)+ ap.add_argument("--hc", type=int, default=4)+ a = ap.parse_args()+ n = a.dim * a.hc+ print("%-10s %10s %10s %12s %10s %10s %s" % ("record", "|h| pos", "median", "pos / med", "max|v| pos", "max|v| all",+ "non-finite (pos / all)"))+ with open(a.dump, "rb") as f:+ while True:+ h = f.read(24)+ if len(h) < 24:+ break+ il, p0, t = (int(v) for v in np.frombuffer(h, dtype=np.int64))+ v = np.frombuffer(f.read(t * n * 4), dtype=np.float32).reshape(t, n)+ p = a.pos - p0+ if not 0 <= p < t:+ continue+ fin = np.isfinite(v)+ vv = np.where(fin, v, 0).astype(np.float64)+ nr = np.linalg.norm(vv, axis=1)+ med = float(np.median(nr))+ mx = np.abs(vv[p])+ j = int(np.argmax(mx))+ print("L%-4d@%-3d %10.3e %10.3e %12.2f %10.3e %10.3e %d / %d (max at stream %d dim %d)" % (+ il, p0, nr[p], med, nr[p] / max(med, 1e-30), mx[j], float(np.abs(vv).max()),+ int((~fin[p]).sum()), int((~fin).sum()), j // a.dim, j % a.dim))+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/dumpscan_y.py b/tools/dsv41/dumpscan_y.pynew file mode 100644index 000000000..bb8cbae5e--- /dev/null+++ b/tools/dsv41/dumpscan_y.py@@ -0,0 +1,48 @@+"""dumpscan_y.py DUMP_Y DUMP_R --pos P [--layers a-b] - one run's expert set rows (DS4_DUMP_Y, DS4_DUMP_R): position+P's rows a layer - its experts, their rows and P's place in them, |h| and |y| against the layer's median row."""+import argparse+import sys++import numpy as np++sys.path.insert(0, __file__.rsplit("/", 1)[0] if "/" in __file__ else ".")+from dumpcmp import read_r, read_y, row_keys # noqa: E402+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("dump_y")+ ap.add_argument("dump_r")+ ap.add_argument("--pos", type=int, required=True)+ ap.add_argument("--layers", default="0-99")+ ap.add_argument("--dim", type=int, default=5120)+ ap.add_argument("--ff", type=int, default=2304)+ a = ap.parse_args()+ l0, l1 = (int(v) for v in a.layers.split("-"))+ R = read_r(a.dump_r)+ for il, pieces in read_y(a.dump_y, a.ff, a.dim):+ if not l0 <= il <= l1:+ continue+ r = [v for (l2, _), v in R.items() if l2 == il]+ if len(r) != 1:+ continue+ keys = row_keys(r[0])+ hn, yn, mine = [], [], []+ for d in pieces:+ for i in range(d["n"]):+ t, e, n, pl = keys[d["a"] + i]+ h = float(np.linalg.norm(d["h"][i].astype(np.float64)))+ y = float(np.linalg.norm(d["y"][i].astype(np.float64)))+ hm = float(np.abs(d["h"][i].astype(np.float32)).max())+ hn.append(h)+ yn.append(y)+ if t == a.pos:+ mine.append((e, n, pl, h, hm, y, d["ring"], "%d+%d/%dx%d" % (d["a"], d["n"], d["ne"], d["mr"])))+ mh, my = float(np.median(hn)), float(np.median(yn))+ print("L%-3d median |h| %.3e |y| %.3e; pos %d: %s" % (il, mh, my, a.pos, "; ".join(+ "e%d (%d rows, #%d) |h| %.2e max %.2e |y| %.2e (x%.1f) ring %d %s" % (e, n, pl, h, hm, y, y / max(my, 1e-30), rg, pc)+ for e, n, pl, h, hm, y, rg, pc in mine)))+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/engram_hash.py b/tools/dsv41/engram_hash.pynew file mode 100644index 000000000..43b861d94--- /dev/null+++ b/tools/dsv41/engram_hash.py@@ -0,0 +1,142 @@+"""tools/ds4/engram_hash.py - DeepSeek-V4.1's Engram n-gram hashes (after DeepSeek's inference/engram.py, MIT).++A position is hashed as the n-grams (2 .. max_ngram_size) that end there, each over n_heads heads; every (n-gram+size, head) pair owns a prime-sized bucket range of the layer's table. Token ids first go through a compressed map+where tokens that normalize alike (NFKC, NFD, accents stripped, lowercased, whitespace runs collapsed) share an id.++ lay = Layout.from_config(text_config)+ tm, n = compressed_token_map(tokenizer_json_path) # needs the `tokenizers` package+ h = Hasher(lay, tm, multipliers(lay, n), pad=tm[text_config["engram_pad_token_id"]])+ h.ids(tokens) -> int64 [T, n_layers, (max_ngram_size - 1) * n_heads], absolute table rows+"""+from __future__ import annotations++from dataclasses import dataclass++import numpy as np+++def _is_prime(n: int) -> bool:+ if n < 2:+ return False+ if n % 2 == 0:+ return n == 2+ i = 3+ while i * i <= n:+ if n % i == 0:+ return False+ i += 2+ return True+++def _next_prime(start: int, seen: set[int]) -> int:+ """The smallest prime above `start` that has not been handed out yet (engram.py's find_next_prime)."""+ c = start + 1+ while not _is_prime(c) or c in seen:+ c += 1+ return c+++@dataclass(frozen=True)+class Layout:+ layer_ids: tuple[int, ...]+ num_embeddings: tuple[int, ...]+ max_ngram: int+ n_heads: int+ head_dim: int+ primes: tuple[tuple[tuple[int, ...], ...], ...] # [layer][n-gram size - 2][head]++ @classmethod+ def from_config(cls, t: dict) -> "Layout":+ layer_ids = tuple(t["engram_layer_ids"])+ mx, nh = t["engram_max_ngram_size"], t["engram_n_heads"]+ primes, seen = [], set()+ for _ in layer_ids:+ per = []+ for _ in range(mx - 1):+ sizes, cur = [], t["engram_vocab_size"] - 1+ for _ in range(nh):+ cur = _next_prime(cur, seen)+ seen.add(cur)+ sizes.append(cur)+ per.append(tuple(sizes))+ primes.append(tuple(per))+ lay = cls(layer_ids, tuple(t["engram_num_embeddings"]), mx, nh, t["engram_head_dim"], tuple(primes))+ for i, n in enumerate(lay.num_embeddings):+ got = sum(p for per in lay.primes[i] for p in per)+ if got != n:+ raise ValueError("engram layer %d: the primes sum to %d, the config says %d rows" % (i, got, n))+ return lay++ def flat_primes(self) -> np.ndarray:+ """[n_layers, (max_ngram - 1) * n_heads] int64, in hash-column order."""+ return np.array([[p for per in layer for p in per] for layer in self.primes], dtype=np.int64)++ def offsets(self) -> np.ndarray:+ f = self.flat_primes()+ return np.concatenate([np.zeros((f.shape[0], 1), np.int64), np.cumsum(f, axis=1)[:, :-1]], axis=1)+++def multipliers(lay: Layout, compressed_vocab: int) -> np.ndarray:+ """[n_layers, max_ngram] int64, odd, bounded so token * multiplier stays inside int64 (engram.py)."""+ bound = max(1, (np.iinfo(np.int64).max // compressed_vocab) // 2)+ rows = []+ for lid in lay.layer_ids:+ g = np.random.default_rng(10007 * lid)+ rows.append(g.integers(low=0, high=bound, size=(lay.max_ngram,), dtype=np.int64) * 2 + 1)+ return np.stack(rows)+++def compressed_token_map(tokenizer_json: str) -> tuple[np.ndarray, int]:+ """engram.py's build_compressed_token_map over the raw Rust tokenizer (no clean-up of spaces)."""+ from tokenizers import Regex, Tokenizer, normalizers++ tok = Tokenizer.from_file(str(tokenizer_json))+ sentinel = ""+ norm = normalizers.Sequence([+ normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(), normalizers.Lowercase(),+ normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), normalizers.Replace(Regex(r"^ $"), sentinel),+ normalizers.Strip(), normalizers.Replace(sentinel, " "),+ ])+ n = tok.get_vocab_size(with_added_tokens=True)+ key_to_new: dict[str, int] = {}+ lookup = np.zeros(n, dtype=np.int64)+ for tid in range(n):+ text = tok.decode([tid], skip_special_tokens=False)+ if "�" in text:+ key = tok.id_to_token(tid)+ else:+ normalized = norm.normalize_str(text)+ key = normalized if normalized else text+ new = key_to_new.get(key)+ if new is None:+ new = len(key_to_new)+ key_to_new[key] = new+ lookup[tid] = new+ return lookup, len(key_to_new)+++class Hasher:+ def __init__(self, lay: Layout, token_map: np.ndarray, mult: np.ndarray, pad: int):+ self.lay, self.tm, self.mult, self.pad = lay, token_map, mult, int(pad)+ self.primes = np.array(lay.primes, dtype=np.int64) # [L, max_ngram - 1, heads]+ self.off = lay.offsets() # [L, cols]++ def ids(self, tokens) -> np.ndarray:+ """Absolute table rows [T, n_layers, cols] for a sequence that starts at position 0 (no dead tokens)."""+ c = self.tm[np.asarray(tokens, dtype=np.int64)]+ T = len(c)+ pos = np.arange(T)+ toks = []+ blocked = np.zeros(T, dtype=bool)+ for s in range(self.lay.max_ngram):+ src = c[np.clip(pos - s, 0, None)]+ blocked = blocked | (pos < s)+ toks.append(np.where(blocked, self.pad, src))+ toks = np.stack(toks, axis=-1) # [T, max_ngram]+ prod = toks[:, None, :] * self.mult[None] # [T, L, max_ngram]+ rolling, out = prod[..., 0], []+ for i in range(1, self.lay.max_ngram):+ rolling = np.bitwise_xor(rolling, prod[..., i])+ out.append(rolling[..., None] % self.primes[None, :, i - 1])+ return np.concatenate(out, axis=-1) + self.off[None]diff --git a/tools/dsv41/exl3_torch.py b/tools/dsv41/exl3_torch.pynew file mode 100644index 000000000..848e2e68d--- /dev/null+++ b/tools/dsv41/exl3_torch.py@@ -0,0 +1,146 @@+"""tools/ds4/exl3_torch.py - EXL3 (exllamav3) matrices decoded on the GPU with torch, for the DeepSeek-V4.1 reference.++The format is include/strata/kernels/exl3.hpp's; this is tools/exl3_pack.py's numpy decoder (exl3_states, exl3_values,+exl3_rows) on torch tensors. decode(st, prefix) returns the full weight as [n, k] (out x in, F.linear's layout):+W = diag(suh) H W' H diag(svh), so x W is H(H(x * suh) W') * svh.+"""+from __future__ import annotations++import math++import numpy as np+import torch++_P = torch.arange(256)+_ROW = (_P >> 3 & 3) * 2 + (_P & 1) + 8 * (_P >> 1 & 1)+_COL = (_P >> 5) + 8 * (_P >> 2 & 1)+_PERM = _ROW * 16 + _COL # tile[..., _PERM[p]] = value of step p+++def _sylvester(n: int) -> torch.Tensor:+ h = torch.ones(1, 1, dtype=torch.float64)+ while h.shape[0] < n:+ h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)], 0)+ return h / math.sqrt(n)+++_HAD = {}+++def had(dev) -> torch.Tensor:+ if dev not in _HAD:+ _HAD[dev] = _sylvester(128).to(dev, torch.float32)+ return _HAD[dev]+++def _f16(b: int) -> float:+ return float(np.array([b], dtype=np.uint16).view(np.float16)[0])+++K_INV, K_BIAS = _f16(0x1EEE), _f16(0xC931)+++def states(words: torch.Tensor, K: int) -> torch.Tensor:+ """(..., 8 K) int32 tile words -> (..., 256) 16-bit states (MSB-first stream, tail-biting), int64."""+ W = 8 * K+ dev = words.device+ p = _P.to(dev)+ b1 = (p + 1 + 256) * K+ i1, i0 = (b1 - 1) >> 5, (b1 - 16) >> 5+ sh = ((i1 + 1) << 5) - b1+ w = words.to(torch.int64) & 0xFFFFFFFF+ m = (w[..., i0 % W] << 32) | w[..., i1 % W]+ return (m >> sh) & 0xFFFF+++def values(st: torch.Tensor, cb: int) -> torch.Tensor:+ """States -> FP16-rounded codebook values (float32)."""+ if cb == 2:+ x = (st * 0x83DCD12D) & 0xFFFFFFFF+ bs = (x & 0xFF) + (x >> 8 & 0xFF) + (x >> 16 & 0xFF) + (x >> 24 & 0xFF)+ v = (1024.0 + bs.to(torch.float64)) * K_INV + K_BIAS+ return v.to(torch.float16).to(torch.float32)+ x = st * (0xCBAC1FED if cb == 1 else 89226354)+ if cb == 0:+ x = x + 64248484+ x = ((x & 0xFFFFFFFF) & 0x8FFF8FFF) ^ 0x3B603B60+ lo = (x & 0xFFFF).to(torch.int16).view(torch.float16).to(torch.float64) # bit patterns as FP16+ hi = (x >> 16).to(torch.int16).view(torch.float16).to(torch.float64)+ return (lo + hi).to(torch.float16).to(torch.float32)+++def decode_raw(trellis: torch.Tensor, suh: torch.Tensor, svh: torch.Tensor, cb: int) -> torch.Tensor:+ """trellis int16 [k/16, n/16, 16 K] (device), suh F16 [k], svh F16 [n] -> W [n, k] float32."""+ kt, nt, w16 = trellis.shape+ K = w16 // 16+ k, n = kt * 16, nt * 16+ dev = trellis.device+ words = trellis.contiguous().view(torch.int32) # [kt, nt, 8K]+ out = torch.empty(n, k, dtype=torch.float32, device=dev)+ H = had(dev)+ step = max(1, (1 << 27) // (nt * 256 * 8)) # 16-row slices per chunk (bounded scratch)+ step = max(8, step - step % 8) # whole 128-row Hadamard blocks+ perm = _PERM.to(dev)+ for a in range(0, kt, step):+ b = min(kt, a + step)+ v = values(states(words[a:b], K), cb) # [b-a, nt, 256] in step order+ tile = torch.empty_like(v)+ tile[..., perm] = v+ inner = tile.view(b - a, nt, 16, 16).permute(0, 2, 1, 3).reshape((b - a) * 16, n)+ inner = (inner.view(-1, n // 128, 128) @ H).view(-1, n) # over n, in blocks of 128+ inner = (H @ inner.view(-1, 128, n)).view(-1, n) # over k, in blocks of 128+ out[:, a * 16:b * 16] = inner.T+ out *= svh.to(torch.float32)[:, None]+ out *= suh.to(torch.float32)[None, :]+ return out+++def decode_many(st, prefixes: list[str], dev) -> list[torch.Tensor]:+ """Several EXL3 matrices of one shape (a layer's experts' w1, say) decoded together: their trellises side by side+ along n (both Hadamards act inside blocks of 128, so the stacked matrix decodes as each would), then each one's+ channel scales. Returns [n, k] float32 views, one a prefix."""+ trs = [tensor(st, p + ".trellis", dev) for p in prefixes]+ kt, nt, _ = trs[0].shape+ cb = codebook(st, prefixes[0])+ ones_k = torch.ones(kt * 16, dtype=torch.float16, device=dev)+ ones_n = torch.ones(nt * 16 * len(prefixes), dtype=torch.float16, device=dev)+ inner = decode_raw(torch.cat(trs, dim=1), ones_k, ones_n, cb) # [N n, k]+ n = nt * 16+ out = []+ for i, p in enumerate(prefixes):+ w = inner[i * n:(i + 1) * n]+ w *= tensor(st, p + ".svh", dev).to(torch.float32)[:, None]+ w *= tensor(st, p + ".suh", dev).to(torch.float32)[None, :]+ out.append(w)+ return out+++def tensor(st, name: str, dev) -> torch.Tensor:+ _, dt, shape, _, _ = st.t[name]+ r = st.raw(name)+ np_dt = {"F32": np.float32, "F16": np.float16, "I16": np.int16, "I32": np.int32, "BF16": np.uint16}[dt]+ a = torch.from_numpy(np.array(r.view(np_dt)).reshape(shape))+ if dt == "BF16":+ a = a.view(torch.int16).view(torch.bfloat16)+ return a.to(dev)+++def is_exl3(st, prefix: str) -> bool:+ return st.has(prefix + ".trellis")+++def codebook(st, prefix: str) -> int:+ return 2 if st.has(prefix + ".mul1") else 1 if st.has(prefix + ".mcg") else 0+++def decode(st, prefix: str, dev) -> torch.Tensor:+ """The EXL3 matrix at `prefix` (.trellis/.suh/.svh) as W [n, k] float32 on dev."""+ return decode_raw(tensor(st, prefix + ".trellis", dev), tensor(st, prefix + ".suh", dev),+ tensor(st, prefix + ".svh", dev), codebook(st, prefix))+++def weight(st, prefix: str, dev) -> torch.Tensor:+ """A linear weight [n, k] float32: EXL3 decoded, or the stored float tensor (prefix.weight)."""+ if is_exl3(st, prefix):+ return decode(st, prefix, dev)+ return torch.from_numpy(np.array(st.f32(prefix + ".weight"))).to(dev)diff --git a/tools/dsv41/pack.py b/tools/dsv41/pack.pynew file mode 100644index 000000000..4ab6edf33--- /dev/null+++ b/tools/dsv41/pack.py@@ -0,0 +1,152 @@+"""tools/ds4/pack.py - the engine's pack for a DeepSeek-V4.1 EXL3 model, beside the model's files (no weights copied).++ python tools/ds4/pack.py --model ~/models/DSV4++writes <model>/maya-ds4/:+ ds4.txt the geometry, `key value` lines (lists comma-separated), from config.json's text_config (and its+ vision_config as vision.<key>, image_token_id)+ tensors.txt every tensor of the directory's safetensors (the index's 43 shards and the base repo's Engram shards+ 47 / 48 beside them): `file <i> <name>` lines, then `t <name> <dtype> <file> <offset> <bytes> <dims...>`+ engram.bin the Engram hash tables (tools/ds4/engram_hash.py): int32 magic 'DSE1', vocab, layers, max_ngram,+ heads, head_dim, pad (compressed); int32 token_map[vocab]; int64 mult[layers][max_ngram];+ int64 primes[layers][cols]; int64 offsets[layers][cols] (cols = (max_ngram - 1) * heads)+ tokenizer/ the server's tokenizer (tools/strata_tokenizer.py's layout, pre "deepseek3") and chat template++The compressed token map needs the `tokenizers` package (it normalizes every token as DeepSeek's training did).+"""+from __future__ import annotations++import argparse+import json+import pathlib+import struct+import sys++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import engram_hash as EH # noqa: E402+from st import SafeTensors # noqa: E402++TEMPLATE = HERE / "chat_template.jinja"+++def need(st: SafeTensors, t: dict) -> list[str]:+ """The tensors the engine reads; a missing one stops the pack here instead of the engine's start."""+ miss = []+ for li in range(t["num_hidden_layers"]):+ L = "layers.%d." % li+ for n in ("attn.wq_a.trellis", "attn.wq_b.trellis", "attn.wkv.trellis", "attn.wo_b.trellis",+ "attn.attn_sink", "attn.q_norm.weight", "attn.kv_norm.weight", "attn_norm.weight", "ffn_norm.weight",+ "ffn.gate.weight", "ffn.gate.bias", "hc_attn_fn", "hc_ffn_fn", "ffn.shared_experts.w1.trellis"):+ if not st.has(L + n):+ miss.append(L + n)+ if li in t["engram_layer_ids"] and not st.has(L + "engram.embed.weight"):+ miss.append(L + "engram.embed.weight (shards 47 / 48 of deepseek-ai/DeepSeek-V4.1-Flash-1017)")+ for n in ("embed.weight", "norm.weight", "head.trellis"):+ if not st.has(n):+ miss.append(n)+ return miss+++def write_engram(out: pathlib.Path, model: pathlib.Path, t: dict):+ lay = EH.Layout.from_config(t)+ tm, n = EH.compressed_token_map(model / "tokenizer.json")+ if n != t["engram_compressed_vocab_size"]:+ raise SystemExit("engram: the compressed vocab has %d ids, the config says %d" %+ (n, t["engram_compressed_vocab_size"]))+ vocab = t["vocab_size"]+ full = np.zeros(vocab, dtype=np.int32) # ids past the tokenizer's (none in use) map to 0+ full[:len(tm)] = tm+ mult = EH.multipliers(lay, n)+ cols = (lay.max_ngram - 1) * lay.n_heads+ with out.open("wb") as f:+ f.write(struct.pack("<4s6i", b"DSE1", vocab, len(lay.layer_ids), lay.max_ngram, lay.n_heads, lay.head_dim,+ int(tm[t["engram_pad_token_id"]])))+ f.write(full.astype("<i4").tobytes())+ f.write(mult.astype("<i8").tobytes())+ f.write(lay.flat_primes().astype("<i8").tobytes())+ f.write(lay.offsets().astype("<i8").tobytes())+ return len(lay.layer_ids), cols+++def write_tokenizer(out: pathlib.Path, model: pathlib.Path):+ """vocab.json (id order), merges.txt, token_type.json, tokenizer.json {pre, special_ids} and the chat template."""+ tj = json.loads((model / "tokenizer.json").read_text(encoding="utf-8"))+ vocab = tj["model"]["vocab"]+ added = {a["id"]: a for a in tj["added_tokens"]}+ n = max(max(vocab.values()), max(added)) + 1+ tokens = ["<|unused▁%d|>" % i for i in range(n)] # (ids no token names: none in this vocabulary)+ types = [1] * n # llama.cpp: 1 normal, 3 control, 4 user-defined+ for s, i in vocab.items():+ tokens[i] = s+ for i, a in added.items():+ tokens[i] = a["content"]+ types[i] = 3 if a["special"] else 4+ merges = tj["model"]["merges"]+ merges = [m if isinstance(m, str) else " ".join(m) for m in merges]+ out.mkdir(parents=True, exist_ok=True)+ if len(set(tokens)) != n:+ raise SystemExit("tokenizer.json: duplicate token strings")+ (out / "vocab.json").write_text(json.dumps({t: i for i, t in enumerate(tokens)}, ensure_ascii=False),+ encoding="utf-8")+ (out / "merges.txt").write_text("\n".join(merges), encoding="utf-8")+ (out / "token_type.json").write_text(json.dumps(types))+ (out / "tokenizer.json").write_text(json.dumps({"pre": "deepseek3", "special_ids": {+ "bos": 0, "eos": 1, "pad": 2}}, indent=1))+ if TEMPLATE.exists():+ (out / "chat_template.jinja").write_text(TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8")+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--model", required=True)+ ap.add_argument("--out", help="default: <model>/maya-ds4")+ a = ap.parse_args()+ model = pathlib.Path(a.model).expanduser().resolve()+ out = pathlib.Path(a.out).expanduser() if a.out else model / "maya-ds4"+ cfg = json.loads((model / "config.json").read_text())+ t = cfg["text_config"]+ q = cfg.get("quantization_config", {})+ if q.get("quant_method") != "exl3":+ raise SystemExit("%s: not an EXL3 model (quant_method %r)" % (model, q.get("quant_method")))+ st = SafeTensors(model)+ miss = need(st, t)+ if miss:+ raise SystemExit("missing tensors:\n " + "\n ".join(miss[:20]))+ out.mkdir(parents=True, exist_ok=True)++ with (out / "ds4.txt").open("w") as f:+ for k, v in t.items():+ if isinstance(v, bool):+ v = int(v)+ if isinstance(v, (int, float)):+ f.write("%s %s\n" % (k, v))+ elif isinstance(v, list) and all(isinstance(x, (int, float)) for x in v):+ f.write("%s %s\n" % (k, ",".join(str(x) for x in v)))+ elif isinstance(v, dict):+ for k2, v2 in v.items():+ if isinstance(v2, (int, float)) and not isinstance(v2, bool):+ f.write("%s.%s %s\n" % (k, k2, v2))+ # the vision tower's geometry (vision.<key>) and the token an image's span carries+ for k, v in (cfg.get("vision_config") or {}).items():+ if isinstance(v, (int, float)) and not isinstance(v, bool):+ f.write("vision.%s %s\n" % (k, v))+ if isinstance(cfg.get("image_token_id"), int):+ f.write("image_token_id %d\n" % cfg["image_token_id"])+ f.write("model_dir %s\n" % model)+ with (out / "tensors.txt").open("w") as f:+ for i, fn in enumerate(st.files):+ f.write("file %d %s\n" % (i, fn))+ for name, (fn, dt, shape, off, nb) in st.t.items():+ f.write("t %s %s %d %d %d %s\n" % (name, dt, st.files.index(fn), off, nb, " ".join(map(str, shape))))+ nl, cols = write_engram(out / "engram.bin", model, t)+ write_tokenizer(out / "tokenizer", model)+ print("%s: %d tensors in %d files, Engram %d layers x %d hash columns -> %s" %+ (model.name, len(st.t), len(st.files), nl, cols, out))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pynew file mode 100644index 000000000..f7cd510ba--- /dev/null+++ b/tools/dsv41/pack_gguf.py@@ -0,0 +1,498 @@+"""tools/ds4/pack_gguf.py - the engine's pack for a DeepSeek-V4.1 GGUF (unsloth's DeepSeek-V4.1-Flash-1017 UD-Q8_K_XL+and the like: dense BF16, routed experts MXFP4, Engram tables MXFP8 or Q6_K), beside the model's files (no weights+copied).++ python tools/ds4/pack_gguf.py --model ~/models/DeepSeek-V4.1-Flash-1017-GGUF/UD-Q8_K_XL++writes <model>/maya-ds4/ in pack.py's format, so the engine reads both:+ ds4.txt the geometry under config.json's names (the GGUF's deepseek41.* keys translated), model_dir, format gguf+ tensors.txt every tensor under the checkpoint's names: `file <i> <name>` lines, then+ `t <name> <dtype> <file> <offset> <bytes> <dims...>` - dims outermost first (torch order: a matrix is+ [out][in]), dtype BF16 / F32 / MXFP4 / MXFP8 (ggml's block layouts, 32 values a block); the routed+ experts as one tensor a layer and role ([n_exp][out][in])+ engram.bin the Engram hash tables, from the GGUF's metadata (pack.py's layout; no tokenizers package needed)+ tokenizer/ the server's tokenizer from the GGUF's vocabulary and merges (pre "deepseek3"), and its chat template++A GGUF has no DSpark drafter: drafter_pack.py makes it from DeepSeek's checkpoint into ../dspark/, and the pack takes it+from there when it is present. Nor a vision tower: vision_pack.py makes it from DeepSeek's checkpoint into ../vision/+(--vision: another folder), and the pack takes it and its geometry (vision.<key>, image_token_id) from there when it is+present; without it the engine reads text only.++A quant that leaves the two Engram tables out (v1.1.16's Maya-S; Maya's quants hold them since) takes them from a set+of GGUF files that holds them, beside it: --engram, else ../Engram/ (unsloth's MXFP8 shard of them, where setups before+v1.1.17 put it) when it is whole.+"""+from __future__ import annotations++import argparse+import json+import os+import pathlib+import re+import struct+import sys++TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17),+ 67: ("MXFP8", 32, 33), # ggml type -> (name, values a block, bytes a block); 67: unsloth's fork+ 43: ("MXFP8", 32, 33), # llama.cpp's own id (unsloth's GGUFs from 2026-10-19 on)+ # the low-bit expert types (the Maya quants' experts; the engine's ds4_iq kernels)+ 10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110), 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176),+ 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98),+ 20: ("IQ4_NL", 32, 18), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136)}+# the Engram set a quant without the tables takes them from by default, beside it (or inside its folder): unsloth's+# MXFP8 shard of them, where setups before v1.1.17 put it for Maya-S (--engram: any other)+ENGRAM_SETS = ("Engram",)+++class Gguf:+ """One GGUF shard's metadata and tensor infos (offsets absolute in the file)."""++ def __init__(self, path: pathlib.Path):+ self.path = path+ with path.open("rb") as f:+ self.f = f+ magic = f.read(4)+ if magic != b"GGUF":+ raise SystemExit("%s: not a GGUF file" % path)+ self.version, nt, nkv = struct.unpack("<IQQ", f.read(20))+ self.kv = {}+ for _ in range(nkv):+ k = self._str()+ self.kv[k] = self._val(struct.unpack("<I", f.read(4))[0])+ self.tensors = []+ for _ in range(nt):+ name = self._str()+ nd = struct.unpack("<I", f.read(4))[0]+ dims = list(struct.unpack("<%dQ" % nd, f.read(8 * nd)))+ typ, off = struct.unpack("<IQ", f.read(12))+ self.tensors.append((name, typ, dims, off))+ align = int(self.kv.get("general.alignment", 32))+ pos = f.tell()+ self.data_start = (pos + align - 1) // align * align+ del self.f++ def _str(self):+ n = struct.unpack("<Q", self.f.read(8))[0]+ return self.f.read(n).decode("utf-8", "replace")++ def _val(self, t):+ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}+ if t in fmt:+ n = struct.calcsize(fmt[t])+ return struct.unpack("<" + fmt[t], self.f.read(n))[0]+ if t == 8:+ return self._str()+ if t == 9:+ at, n = struct.unpack("<IQ", self.f.read(12))+ if at in fmt:+ n_b = struct.calcsize(fmt[at])+ return list(struct.unpack("<%d%s" % (n, fmt[at]), self.f.read(n * n_b)))+ return [self._val(at) for _ in range(n)]+ raise SystemExit("%s: metadata type %d" % (self.path, t))+++# GGUF name (blk.N. stripped) -> the checkpoint's name under layers.N. (pack.py / the engine)+LAYER = {+ "attn_norm.weight": "attn_norm.weight",+ "attn_sinks.weight": "attn.attn_sink",+ "attn_q_a.weight": "attn.wq_a.weight",+ "attn_q_a_norm.weight": "attn.q_norm.weight",+ "attn_q_b.weight": "attn.wq_b.weight",+ "attn_kv.weight": "attn.wkv.weight",+ "attn_kv_a_norm.weight": "attn.kv_norm.weight",+ "attn_output_a.weight": "attn.wo_a.slice", # one entry a group (the engine's slices)+ "attn_output_b.weight": "attn.wo_b.weight",+ "attn_compressor_kv.weight": "attn.compressor.wkv.weight",+ "attn_compressor_gate.weight": "attn.compressor.wgate.weight",+ "attn_compressor_norm.weight": "attn.compressor.norm.weight",+ "indexer.attn_q_b.weight": "attn.indexer.wq_b.weight",+ "indexer.attn_k.weight": "attn.indexer.wk.weight",+ "indexer.k_norm.weight": "attn.indexer.k_norm.weight",+ "indexer.proj.weight": "attn.indexer.weights_proj.weight",+ "hc_attn_fn.weight": "hc_attn_fn",+ "hc_attn_base.weight": "hc_attn_base",+ "hc_attn_scale.weight": "hc_attn_scale",+ "hc_ffn_fn.weight": "hc_ffn_fn",+ "hc_ffn_base.weight": "hc_ffn_base",+ "hc_ffn_scale.weight": "hc_ffn_scale",+ "ffn_norm.weight": "ffn_norm.weight",+ "ffn_gate_inp.weight": "ffn.gate.weight",+ "exp_probs_b.bias": "ffn.gate.bias",+ "exp_probs_b_vl.bias": "ffn.gate.bias_vl",+ "ffn_gate_shexp.weight": "ffn.shared_experts.w1.weight",+ "ffn_up_shexp.weight": "ffn.shared_experts.w3.weight",+ "ffn_down_shexp.weight": "ffn.shared_experts.w2.weight",+ "ffn_gate_exps.weight": "ffn.experts.w1", # [n_exp][ff][dim]: one entry an expert+ "ffn_up_exps.weight": "ffn.experts.w3",+ "ffn_down_exps.weight": "ffn.experts.w2", # [n_exp][dim][ff]+ "engram_embd.weight": "engram.embed.weight",+ "engram_kv.weight": "engram.wkv.weight",+ "engram_gate.weight": "engram.gate.weight", # q_weight * k_weight, folded by the converter+}+GLOBAL = {"token_embd.weight": "embed.weight", "output_norm.weight": "norm.weight", "output.weight": "head.weight"}+++def geometry(kv: dict, vocab: int) -> dict:+ """config.json's text_config names (what the engine reads) from the GGUF's deepseek41.* keys."""+ a = kv["general.architecture"]+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731+ nl = g("block_count")+ ratios = list(g("attention.compress_ratios", []))+ out = {+ "vocab_size": vocab,+ "hidden_size": g("embedding_length"),+ "num_hidden_layers": nl,+ "num_attention_heads": g("attention.head_count"),+ "head_dim": g("attention.key_length"),+ "qk_rope_head_dim": g("rope.dimension_count"),+ "q_lora_rank": g("attention.q_lora_rank"),+ "o_lora_rank": g("attention.output_lora_rank"),+ "o_groups": g("attention.output_group_count"),+ "n_routed_experts": g("expert_count"),+ "num_experts_per_tok": g("expert_used_count"),+ "moe_intermediate_size": g("expert_feed_forward_length"),+ "sliding_window": g("attention.sliding_window"),+ "hc_mult": g("hyper_connection.count"),+ "hc_sinkhorn_iters": g("hyper_connection.sinkhorn_iterations"),+ "hc_eps": g("hyper_connection.epsilon"),+ "index_n_heads": g("attention.indexer.head_count"),+ "index_head_dim": g("attention.indexer.key_length"),+ "index_topk": g("attention.indexer.top_k"),+ "candidate_source_layer_id": g("attention.indexer.candidate_source_layer"),+ "candidate_topk_blocks": g("attention.indexer.candidate_top_k_blocks"),+ "candidate_block_size": g("attention.indexer.candidate_block_size"),+ "rope_scaling.original_max_position_embeddings": g("rope.scaling.original_context_length"),+ "rope_scaling.factor": g("rope.scaling.factor"),+ "rope_scaling.beta_fast": g("rope.scaling.yarn_beta_fast"),+ "rope_scaling.beta_slow": g("rope.scaling.yarn_beta_slow"),+ "eos_token_id": kv.get("tokenizer.ggml.eos_token_id", 1),+ "rms_norm_eps": g("attention.layer_norm_rms_epsilon"),+ "routed_scaling_factor": g("expert_weights_scale"),+ "swiglu_limit": (g("swiglu_clamp_exp") or [0.0])[0],+ "rope_theta": g("rope.freq_base"),+ "compress_rope_theta": g("attention.compress_rope_freq_base"),+ "compress_ratios": ratios,+ "kv_source_layer_ids": list(g("attention.kv_source_layers", [])),+ "index_source_layer_ids": list(g("attention.indexer.source_layers", [])),+ "engram_layer_ids": list(g("engram.layers", [])),+ "max_position_embeddings": g("context_length"),+ "dspark_block_size": g("dspark.block_size", 0),+ "dspark_noise_token_id": g("dspark.noise_token_id", 0),+ "dspark_markov_rank": g("dspark.markov_rank", 0),+ "dspark_n_routed_experts": g("dspark.expert_count", 0),+ "dspark_num_experts_per_tok": g("dspark.expert_used_count", 0),+ "dspark_target_layer_ids": list(g("dspark.target_layers", [])),+ }+ clamp = g("swiglu_clamp_exp") or []+ if clamp and any(abs(c - clamp[0]) > 1e-6 for c in clamp):+ raise SystemExit("swiglu_clamp_exp differs between layers %s: the engine takes one swiglu_limit" % clamp[:8])+ return out+++def ple_to_engram(kv: dict):+ """llama.cpp's own names for the Engram metadata (unsloth's GGUFs from 2026-10-19 on: ple.*, the hash heads' offsets+ counted across the layers' tables) -> the engram.* keys this pack reads (each layer's offsets from its own table)."""+ a = kv.get("general.architecture")+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731+ if g("engram.layers") or not g("ple.layers"):+ return+ layers, ngram, heads = list(g("ple.layers")), g("ple.ngram_size"), g("ple.heads_per_ngram")+ offs, sizes = list(g("ple.head_offsets")), list(g("ple.head_vocab_sizes"))+ cols = (ngram - 1) * heads+ if len(offs) != len(layers) * cols or len(sizes) != len(offs):+ raise SystemExit("ple: %d head offsets, %d sizes for %d layers x %d columns" %+ (len(offs), len(sizes), len(layers), cols))+ per, rows = [], []+ for i in range(len(layers)):+ o, s = offs[i * cols:(i + 1) * cols], sizes[i * cols:(i + 1) * cols]+ per += [x - o[0] for x in o]+ rows.append(o[-1] + s[-1] - o[0])+ for k, v in (("layers", layers), ("max_ngram_size", ngram), ("head_count", heads),+ ("head_dim", g("embedding_length_per_layer_input")), ("multipliers", list(g("ple.layer_multipliers"))),+ ("primes", sizes), ("offsets", per), ("num_embeddings", rows)):+ kv["%s.engram.%s" % (a, k)] = v+++def split_ple(g: Gguf, kv: dict):+ """llama.cpp's one Engram tensor (per_layer_token_embd: every layer's table, one after the other) -> each layer's+ own blk.N.engram_embd as the earlier GGUFs had it: the same bytes, read in place."""+ a = kv["general.architecture"]+ ple = [t for t in g.tensors if t[0] == "per_layer_token_embd.weight"]+ if not ple:+ return+ name, typ, dims, off = ple[0]+ layers, rows = kv.get("%s.engram.layers" % a, []), kv.get("%s.engram.num_embeddings" % a, [])+ if len(dims) != 2 or typ not in TYPES or len(rows) != len(layers) or sum(rows) != dims[1]:+ raise SystemExit("%s: %s %s does not split in the Engram layers %s of %s rows" % (g.path.name, name, dims, layers,+ rows))+ _, per, bpb = TYPES[typ]+ row = dims[0] // per * bpb+ parts, start = [], 0+ for il, n in zip(layers, rows):+ parts.append(("blk.%d.engram_embd.weight" % il, typ, [dims[0], n], off + start * row))+ start += n+ g.tensors = [t for t in g.tensors if t[0] != name] + parts+++def short(g: Gguf):+ """Why a GGUF file is shorter than its own tensor directory says (a download still running), or None."""+ need = g.data_start+ for _, typ, dims, off in g.tensors:+ if typ in TYPES:+ _, per, bpb = TYPES[typ]+ nel = 1+ for d in dims:+ nel *= d+ need = max(need, g.data_start + off + nel // per * bpb)+ have = g.path.stat().st_size+ return None if have >= need else "%s: %d of %d bytes" % (g.path.name, have, need)+++def engram_set(model: pathlib.Path, explicit: str | None):+ """The Engram tables' GGUF set for a quant that leaves them out: (its folder, its files read), or (None, []).+ explicit (--engram): that folder, whole; else the first whole one of ENGRAM_SETS beside the quant or in its+ folder - one not whole yet (its download still running) is passed over."""+ dirs = ([pathlib.Path(explicit).expanduser().resolve()] if explicit else+ [d for n in ENGRAM_SETS for d in (model / n, model.parent / n)])+ for d in dirs:+ paths = sorted(d.glob("*.gguf")) if d.is_dir() else []+ if not paths:+ if explicit:+ raise SystemExit("%s: no .gguf files (--engram: the folder of the Engram tables' GGUF set)" % d)+ continue+ try:+ egs = [Gguf(p) for p in paths]+ why = next((w for w in map(short, egs) if w), None)+ except (Exception, SystemExit) as e: # noqa: BLE001 - a header cut short raises anything+ why = str(e) or type(e).__name__+ if why is None:+ return d, egs+ if explicit:+ raise SystemExit("%s: not whole (%s)" % (d, why))+ print("%s: not whole (%s) - not used" % (d, why))+ return None, []+++def write_engram(out: pathlib.Path, kv: dict, vocab: int):+ a = kv["general.architecture"]+ g = lambda k, d=None: kv.get("%s.%s" % (a, k), d) # noqa: E731+ layers = list(g("engram.layers", []))+ if not layers:+ return 0, 0+ ngram, heads, hd = g("engram.max_ngram_size"), g("engram.head_count"), g("engram.head_dim")+ tm = list(g("engram.token_map"))+ mult, primes, offs = list(g("engram.multipliers")), list(g("engram.primes")), list(g("engram.offsets"))+ cols = (ngram - 1) * heads+ nl = len(layers)+ if len(mult) != nl * ngram or len(primes) != nl * cols or len(offs) != nl * cols:+ raise SystemExit("engram: %d multipliers, %d primes, %d offsets for %d layers x %d columns (ngram %d)" %+ (len(mult), len(primes), len(offs), nl, cols, ngram))+ pad = int(tm[g("engram.pad_token_id")])+ full = tm + [0] * max(0, vocab - len(tm))+ with out.open("wb") as f:+ f.write(struct.pack("<4s6i", b"DSE1", vocab, nl, ngram, heads, hd, pad))+ f.write(struct.pack("<%di" % vocab, *full[:vocab]))+ f.write(struct.pack("<%dq" % len(mult), *mult))+ f.write(struct.pack("<%dq" % len(primes), *primes))+ f.write(struct.pack("<%dq" % len(offs), *offs))+ return nl, cols+++def write_tokenizer(out: pathlib.Path, kv: dict, template: str | None):+ tokens = kv["tokenizer.ggml.tokens"]+ types = kv.get("tokenizer.ggml.token_type") or [1] * len(tokens)+ merges = kv.get("tokenizer.ggml.merges") or []+ if len(set(tokens)) != len(tokens):+ raise SystemExit("the GGUF's vocabulary has duplicate token strings")+ out.mkdir(parents=True, exist_ok=True)+ (out / "vocab.json").write_text(json.dumps({t: i for i, t in enumerate(tokens)}, ensure_ascii=False),+ encoding="utf-8")+ (out / "merges.txt").write_text("\n".join(merges), encoding="utf-8")+ (out / "token_type.json").write_text(json.dumps([int(t) for t in types]))+ special = {"bos": int(kv.get("tokenizer.ggml.bos_token_id", 0)), "eos": int(kv.get("tokenizer.ggml.eos_token_id", 1)),+ "pad": int(kv.get("tokenizer.ggml.padding_token_id", 2))}+ (out / "tokenizer.json").write_text(json.dumps({"pre": "deepseek3", "special_ids": special}, indent=1))+ if template:+ (out / "chat_template.jinja").write_text(template, encoding="utf-8")+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--model", required=True, help="the directory of the GGUF's shards")+ ap.add_argument("--out", help="default: <model>/maya-ds4")+ ap.add_argument("--vision", help="vision_pack.py's folder (default: <model>/../vision; taken when it is there)")+ ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")+ ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out "+ "(default: <model>/../Engram, when it is whole)")+ ap.add_argument("--shards", nargs="+", help="the GGUF's files in --model (default: every .gguf there - one "+ "model's; the setup names them: a folder may hold an earlier layout)")+ a = ap.parse_args()+ model = pathlib.Path(a.model).expanduser().resolve()+ out = pathlib.Path(a.out).expanduser() if a.out else model / "maya-ds4"+ shards = [model / n for n in a.shards] if a.shards else sorted(model.glob("*.gguf"))+ if not shards or not all(p.is_file() for p in shards):+ raise SystemExit("%s: no .gguf files" % model if not shards else+ "%s: not there" % next(p for p in shards if not p.is_file()))+ gs = [Gguf(p) for p in shards]+ kv = next((g.kv for g in gs if "general.architecture" in g.kv), None)+ if kv is None or kv["general.architecture"] not in ("deepseek41",):+ raise SystemExit("%s: not a DeepSeek-V4.1 GGUF (architecture %r)" % (model, kv and kv.get("general.architecture")))+ split = int(kv.get("split.count", 1))+ if split != len(shards):+ counts = sorted({m.group(1) for m in (re.search(r"-of-(\d{5})\.gguf$", p.name) for p in shards) if m})+ raise SystemExit("%s: %d of the %d shards are here" % (model, len(shards), split) if len(counts) < 2 else+ "%s: the shards of more than one GGUF split (-of-%s): name the model's own with --shards" %+ (model, ", -of-".join(counts)))+ ple_to_engram(kv)+ for g in gs:+ split_ple(g, kv)+ # the Engram tables may come from a shared set of GGUF files beside the quant (the same tensor names: engram_set) -+ # its engram tensors fill in the ones the quant's shards leave out, its engram metadata too. v1.1.16's Maya-S:+ # unsloth's UD-Q8_K_XL shard that holds only the tables, as it is (00008 of 9; before 2026-10-19 shards 00008 and+ # 00009 of 10, a table each), so no shard count is checked - every Engram layer's tables must be found, below.+ # Looked for only when the quant's shards leave some out.+ have = {t[0] for g in gs for t in g.tensors}+ arch = kv["general.architecture"]+ layers = kv.get("%s.engram.layers" % arch, [])+ lacking = not layers or any("blk.%d.%s.weight" % (il, part) not in have+ for il in layers for part in ("engram_embd", "engram_kv", "engram_gate"))+ eg_dir, egs = engram_set(model, a.engram) if lacking or a.engram else (None, [])+ eg_from = {}+ if eg_dir is not None:+ eshards = [g.path for g in egs]+ for g in egs:+ for k, v in g.kv.items():+ if ".engram." in k or ".ple." in k or k.endswith(".embedding_length_per_layer_input"):+ kv.setdefault(k, v)+ ple_to_engram(kv)+ for g in egs:+ split_ple(g, kv)+ for p, g in zip(eshards, egs):+ picked = [t for t in g.tensors if re.match(r"blk\.\d+\.engram_", t[0]) and t[0] not in have]+ if picked:+ g.tensors = picked+ eg_from[p] = g+ for il in kv.get("%s.engram.layers" % arch, []):+ for part in ("engram_embd", "engram_kv", "engram_gate"):+ nm = "blk.%d.%s.weight" % (il, part)+ if nm not in have and not any(t[0] == nm for g in eg_from.values() for t in g.tensors):+ raise SystemExit("%s: missing - not in the quant's shards nor in an Engram set (--engram, or "+ "<model>/../Engram/: unsloth's UD-Q8_K_XL shard of the Engram tables)" % nm)+ rel = lambda p: os.path.relpath(p, model).replace(os.sep, "/") # noqa: E731+ gs_all = gs + list(eg_from.values())+ names = [g.path.name for g in gs] + [rel(p) for p in eg_from]++ tensors, unknown = [], []+ for fi, g in enumerate(gs_all):+ for name, typ, dims, off in g.tensors:+ if typ not in TYPES:+ raise SystemExit("%s: %s has ggml type %d, which the engine does not read" % (g.path.name, name, typ))+ tname, per, bpb = TYPES[typ]+ nel = 1+ for d in dims:+ nel *= d+ if dims[0] % per:+ raise SystemExit("%s: %s rows of %d are not whole %s blocks" % (g.path.name, name, dims[0], tname))+ nbytes = nel // per * bpb+ m = re.match(r"blk\.(\d+)\.(.+)$", name)+ if m and m.group(2) in LAYER:+ ck = "layers.%s.%s" % (m.group(1), LAYER[m.group(2)])+ elif name in GLOBAL:+ ck = GLOBAL[name]+ else:+ unknown.append(name)+ continue+ shape = list(reversed(dims)) # ggml ne0 innermost -> torch order, outermost first+ base = g.data_start + off+ if ck.endswith(".attn.wo_a.slice"):+ # [o_groups * o_lora][in] -> o_groups entries of [o_lora][in] (group g: rows g * o_lora ..)+ n = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])+ if len(shape) != 2 or shape[0] % n:+ raise SystemExit("%s: %s %s does not split in %d groups" % (g.path.name, name, shape, n))+ part = nbytes // n+ for i in range(n):+ tensors.append(("%s.%d.weight" % (ck, i), tname, fi, base + i * part, part,+ [shape[0] // n, shape[1]]))+ continue+ if ".ffn.experts." in ck:+ # [n_exp][out][in] -> an entry an expert: layers.N.ffn.experts.E.w1.weight+ n, part = shape[0], nbytes // shape[0]+ pre, role = ck.rsplit(".", 1)+ for i in range(n):+ tensors.append(("%s.%d.%s.weight" % (pre, i, role), tname, fi, base + i * part, part, shape[1:]))+ continue+ tensors.append((ck, tname, fi, base, nbytes, shape))+ if unknown:+ raise SystemExit("tensors the pack does not map (%d): %s" % (len(unknown), ", ".join(unknown[:12])))+ # the DSpark drafter (drafter_pack.py's dspark.bin beside the quants' folders, from DeepSeek's checkpoint): its+ # three blocks as mtp.0-2, so the engine drafts with them+ files = names+ if eg_from:+ print("Engram tables from %s (%d shards)" % (eg_dir, len(eg_from)))+ drafter = pathlib.Path(a.drafter).expanduser().resolve() if a.drafter else model.parent / "dspark" / "dspark.json"+ if a.drafter and not drafter.is_file():+ raise SystemExit("%s: no drafter there" % drafter)+ drafter_in = drafter.is_file() and drafter.with_suffix(".bin").is_file()+ if drafter_in:+ files.append(rel(drafter.with_suffix(".bin")))+ groups = int(kv["%s.attention.output_group_count" % kv["general.architecture"]])+ for name, dt, off, nb, shape in json.loads(drafter.read_text())["tensors"]:+ if name.endswith(".attn.wo_a.weight"): # (one entry a group, as the trunk's)+ for i in range(groups):+ tensors.append(("%s.slice.%d.weight" % (name[:-len(".weight")], i), dt, len(files) - 1,+ off + i * (nb // groups), nb // groups, [shape[0] // groups, shape[1]]))+ continue+ tensors.append((name, dt, len(files) - 1, off, nb, shape))+ # the vision tower (vision_pack.py's vision.bin, from DeepSeek's checkpoint): its tensors under the checkpoint's+ # names, the picture span's marker rows, and the geometry config.json gives them+ vdir = pathlib.Path(a.vision).expanduser().resolve() if a.vision else model.parent / "vision"+ vision = {}+ if (vdir / "vision.json").is_file() and (vdir / "vision.bin").is_file():+ try:+ rel = os.path.relpath(vdir / "vision.bin", model).replace(os.sep, "/")+ except ValueError: # (Windows: another drive)+ raise SystemExit("%s: the vision folder must be on the model's drive" % vdir)+ if " " in rel:+ raise SystemExit("%s: the vision folder's path relative to the model has a space" % vdir)+ files.append(rel)+ info = json.loads((vdir / "vision.json").read_text())+ for name, dt, off, nb, shape in info["tensors"]:+ tensors.append((name, dt, len(files) - 1, off, nb, shape))+ vision = info["config"]+ vocab = next(t[5][0] for t in tensors if t[0] == "embed.weight")+ geo = geometry(kv, vocab)++ out.mkdir(parents=True, exist_ok=True)+ with (out / "ds4.txt").open("w") as f:+ for k, v in geo.items():+ if v is None:+ continue+ if isinstance(v, bool):+ v = int(v)+ f.write("%s %s\n" % (k, ",".join(str(x) for x in v) if isinstance(v, list) else v))+ for k, v in vision.items():+ f.write("%s %s\n" % (k, v))+ f.write("format gguf\n")+ f.write("model_dir %s\n" % model)+ with (out / "tensors.txt").open("w") as f:+ for i, name in enumerate(files):+ f.write("file %d %s\n" % (i, name))+ for name, dt, fi, off, nb, shape in tensors:+ f.write("t %s %s %d %d %d %s\n" % (name, dt, fi, off, nb, " ".join(map(str, shape))))+ nl, cols = write_engram(out / "engram.bin", kv, vocab)+ # the server's chat template (pack.py's, so both formats chat alike); the GGUF's own when it is not there+ template = pathlib.Path(__file__).resolve().parent / "chat_template.jinja"+ write_tokenizer(out / "tokenizer", kv,+ template.read_text(encoding="utf-8") if template.exists() else kv.get("tokenizer.chat_template"))+ by = {}+ for t in tensors:+ by[t[1]] = by.get(t[1], 0) + t[4]+ extras = [w for w, there in (("the DSpark drafter", drafter_in), ("the vision tower", bool(vision))) if there]+ print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %+ (model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),+ nl, cols, "".join(", " + w for w in extras), out))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/recipe_maya_s_test.json b/tools/dsv41/recipe_maya_s_test.jsonnew file mode 100644index 000000000..320ff5e75--- /dev/null+++ b/tools/dsv41/recipe_maya_s_test.json@@ -0,0 +1,13 @@+{+ "comment": "a quick Maya-S-like test model (~200 GB): ggml's IQ2_XXS quantizer runs at ~1 M weights/s a thread (a full Maya-S takes ~8 h on 12 threads), so most layers take Q2_K / Q3_K (20-80 M/s) and a few layers each low-bit type the engine reads, on real weights",+ "experts": {"gate": "Q2_K", "up": "Q2_K", "down": "Q3_K"},+ "layers": [+ {"layers": "10", "gate": "IQ2_XXS", "up": "IQ2_XXS", "down": "IQ3_XXS"},+ {"layers": "20", "gate": "IQ4_XS", "up": "IQ4_XS", "down": "Q4_K"},+ {"layers": "25", "gate": "IQ3_XXS", "up": "IQ3_XXS", "down": "IQ3_S"},+ {"layers": "30", "gate": "IQ2_S", "up": "IQ2_S", "down": "IQ3_S"}+ ],+ "dense": "Q8_0",+ "dense_keep": ["hc_", "engram", "norm", "attn_sinks", "exp_probs_b", "ffn_gate_inp"],+ "engram": "skip"+}diff --git a/tools/dsv41/ref.py b/tools/dsv41/ref.pynew file mode 100644index 000000000..9659aa8b7--- /dev/null+++ b/tools/dsv41/ref.py@@ -0,0 +1,422 @@+"""tools/ds4/ref.py - a whole-sequence DeepSeek-V4.1 forward in torch over an EXL3 checkpoint (the engine's oracle).++ python tools/ds4/ref.py --model ~/models/DSV4 --text "The capital of France is" [--dump out.npz] [--gen 8]++It follows DeepSeek's inference/model.py (MIT) at start_pos 0 - the prompt path, every position at once - with each+layer's weights decoded from EXL3 when the layer runs (tools/ds4/exl3_torch.py) and freed after it, so one GPU of+24 GB carries the 211 GB model: Hyper-Connections with the lagged pre-mix, sliding-window plus compressed sparse+attention with the two-level indexer and its candidate blocks, the cross-layer KV / indexer sharing, Engram, the+sqrt-softplus router, and the FP8 / FP4 activation rounding of the reference kernels (simulated, as their `inplace`+paths do). Activations are F32 throughout (the reference keeps BF16 between ops).++--dump writes every layer's residual stream ([T, hc, dim] F32 after each block), the logits and the token ids, for+the engine's parity checks.+"""+from __future__ import annotations++import argparse+import json+import math+import pathlib+import sys+import time++import numpy as np+import torch+import torch.nn.functional as F++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import engram_hash as EH # noqa: E402+import exl3_torch as ET # noqa: E402+from st import SafeTensors # noqa: E402+++# ------------------------------------------------------------------ the reference kernels' roundings+def fp8_round(x: torch.Tensor, block: int = 32) -> torch.Tensor:+ """act_quant(..., scale_fmt='ue8m0', inplace=True): per `block` values a power-of-two scale, E4M3 values."""+ sh = x.shape+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)+ amax = v.abs().amax(-1, keepdim=True).clamp_min(1e-4)+ s = torch.exp2(torch.ceil(torch.log2(amax / 448.0)))+ y = (v / s).clamp(-448.0, 448.0).to(torch.float8_e4m3fn).float() * s+ return y.reshape(sh)+++_FP4 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])+++def _e2m1(v: torch.Tensor) -> torch.Tensor:+ """Round to the E2M1 grid, nearest, ties to the even mantissa (values already in [-6, 6])."""+ a = v.abs()+ g = _FP4.to(v.device)+ d = (a.unsqueeze(-1) - g).abs()+ best = d.argmin(-1)+ # ties: argmin takes the lower index; the even mantissa is the lower one at 0.25, 1.25, 2.5, 5 and the upper one+ # at 0.75, 1.75, 3.5 (grid indices: mantissa bit = index & 1 for index >= 1, 0 at index 0)+ lo = best+ hi = (best + 1).clamp(max=7)+ tie = (d.gather(-1, lo.unsqueeze(-1)).squeeze(-1) == d.gather(-1, hi.unsqueeze(-1)).squeeze(-1)) & (hi != lo)+ take_hi = tie & ((lo & 1) == 1)+ idx = torch.where(take_hi, hi, lo)+ return torch.copysign(g[idx], v)+++def fp4_round(x: torch.Tensor, block: int, e4m3_scale: bool) -> torch.Tensor:+ """fp4_act_quant(..., inplace=True): E2M1 values with an E8M0 (power of two) or an E4M3 scale per block."""+ sh = x.shape+ v = x.float().reshape(*sh[:-1], sh[-1] // block, block)+ amax = v.abs().amax(-1, keepdim=True)+ if e4m3_scale:+ amax = amax.clamp_min(6 * 2.0 ** -9)+ s = (amax / 6.0).to(torch.float8_e4m3fn).float()+ else:+ amax = amax.clamp_min(6 * 2.0 ** -126)+ s = torch.exp2(torch.ceil(torch.log2(amax / 6.0)))+ y = _e2m1((v / s).clamp(-6.0, 6.0)) * s+ return y.reshape(sh)+++def rms(x: torch.Tensor, w: torch.Tensor | None, eps: float) -> torch.Tensor:+ x = x.float()+ y = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps)+ return y * w if w is not None else y+++def freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow, dev) -> torch.Tensor:+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ if original_seq_len > 0:+ def corr(rot):+ return dim * math.log(original_seq_len / (rot * 2 * math.pi)) / (2 * math.log(base))+ low = max(math.floor(corr(beta_fast)), 0)+ high = min(math.ceil(corr(beta_slow)), dim - 1)+ ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1)+ smooth = 1 - ramp+ freqs = freqs / factor * (1 - smooth) + freqs * smooth+ f = torch.outer(torch.arange(seqlen, dtype=torch.float32), freqs)+ return torch.polar(torch.ones_like(f), f).to(dev)+++def rope(x: torch.Tensor, fc: torch.Tensor, inverse: bool = False) -> torch.Tensor:+ """Adjacent pairs as complex numbers. x [T, d] or [T, h, d]; fc [T, d/2]."""+ c = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2).contiguous())+ if inverse:+ fc = fc.conj()+ if x.ndim == 3:+ fc = fc[:, None, :]+ return torch.view_as_real(c * fc).flatten(-2)+++# ------------------------------------------------------------------ the model+class Ref:+ def __init__(self, model: pathlib.Path, dev: str = "cuda:0", max_seq: int = 8192):+ self.dir = model+ self.cfg = json.loads((model / "config.json").read_text())+ self.t = t = self.cfg["text_config"]+ self.st = SafeTensors(model)+ self.dev = dev+ self.dim, self.hc = t["hidden_size"], t["hc_mult"]+ self.eps = t["rms_norm_eps"]+ self.nl = t["num_hidden_layers"]+ self.ratios = t["compress_ratios"]+ self.kv_src = set(t["kv_source_layer_ids"])+ self.ix_src = set(t["index_source_layer_ids"])+ self.cand_src = t["candidate_source_layer_id"]+ rs = t["rope_scaling"]+ rd = t["qk_rope_head_dim"]+ self.fc_window = freqs_cis(rd, max_seq, 0, t["rope_theta"], rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)+ self.fc_comp = freqs_cis(rd, max_seq, rs["original_max_position_embeddings"], t["compress_rope_theta"],+ rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)+ self.lay = EH.Layout.from_config(t)+ tm, n = EH.compressed_token_map(model / "tokenizer.json")+ if n != t["engram_compressed_vocab_size"]:+ raise SystemExit("engram: compressed vocab %d, the config says %d" % (n, t["engram_compressed_vocab_size"]))+ self.hasher = EH.Hasher(self.lay, tm, EH.multipliers(self.lay, n), tm[t["engram_pad_token_id"]])+ self.timing: dict[str, float] = {}++ # weights+ def W(self, prefix: str) -> torch.Tensor:+ return ET.weight(self.st, prefix, self.dev)++ def V(self, name: str) -> torch.Tensor:+ return torch.from_numpy(np.array(self.st.f32(name))).to(self.dev)++ # pieces+ def hc_mixes(self, x, fn, scale, base):+ hc = self.hc+ xf = x.flatten(1).float()+ r = torch.rsqrt(xf.square().mean(-1, keepdim=True) + self.eps)+ mixes = (xf @ fn.T) * r # [T, (2 + hc) hc]+ pre = torch.sigmoid(mixes[:, :hc] * scale[0] + base[:hc]) + self.t["hc_eps"]+ post = 2 * torch.sigmoid(mixes[:, hc:2 * hc] * scale[1] + base[hc:2 * hc])+ comb = (mixes[:, 2 * hc:] * scale[2] + base[2 * hc:]).view(-1, hc, hc)+ eps = self.t["hc_eps"]+ comb = comb.softmax(-1) + eps+ comb = comb / (comb.sum(-2, keepdim=True) + eps)+ for _ in range(self.t["hc_sinkhorn_iters"] - 1):+ comb = comb / (comb.sum(-1, keepdim=True) + eps)+ comb = comb / (comb.sum(-2, keepdim=True) + eps)+ return pre, post, comb++ @staticmethod+ def hc_pre(x, mix): # [T, hc, d], [T, hc] -> [T, d]+ return (mix.unsqueeze(-1) * x).sum(1)++ @staticmethod+ def hc_post(y, res, post, comb): # y [T, d], res [T, hc, d]+ return post.unsqueeze(-1) * y.unsqueeze(1) + (comb.unsqueeze(-1) * res.unsqueeze(2)).sum(1)++ def engram(self, li: int, x: torch.Tensor, hash_ids: np.ndarray) -> torch.Tensor:+ """x [T, hc, d]; hash_ids [T, cols] absolute rows."""+ P = "layers.%d.engram." % li+ T, cols = hash_ids.shape+ flat = hash_ids.reshape(-1)+ wt = self.st.raw(P + "embed.weight").view(np.uint8).reshape(-1, self.lay.head_dim)+ sc = self.st.raw(P + "embed.scale").view(np.uint8).reshape(-1, self.lay.head_dim // 32)+ order = np.argsort(flat) # sequential reads+ rows = np.empty((flat.size, self.lay.head_dim), np.uint8)+ srow = np.empty((flat.size, self.lay.head_dim // 32), np.uint8)+ rows[order] = wt[flat[order]]+ srow[order] = sc[flat[order]]+ v = torch.from_numpy(rows).to(self.dev).view(torch.float8_e4m3fn).float()+ s = torch.from_numpy(srow).to(self.dev).view(torch.float8_e8m0fnu).float()+ e = (v.view(-1, self.lay.head_dim // 32, 32) * s.unsqueeze(-1)).view(T, cols * self.lay.head_dim)+ e = e.to(torch.bfloat16).float() # ParallelEngramEmbedding returns BF16+ kv = e @ self.W(P + "wkv").T # [T, (hc + 1) d]+ d, hc = self.dim, self.hc+ key, value = kv[:, :hc * d].view(T, hc, d), kv[:, hc * d:]+ w = self.V(P + "q_weight") * self.V(P + "k_weight") # [hc, d]+ h = x.float()+ rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)+ dot = (h * w * key).sum(-1) * rstd * d ** -0.5+ gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))+ return h + gate.unsqueeze(-1) * value.unsqueeze(1)++ def sparse_attn(self, q, kv, sink, idx, scale):+ """q [T, H, D], kv [N, D], sink [H], idx [T, K] int (-1 none) -> [T, H, D]."""+ T, H, D = q.shape+ out = torch.empty_like(q)+ for a in range(0, T, 256):+ b = min(T, a + 256)+ ii = idx[a:b]+ valid = ii >= 0+ g = kv[ii.clamp_min(0)] # [t, K, D]+ s = torch.einsum("thd,tkd->thk", q[a:b], g) * scale+ s = s.masked_fill(~valid.unsqueeze(1), float("-inf"))+ m = torch.maximum(s.amax(-1), sink[None, :]) # [t, H]+ m = torch.maximum(m, torch.full_like(m, -1e30))+ p = torch.exp(s - m.unsqueeze(-1))+ den = p.sum(-1) + torch.exp(sink[None, :] - m)+ out[a:b] = torch.einsum("thk,tkd->thd", p, g) / den.unsqueeze(-1)+ return out++ @staticmethod+ def candidates(logits, compress_lens, topk_blocks, block):+ width = logits.size(-1)+ scores = F.pad(logits, (0, -width % block), value=float("-inf"))+ scores = scores.unflatten(-1, (-1, block)).amax(-1)+ nb = scores.size(-1)+ last = (compress_lens - 1) // block+ scores = scores.masked_fill(torch.arange(nb, device=logits.device) == last, float("inf"))+ top = scores.topk(min(topk_blocks, nb), dim=-1)+ keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > float("-inf"))+ return keep.repeat_interleave(block, dim=-1)[..., :width]++ def attention(self, li: int, x: torch.Tensor, S: dict) -> torch.Tensor:+ t = self.t+ A = ("layers.%d.attn." % li) if li < self.nl else ("mtp.%d.attn." % (li - self.nl))+ T = x.shape[0]+ H, D, rd = t["num_attention_heads"], t["head_dim"], t["qk_rope_head_dim"]+ ratio = self.ratios[li]+ fc = (self.fc_comp if ratio else self.fc_window)[:T]+ qr = rms(x @ self.W(A + "wq_a").T, self.V(A + "q_norm.weight"), self.eps)+ q = (qr @ self.W(A + "wq_b").T).view(T, H, D)+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)+ kv = rms(x @ self.W(A + "wkv").T, self.V(A + "kv_norm.weight"), self.eps)+ kv = torch.cat([kv[:, :-rd], rope(kv[:, -rd:], fc)], -1)+ kv = fp8_round(kv)+ win = t["sliding_window"]+ end = torch.arange(T, device=self.dev).unsqueeze(1)+ widx = (end - win + 1).clamp(0) + torch.arange(min(T, win), device=self.dev)+ widx = torch.where(widx > end, -1, widx)+ all_kv, all_idx = kv, widx+ if ratio:+ is_src = li < self.nl and li in self.kv_src+ latent = None+ if is_src:+ C = A + "compressor."+ if ratio == 1:+ latent = rms(x @ self.W(C + "wkv").T, self.V(C + "norm.weight"), self.eps)+ elif T >= ratio:+ cut = T - T % ratio+ kvc = x[:cut] @ self.W(C + "wkv").T+ sc = x[:cut] @ self.W(C + "wgate").T+ kvc, sc = kvc.view(-1, ratio, D), sc.view(-1, ratio, D)+ latent = rms((kvc * sc.softmax(1)).sum(1), self.V(C + "norm.weight"), self.eps)+ # the indexer (index sources) before the latent is rotated+ if li < self.nl and li in self.ix_src:+ S["topk"] = self.indexer(li, A, x, qr, latent, ratio, T, fc, S)+ if latent is not None:+ lat = torch.cat([latent[:, :-rd], rope(latent[:, -rd:], fc[:T - T % ratio:ratio])], -1)+ S["ckv"] = fp4_round(lat, 16, True)+ ckv = S["ckv"][:T // ratio] if S.get("ckv") is not None else kv.new_zeros(0, D)+ all_kv = torch.cat([kv, ckv], 0)+ all_idx = torch.cat([widx, S["topk"]], -1)+ o = self.sparse_attn(q, all_kv, self.V(A + "attn_sink"), all_idx, D ** -0.5)+ o = torch.cat([o[..., :-rd], rope(o[..., -rd:], fc, inverse=True)], -1)+ G = t["o_groups"]+ o = fp8_round(o.reshape(T, G, -1))+ parts = [o[:, g] @ self.W(A + "wo_a.slice.%d" % g).T for g in range(G)]+ return torch.cat(parts, -1) @ self.W(A + "wo_b").T++ def indexer(self, li, A, x, qr, latent, ratio, T, fc, S):+ t = self.t+ I = A + "indexer."+ nh, hd, rd = t["index_n_heads"], t["index_head_dim"], t["qk_rope_head_dim"]+ clen = T // ratio+ if clen == 0:+ return torch.empty(T, 0, dtype=torch.long, device=self.dev)+ if li in self.kv_src and latent is not None: # this layer owns the index keys+ k = rms(latent @ self.W(I + "wk").T, self.V(I + "k_norm.weight"), self.eps)+ k = torch.cat([k[:, :-rd], rope(k[:, -rd:], fc[:T - T % ratio:ratio])], -1)+ S["ik"] = fp4_round(k, 32, False)+ q = (qr @ self.W(I + "wq_b").T).view(T, nh, hd)+ q = torch.cat([q[..., :-rd], rope(q[..., -rd:], fc)], -1)+ q = fp4_round(q, 32, False)+ ik = S["ik"][:clen]+ w = (x @ self.V(I + "weights_proj.weight").T) * (hd ** -0.5 * nh ** -0.5) # [T, nh]+ score = torch.zeros(T, clen, device=self.dev)+ for a in range(0, T, 512):+ b = min(T, a + 512)+ s = torch.einsum("thd,kd->thk", q[a:b], ik).relu_()+ score[a:b] = (s * w[a:b].unsqueeze(-1)).sum(1)+ lens = (torch.arange(1, T + 1, device=self.dev) // ratio).unsqueeze(-1)+ score = score.masked_fill(torch.arange(clen, device=self.dev) >= lens, float("-inf"))+ if li == self.cand_src:+ S["cand"] = self.candidates(score, lens, t["candidate_topk_blocks"], t["candidate_block_size"])+ elif 0 <= self.cand_src < li:+ score = score.masked_fill(~S["cand"], float("-inf"))+ topk = min(t["index_topk"], clen)+ idx = score.topk(topk, dim=-1, sorted=False).indices.sort(-1).values+ return torch.where(idx < lens, idx + T, -1)++ def moe(self, li: int, x: torch.Tensor) -> torch.Tensor:+ t = self.t+ M = "layers.%d.ffn." % li+ T = x.shape[0]+ gw = self.V(M + "gate.weight")+ scores = F.softplus(x.float() @ gw.T).sqrt()+ idx = (scores + self.V(M + "gate.bias")).topk(t["num_experts_per_tok"], dim=-1)[1]+ wts = scores.gather(1, idx)+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * t["routed_scaling_factor"]+ lim = t["swiglu_limit"]+ y = torch.zeros(T, self.dim, device=self.dev)++ def expert(P, xs):+ g = (xs @ self.W(P + "w1").T).clamp(max=lim)+ u = (xs @ self.W(P + "w3").T).clamp(-lim, lim)+ return F.silu(g) * u, P++ uniq = torch.unique(idx).tolist()+ for a in range(0, len(uniq), 24): # 24 experts decoded at once+ batch = uniq[a:a + 24]+ Ps = [M + "experts.%d." % e for e in batch]+ w1 = ET.decode_many(self.st, [P + "w1" for P in Ps], self.dev)+ w3 = ET.decode_many(self.st, [P + "w3" for P in Ps], self.dev)+ w2 = ET.decode_many(self.st, [P + "w2" for P in Ps], self.dev)+ for i, e in enumerate(batch):+ rows, slot = torch.where(idx == e)+ xs = x[rows]+ h = F.silu((xs @ w1[i].T).clamp(max=lim)) * (xs @ w3[i].T).clamp(-lim, lim)+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ w2[i].T)+ del w1, w2, w3+ h, _ = expert(M + "shared_experts.", x)+ y += h @ self.W(M + "shared_experts.w2").T+ self.last_route = (idx, wts)+ return y++ @torch.inference_mode()+ def forward(self, ids: list[int], dump: list | None = None) -> torch.Tensor:+ T = len(ids)+ hashes = self.hasher.ids(ids) # [T, n_engram, cols]+ emb = self.st.raw("embed.weight").view(np.uint16).reshape(-1, self.dim)+ h = torch.from_numpy(np.array(emb[np.asarray(ids)])).to(self.dev).view(torch.bfloat16).float()+ h = h.unsqueeze(1).repeat(1, self.hc, 1)+ pre = torch.zeros(T, self.hc, device=self.dev)+ pre[:, 0] = 1.0+ S: dict = {}+ for li in range(self.nl):+ t0 = time.time()+ B = "layers.%d." % li+ if li in self.lay.layer_ids:+ h = self.engram(li, h, hashes[:, self.lay.layer_ids.index(li)])+ res = h+ a_pre, a_post, a_comb = self.hc_mixes(h, self.V(B + "hc_attn_fn"), self.V(B + "hc_attn_scale"),+ self.V(B + "hc_attn_base"))+ x = rms(self.hc_pre(h, pre), self.V(B + "attn_norm.weight"), self.eps)+ h = self.hc_post(self.attention(li, x, S), res, a_post, a_comb)+ res = h+ f_pre, f_post, f_comb = self.hc_mixes(h, self.V(B + "hc_ffn_fn"), self.V(B + "hc_ffn_scale"),+ self.V(B + "hc_ffn_base"))+ x = rms(self.hc_pre(h, a_pre), self.V(B + "ffn_norm.weight"), self.eps)+ h = self.hc_post(self.moe(li, x), res, f_post, f_comb)+ pre = f_pre+ if dump is not None:+ dump.append(h.cpu().numpy())+ self.timing[li] = time.time() - t0+ x = rms(self.hc_pre(h, pre), self.V("norm.weight"), self.eps)+ head = self.W("head")+ return x @ head.T # [T, vocab]+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--model", default=str(pathlib.Path("~/models/DSV4").expanduser()))+ ap.add_argument("--text", help="raw text (tokenized as is, no BOS added)")+ ap.add_argument("--ids", help="comma-separated token ids")+ ap.add_argument("--chat", help="a user message, as DeepSeek-V4.1's chat (non-thinking) prompt")+ ap.add_argument("--gen", type=int, default=0, help="greedy tokens to append (a whole forward each)")+ ap.add_argument("--dump", help="write the per-layer residual streams, the logits and the ids (.npz)")+ ap.add_argument("--no-hs", action="store_true", help="--dump without the per-layer residual streams")+ ap.add_argument("--dev", default="cuda:0")+ a = ap.parse_args()+ model = pathlib.Path(a.model).expanduser()+ from tokenizers import Tokenizer+ tok = Tokenizer.from_file(str(model / "tokenizer.json"))+ if a.ids:+ ids = [int(x) for x in a.ids.split(",")]+ elif a.chat:+ ids = tok.encode("<|begin▁of▁sentence|><|User|>" + a.chat + "<|Assistant|></think>",+ add_special_tokens=False).ids+ else:+ ids = tok.encode(a.text or "The capital of France is", add_special_tokens=False).ids+ r = Ref(model, a.dev)+ print("tokens:", len(ids), ids[:32], flush=True)+ dump = [] if a.dump and not a.no_hs else None+ t0 = time.time()+ logits = r.forward(ids, dump)+ print("forward %.1fs" % (time.time() - t0), flush=True)+ lp = torch.log_softmax(logits.float(), -1)+ if len(ids) > 1:+ tgt = torch.tensor(ids[1:], device=lp.device)+ nll = -lp[:-1].gather(1, tgt[:, None]).squeeze(1)+ print("ppl over %d tokens: %.4f" % (len(ids) - 1, math.exp(nll.mean().item())))+ top = lp[-1].topk(8)+ print("next:", [(tok.decode([i]), round(v, 3)) for v, i in zip(top.values.tolist(), top.indices.tolist())])+ if a.dump:+ np.savez(a.dump, ids=np.array(ids), logits=logits.cpu().numpy(),+ hs=np.stack(dump) if dump else np.zeros(0))+ out = list(ids)+ for _ in range(a.gen):+ nxt = int(lp[-1].argmax())+ out.append(nxt)+ print("gen:", repr(tok.decode(out[len(ids):], skip_special_tokens=False)), flush=True)+ if nxt == 1:+ break+ logits = r.forward(out)+ lp = torch.log_softmax(logits.float(), -1)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/ref_gguf.py b/tools/dsv41/ref_gguf.pynew file mode 100644index 000000000..60e8abf85--- /dev/null+++ b/tools/dsv41/ref_gguf.py@@ -0,0 +1,301 @@+"""tools/ds4/ref_gguf.py - tools/ds4/ref.py's oracle over a GGUF (tools/ds4/pack_gguf.py's pack): the same forward,+each weight dequantised in torch from the GGUF's own blocks (BF16, Q8_0, MXFP4, MXFP8) - a check of maya-ds4's GGUF+path that shares none of its code.++ python tools/ds4/ref_gguf.py --pack <model>/maya-ds4 --ids 0,671,6102 [--dump out.npz] [--gen 8]++The token ids come from the server's tokenizer (maya-ds4 --serve, or the engine's own --ids runs); --dump writes+what tools/ds4/compare.py reads.+"""+from __future__ import annotations++import argparse+import math+import os+import pathlib+import struct+import sys+import time+from types import SimpleNamespace++import numpy as np+import torch+import torch.nn.functional as F++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import ref as R # noqa: E402++BLOCK = {"Q8_0": 34, "MXFP4": 17, "MXFP8": 33}+E2M1 = torch.tensor([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=torch.float32)+++class Pack:+ """pack_gguf.py's tensors.txt over the GGUF's files (memory-mapped), and ds4.txt's geometry."""++ def __init__(self, d: pathlib.Path):+ self.kv = {}+ for line in (d / "ds4.txt").read_text().splitlines():+ k, _, v = line.partition(" ")+ self.kv[k] = v+ model = pathlib.Path(self.kv["model_dir"])+ self.mm, self.t = {}, {}+ for line in (d / "tensors.txt").read_text().splitlines():+ f = line.split()+ if f[0] == "file":+ self.mm[int(f[1])] = np.memmap(model / f[2], dtype=np.uint8, mode="r")+ elif f[0] == "t":+ self.t[f[1]] = (f[2], int(f[3]), int(f[4]), int(f[5]), tuple(int(x) for x in f[6:]))+ self._embed = None++ def has(self, name):+ return name in self.t++ def raw(self, name) -> np.ndarray:+ dt, fi, off, nb, _ = self.t[name]+ return self.mm[fi][off:off + nb]++ def num(self, k, d=0.0):+ return float(self.kv.get(k, d))++ def ints(self, k):+ return [int(x) for x in self.kv.get(k, "").split(",") if x]+++def dequant(p: Pack, name: str, dev) -> torch.Tensor:+ """A tensor as F32 on dev, shaped as stored ([out][in] for a matrix)."""+ dt, _, _, _, shape = p.t[name]+ r = torch.from_numpy(np.array(p.raw(name))).to(dev)+ if dt == "F32":+ v = r.view(torch.float32)+ elif dt == "F16":+ v = r.view(torch.float16).float()+ elif dt == "BF16":+ v = r.view(torch.bfloat16).float()+ elif dt == "Q8_0":+ b = r.view(-1, 34)+ v = (b[:, :2].contiguous().view(torch.float16).float() * b[:, 2:].view(torch.int8).float()).reshape(-1)+ elif dt == "MXFP4":+ b = r.view(-1, 17)+ s = torch.ldexp(torch.ones(1, device=dev), b[:, :1].int() - 128) # 2^(e - 128)+ q = b[:, 1:].long()+ lut = E2M1.to(dev)+ v = (torch.cat([lut[q & 15], lut[q >> 4]], 1) * s).reshape(-1) # values 0..15, then 16..31+ elif dt == "MXFP8":+ b = r.view(-1, 33)+ s = torch.ldexp(torch.ones(1, device=dev), b[:, :1].int() - 127)+ v = (b[:, 1:].contiguous().view(torch.float8_e4m3fn).float() * s).reshape(-1)+ else: # the low-bit expert types (IQ2_XXS .. Q4_K): gguf-py's numpy dequantisers (ggml's dequantize_row_*)+ v = torch.from_numpy(gq_values(np.array(p.raw(name)), dt, name)).to(dev)+ return v.reshape(shape)+++def gq_values(raw: np.ndarray, dt: str, name: str) -> np.ndarray:+ """ggml blocks of type dt as F32 (flat) through gguf-py's dequantisers - the installed package, else a llama.cpp+ checkout's ($GGUF_PY, ~/llama.cpp/gguf-py)."""+ try:+ try:+ import gguf+ except ImportError:+ for d in (os.environ.get("GGUF_PY", ""), os.path.expanduser("~/llama.cpp/gguf-py")):+ if d and os.path.isdir(d):+ sys.path.insert(0, d)+ import gguf+ from gguf.quants import dequantize as gq_dequant+ qt = gguf.GGMLQuantizationType[dt]+ except (ImportError, KeyError):+ raise SystemExit("%s: dtype %s (no gguf-py with a dequantiser for it: GGUF_PY=<llama.cpp>/gguf-py)" % (name, dt))+ return np.ascontiguousarray(gq_dequant(raw, qt), dtype=np.float32).reshape(-1)+++class Hasher:+ """engram_hash.Hasher from the pack's engram.bin (the token map, multipliers, primes and offsets)."""++ def __init__(self, path: pathlib.Path):+ b = path.read_bytes()+ vocab, nl, ngram, heads, hd, pad = struct.unpack_from("<6i", b, 4)+ cols = (ngram - 1) * heads+ o = 28+ self.tm = np.frombuffer(b, np.int32, vocab, o).astype(np.int64)+ o += 4 * vocab+ self.mult = np.frombuffer(b, np.int64, nl * ngram, o).reshape(nl, ngram)+ o += 8 * nl * ngram+ self.primes = np.frombuffer(b, np.int64, nl * cols, o).reshape(nl, ngram - 1, heads)+ o += 8 * nl * cols+ self.off = np.frombuffer(b, np.int64, nl * cols, o).reshape(nl, cols)+ self.ngram, self.pad, self.hd = ngram, pad, hd++ def ids(self, tokens) -> np.ndarray: # engram_hash.Hasher.ids+ c = self.tm[np.asarray(tokens, dtype=np.int64)]+ T = len(c)+ pos = np.arange(T)+ toks, blocked = [], np.zeros(T, dtype=bool)+ for s in range(self.ngram):+ src = c[np.clip(pos - s, 0, None)]+ blocked = blocked | (pos < s)+ toks.append(np.where(blocked, self.pad, src))+ toks = np.stack(toks, axis=-1)+ prod = toks[:, None, :] * self.mult[None]+ rolling, out = prod[..., 0], []+ for i in range(1, self.ngram):+ rolling = np.bitwise_xor(rolling, prod[..., i])+ out.append(rolling[..., None] % self.primes[None, :, i - 1])+ return np.concatenate(out, axis=-1) + self.off[None]+++def q6k_rows(rows: np.ndarray, name: str) -> np.ndarray:+ """Engram rows of Q6_K superblocks as F32 (ggml's dequantize_row_q6_K, through gguf-py)."""+ return gq_values(rows.reshape(-1), "Q6_K", name)+++class RefGguf(R.Ref):+ def __init__(self, pack: pathlib.Path, dev: str = "cuda:0", max_seq: int = 8192):+ self.dir = pack+ self.st = p = Pack(pack)+ g = p.num+ t = {k: int(g(k)) for k in ("hidden_size", "hc_mult", "num_hidden_layers", "candidate_source_layer_id",+ "qk_rope_head_dim", "num_experts_per_tok", "sliding_window", "o_groups",+ "num_attention_heads", "index_topk", "index_n_heads", "index_head_dim", "head_dim",+ "candidate_topk_blocks", "candidate_block_size")}+ t.update(hc_eps=g("hc_eps"), hc_sinkhorn_iters=int(g("hc_sinkhorn_iters")),+ rms_norm_eps=g("rms_norm_eps"), routed_scaling_factor=g("routed_scaling_factor"),+ swiglu_limit=g("swiglu_limit"), rope_theta=g("rope_theta"), compress_rope_theta=g("compress_rope_theta"),+ compress_ratios=p.ints("compress_ratios"), kv_source_layer_ids=p.ints("kv_source_layer_ids"),+ index_source_layer_ids=p.ints("index_source_layer_ids"),+ rope_scaling={"factor": g("rope_scaling.factor"), "beta_fast": g("rope_scaling.beta_fast"),+ "beta_slow": g("rope_scaling.beta_slow"),+ "original_max_position_embeddings": int(g("rope_scaling.original_max_position_embeddings"))})+ self.cfg = {"text_config": t}+ self.t = t+ self.dev = dev+ self.dim, self.hc = t["hidden_size"], t["hc_mult"]+ self.eps = t["rms_norm_eps"]+ self.nl = t["num_hidden_layers"]+ self.ratios = t["compress_ratios"]+ self.kv_src = set(t["kv_source_layer_ids"])+ self.ix_src = set(t["index_source_layer_ids"])+ self.cand_src = t["candidate_source_layer_id"]+ rs, rd = t["rope_scaling"], t["qk_rope_head_dim"]+ self.fc_window = R.freqs_cis(rd, max_seq, 0, t["rope_theta"], rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)+ self.fc_comp = R.freqs_cis(rd, max_seq, rs["original_max_position_embeddings"], t["compress_rope_theta"],+ rs["factor"], rs["beta_fast"], rs["beta_slow"], dev)+ self.hasher = Hasher(pack / "engram.bin")+ self.lay = SimpleNamespace(layer_ids=tuple(p.ints("engram_layer_ids")), head_dim=self.hasher.hd)+ self.timing = {}+ self._embed = None++ def W(self, prefix: str) -> torch.Tensor:+ return dequant(self.st, prefix + ".weight", self.dev)++ def V(self, name: str) -> torch.Tensor:+ return dequant(self.st, name, self.dev).float()++ def engram(self, li, x, hash_ids):+ P = "layers.%d.engram." % li+ T, cols = hash_ids.shape+ hd = self.lay.head_dim+ dt = self.st.t[P + "embed.weight"][0]+ rb = hd // 256 * 210 if dt == "Q6_K" else hd // 32 * 33 # (a row's bytes: Q6_K or MXFP8 blocks)+ tab = self.st.raw(P + "embed.weight").reshape(-1, rb)+ flat = hash_ids.reshape(-1)+ order = np.argsort(flat)+ rows = np.empty((flat.size, rb), np.uint8)+ rows[order] = tab[flat[order]]+ if dt == "Q6_K":+ e = torch.from_numpy(q6k_rows(rows, P)).to(self.dev).view(T, cols * hd)+ else:+ b = torch.from_numpy(rows).to(self.dev).view(-1, 33)+ s = torch.ldexp(torch.ones(1, device=self.dev), b[:, :1].int() - 127)+ e = (b[:, 1:].contiguous().view(torch.float8_e4m3fn).float() * s).view(T, cols * hd)+ e = e.to(torch.bfloat16).float()+ kv = e @ self.W(P + "wkv").T+ d, hc = self.dim, self.hc+ key, value = kv[:, :hc * d].view(T, hc, d), kv[:, hc * d:]+ w = self.V(P + "gate.weight") # q_weight * k_weight (folded)+ h = x.float()+ rstd = torch.rsqrt(h.square().mean(-1) + self.eps) * torch.rsqrt(key.square().mean(-1) + self.eps)+ dot = (h * w * key).sum(-1) * rstd * d ** -0.5+ gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(1e-6).sqrt(), dot))+ return h + gate.unsqueeze(-1) * value.unsqueeze(1)++ def moe(self, li, x):+ t = self.t+ M = "layers.%d.ffn." % li+ T = x.shape[0]+ gw = self.V(M + "gate.weight")+ scores = F.softplus(x.float() @ gw.T).sqrt()+ idx = (scores + self.V(M + "gate.bias")).topk(t["num_experts_per_tok"], dim=-1)[1]+ wts = scores.gather(1, idx)+ wts = wts / (wts.sum(-1, keepdim=True) + 1e-20) * t["routed_scaling_factor"]+ lim = t["swiglu_limit"]+ y = torch.zeros(T, self.dim, device=self.dev)+ for e in torch.unique(idx).tolist():+ P = M + "experts.%d." % e+ rows, slot = torch.where(idx == e)+ xs = x[rows]+ h = F.silu((xs @ self.W(P + "w1").T).clamp(max=lim)) * (xs @ self.W(P + "w3").T).clamp(-lim, lim)+ y.index_add_(0, rows, (h * wts[rows, slot, None]) @ self.W(P + "w2").T)+ S = M + "shared_experts."+ h = F.silu((x @ self.W(S + "w1").T).clamp(max=lim)) * (x @ self.W(S + "w3").T).clamp(-lim, lim)+ y += h @ self.W(S + "w2").T+ self.last_route = (idx, wts)+ return y++ @torch.inference_mode()+ def forward(self, ids, dump=None):+ # ref.py's forward reads the embedding rows as BF16: a Q8_0 table is dequantised for the rows asked+ dt = self.st.t["embed.weight"][0]+ if dt != "BF16":+ full = self.st.raw("embed.weight").reshape(-1, self.dim // 32 * 34)+ sel = torch.from_numpy(np.array(full[np.asarray(ids)])).to(self.dev).view(-1, 34)+ v = (sel[:, :2].contiguous().view(torch.float16).float() * sel[:, 2:].view(torch.int8).float())+ rows = v.reshape(len(ids), self.dim).to(torch.bfloat16).view(torch.uint16).cpu().numpy()+ raw0 = self.st.raw+ emb = np.zeros((max(ids) + 1, self.dim), np.uint16)+ emb[np.asarray(ids)] = rows+ self.st.raw = lambda n, e=emb, r=raw0: e.view(np.uint8) if n == "embed.weight" else r(n)+ try:+ return super().forward(ids, dump)+ finally:+ self.st.raw = raw0+ return super().forward(ids, dump)+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--pack", required=True, help="pack_gguf.py's pack (<model>/maya-ds4)")+ ap.add_argument("--ids", required=True, help="comma-separated token ids")+ ap.add_argument("--gen", type=int, default=0, help="greedy tokens to append (a whole forward each)")+ ap.add_argument("--dump", help="write the per-layer residual streams, the logits and the ids (.npz)")+ ap.add_argument("--no-hs", action="store_true", help="--dump without the per-layer residual streams")+ ap.add_argument("--dev", default="cuda:0")+ a = ap.parse_args()+ ids = [int(x) for x in a.ids.split(",")]+ r = RefGguf(pathlib.Path(a.pack).expanduser(), a.dev)+ print("tokens:", len(ids), ids[:32], flush=True)+ dump = [] if a.dump and not a.no_hs else None+ t0 = time.time()+ logits = r.forward(ids, dump)+ print("forward %.1fs" % (time.time() - t0), flush=True)+ lp = torch.log_softmax(logits.float(), -1)+ if len(ids) > 1:+ tgt = torch.tensor(ids[1:], device=lp.device)+ nll = -lp[:-1].gather(1, tgt[:, None]).squeeze(1)+ print("ppl over %d tokens: %.4f" % (len(ids) - 1, math.exp(nll.mean().item())))+ top = lp[-1].topk(8)+ print("next:", list(zip(top.indices.tolist(), [round(v, 3) for v in top.values.tolist()])))+ if a.dump:+ np.savez(a.dump, ids=np.array(ids), logits=logits.cpu().numpy(), hs=np.stack(dump) if dump else np.zeros(0))+ out = list(ids)+ for _ in range(a.gen):+ nxt = int(lp[-1].argmax())+ out.append(nxt)+ print("gen ids:", out[len(ids):], flush=True)+ if nxt == 1:+ break+ lp = torch.log_softmax(r.forward(out).float(), -1)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/requant_gguf.py b/tools/dsv41/requant_gguf.pynew file mode 100644index 000000000..0740c7492--- /dev/null+++ b/tools/dsv41/requant_gguf.py@@ -0,0 +1,449 @@+"""tools/ds4/requant_gguf.py - a DeepSeek-V4.1 GGUF requantized by a recipe (the routed experts in ggml's low-bit types,+the dense matrices in Q8_0), through ggml's own quantizers (libggml-base, ctypes) - a quick test model for maya-ds4's+low-bit expert kernels until the real Maya quants exist.++ python tools/ds4/requant_gguf.py --src ~/models/DeepSeek-V4.1-Flash-1017-GGUF/UD-Q8_K_XL \\+ --out ~/ds4test/Maya-S-test --recipe recipe.json --lib ~/llama.cpp/build/bin/libggml-base.so --jobs 10++The recipe (JSON):+ {"experts": {"gate": "IQ2_XXS", "up": "IQ2_XXS", "down": "IQ3_XXS"},+ "layers": [{"layers": "0-1,38-39", "gate": "IQ2_S", "up": "IQ2_S", "down": "IQ3_S"}],+ "dense": "Q8_0",+ "dense_keep": ["hc_", "engram", "norm", "attn_sinks", "exp_probs_b", "ffn_gate_inp"],+ "engram": "skip"}+- experts: each role's type (gate = ffn_gate_exps, up = ffn_up_exps, down = ffn_down_exps); "layers" entries override it+ for the listed blocks (later entries win). A role's type may be "keep".+- dense: the type of the other float matrices (2-D or more, F32 / F16 / BF16, rows a multiple of 32), except names+ containing a dense_keep string, which stay as they are; "keep" keeps them all.+- engram: "keep" copies the Engram tables (engram_*) into the output, "skip" leaves them out (a shared Engram shard set+ beside the quant: pack_gguf.py reads <model>/Engram/*.gguf or <model>/../Engram/*.gguf); --engram-link then links the+ source's shards there as that set.++The output has one shard a source shard (the same tensors in the same order, the metadata copied; split.tensors.count+updated), tensors at the source's alignment. Workers (processes) read the source with O_DIRECT, so the page cache that+running engines rely on is neither evicted nor filled, and drop what they wrote from it; they run at SCHED_IDLE.++IQ2_XXS / IQ2_XS / IQ1_* need an importance matrix: --imatrix (llama.cpp's legacy .dat) or all ones.+"""+from __future__ import annotations++import argparse+import ctypes+import json+import mmap+import multiprocessing as mp+import os+import pathlib+import re+import struct+import sys+import time++import numpy as np++# ggml type id -> (name, values a block, bytes a block)+GT = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 11: ("Q3_K", 256, 110),+ 12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176), 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66),+ 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50), 20: ("IQ4_NL", 32, 18),+ 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),+ 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 67: ("MXFP8", 32, 33)}+BYNAME = {v[0]: k for k, v in GT.items()}+NEED_IMATRIX = {16, 17, 19, 29}+ROLE = {"ffn_gate_exps.weight": "gate", "ffn_up_exps.weight": "up", "ffn_down_exps.weight": "down"}+ALIGN_IO = 4096+++def nbytes_of(typ: int, dims: list[int]) -> int:+ _, per, bpb = GT[typ]+ n = 1+ for d in dims:+ n *= d+ return n // per * bpb+++class Shard:+ """A GGUF shard: its KV records (raw bytes each, in order) and tensor infos (absolute data offsets)."""++ def __init__(self, path: pathlib.Path):+ self.path = path+ with path.open("rb") as f:+ hdr = f.read(24)+ if hdr[:4] != b"GGUF":+ raise SystemExit("%s: not a GGUF file" % path)+ self.version, nt, nkv = struct.unpack("<IQQ", hdr[4:])+ buf = f.read(64 << 20) # (the metadata: the vocabulary is a few MB)+ pos = 0++ def take(n):+ nonlocal pos, buf+ while pos + n > len(buf):+ more = f.read(64 << 20)+ if not more:+ raise SystemExit("%s: truncated metadata" % path)+ buf += more+ v = buf[pos:pos + n]+ pos += n+ return v++ def s():+ return take(struct.unpack("<Q", take(8))[0]).decode("utf-8", "replace")++ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}++ def val(t):+ if t in fmt:+ return struct.unpack("<" + fmt[t], take(struct.calcsize(fmt[t])))[0]+ if t == 8:+ return s()+ if t == 9:+ at, n = struct.unpack("<IQ", take(12))+ if at in fmt:+ return list(struct.unpack("<%d%s" % (n, fmt[at]), take(n * struct.calcsize(fmt[at]))))+ return [val(at) for _ in range(n)]+ raise SystemExit("%s: metadata type %d" % (path, t))++ self.kv, self.kv_raw = {}, []+ for _ in range(nkv):+ p0 = pos+ k = s()+ self.kv[k] = val(struct.unpack("<I", take(4))[0])+ self.kv_raw.append((k, bytes(buf[p0:pos])))+ self.tensors = []+ for _ in range(nt):+ name = s()+ nd = struct.unpack("<I", take(4))[0]+ dims = list(struct.unpack("<%dQ" % nd, take(8 * nd)))+ typ, off = struct.unpack("<IQ", take(12))+ self.tensors.append([name, typ, dims, off])+ self.align = int(self.kv.get("general.alignment", 32))+ start = 24 + pos+ self.data_start = (start + self.align - 1) // self.align * self.align+ for t in self.tensors:+ if t[1] not in GT:+ raise SystemExit("%s: %s has ggml type %d, which this tool does not read" % (path.name, t[0], t[1]))+ t[3] += self.data_start+++def kv_int(raw: bytes, v: int) -> bytes:+ """An integer KV record with a new value, in its own type."""+ kl = struct.unpack("<Q", raw[:8])[0]+ t = struct.unpack("<I", raw[8 + kl:12 + kl])[0]+ fmt = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 10: "Q", 11: "q"}[t]+ return raw[:12 + kl] + struct.pack("<" + fmt, v)+++def layer_set(spec: str) -> set[int]:+ out = set()+ for part in str(spec).split(","):+ part = part.strip()+ if not part:+ continue+ if "-" in part:+ a, b = part.split("-")+ out.update(range(int(a), int(b) + 1))+ else:+ out.add(int(part))+ return out+++def plan_type(recipe: dict, name: str, typ: int, dims: list[int]) -> int | None:+ """The output type of a tensor (None: copied as it is, -1: left out)."""+ m = re.match(r"blk\.(\d+)\.(.+)$", name)+ short = m.group(2) if m else name+ if short.startswith("engram_"):+ return None if recipe.get("engram", "skip") == "keep" else -1+ if m and short in ROLE:+ role = ROLE[short]+ t = recipe.get("experts", {}).get(role, "keep")+ for ov in recipe.get("layers", []):+ if int(m.group(1)) in layer_set(ov["layers"]) and role in ov:+ t = ov[role]+ if t == "keep":+ return None+ if t not in BYNAME:+ raise SystemExit("recipe: unknown type %r" % t)+ q = BYNAME[t]+ if dims[0] % GT[q][1]:+ raise SystemExit("%s: rows of %d are not whole %s blocks" % (name, dims[0], t))+ return q+ dense = recipe.get("dense", "keep")+ if dense == "keep" or typ not in (1, 30) or len(dims) < 2 or dims[0] % 32: # (F32: the router, mHC - kept)+ return None+ if any(k in short for k in recipe.get("dense_keep", [])):+ return None+ q = BYNAME[dense]+ if dims[0] % GT[q][1]:+ return None+ return q+++# ---------------------------------------------------------------- workers+_lib = None+_fds = {}+_imat = None+++def _init(lib_path: str, imatrix: str | None):+ global _lib, _imat+ try:+ os.sched_setscheduler(0, os.SCHED_IDLE, os.sched_param(0))+ except (AttributeError, OSError):+ try:+ os.nice(19)+ except OSError:+ pass+ _lib = ctypes.CDLL(lib_path)+ _lib.ggml_quantize_chunk.restype = ctypes.c_size_t+ _lib.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,+ ctypes.c_int64, ctypes.c_void_p]+ _lib.ggml_quantize_init.argtypes = [ctypes.c_int]+ _imat = load_imatrix(imatrix) if imatrix else None+++def load_imatrix(path: str) -> dict:+ """llama.cpp's legacy imatrix .dat: {tensor name: float32 values (ne0, or ne0 * n_expert for experts)}."""+ out = {}+ with open(path, "rb") as f:+ n = struct.unpack("<i", f.read(4))[0]+ for _ in range(n):+ ln = struct.unpack("<i", f.read(4))[0]+ name = f.read(ln).decode()+ ncall, nval = struct.unpack("<ii", f.read(8))+ v = np.frombuffer(f.read(4 * nval), dtype=np.float32).copy()+ out[name] = v / max(1, ncall)+ return out+++def _fd(path: str, direct: bool):+ key = (path, direct)+ if key not in _fds:+ flags = os.O_RDONLY | (getattr(os, "O_DIRECT", 0) if direct else 0)+ try:+ _fds[key] = os.open(path, flags)+ except OSError:+ _fds[key] = os.open(path, os.O_RDONLY)+ return _fds[key]+++def read_src(path: str, off: int, n: int) -> memoryview:+ a0 = off // ALIGN_IO * ALIGN_IO+ a1 = (off + n + ALIGN_IO - 1) // ALIGN_IO * ALIGN_IO+ buf = mmap.mmap(-1, a1 - a0)+ try:+ got = os.preadv(_fd(path, True), [buf], a0)+ except OSError: # (no O_DIRECT here: a plain read)+ got = os.preadv(_fd(path, False), [buf], a0)+ if got < off - a0 + n:+ raise IOError("%s: short read at %d" % (path, off))+ return memoryview(buf)[off - a0:off - a0 + n]+++E2M1 = np.array([0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12], dtype=np.float32)+++def to_f32(typ: int, raw: memoryview, n: int) -> np.ndarray:+ b = np.frombuffer(raw, dtype=np.uint8)+ if typ == 0:+ return np.frombuffer(raw, dtype=np.float32).copy()+ if typ == 1:+ return np.frombuffer(raw, dtype=np.float16).astype(np.float32)+ if typ == 30:+ return (np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16).view(np.float32)+ if typ == 8:+ blk = b.reshape(-1, 34)+ d = blk[:, :2].copy().view(np.float16).astype(np.float32)+ q = blk[:, 2:].view(np.int8).astype(np.float32)+ return (q * d).reshape(-1)+ if typ == 39: # ggml's MXFP4: [e8m0][16 code bytes], value j = kvalues[code] * 2^(e - 128)+ blk = b.reshape(-1, 17)+ e = blk[:, 0].astype(np.int32)+ d = np.ldexp(np.float32(1.0), e - 128).astype(np.float32)+ q = blk[:, 1:]+ v = np.empty((blk.shape[0], 32), dtype=np.float32)+ v[:, :16] = E2M1[q & 15]+ v[:, 16:] = E2M1[q >> 4]+ return (v * d[:, None]).reshape(-1)+ raise ValueError("source type %d" % typ)+++def _job(j: dict):+ """One piece of a tensor: rows [r0, r1) (of ne0 values each) quantized, or bytes copied; written in place."""+ t0 = time.time()+ out_fd = os.open(j["out"], os.O_WRONLY)+ try:+ if j["q"] is None: # a copy, in pieces of 256 MB+ n, done = j["n"], 0+ while done < n:+ k = min(256 << 20, n - done)+ os.pwrite(out_fd, read_src(j["src"], j["src_off"] + done, k), j["out_off"] + done)+ done += k+ else:+ ne0, r0, r1, st, q = j["ne0"], j["r0"], j["r1"], j["typ"], j["q"]+ srb = GT[st][2] * ne0 // GT[st][1]+ raw = read_src(j["src"], j["src_off"] + r0 * srb, (r1 - r0) * srb)+ x = np.ascontiguousarray(to_f32(st, raw, (r1 - r0) * ne0), dtype=np.float32)+ del raw+ qrb = GT[q][2] * ne0 // GT[q][1]+ dst = np.empty((r1 - r0) * qrb, dtype=np.uint8)+ imat = None+ if q in NEED_IMATRIX or (_imat is not None and j["iname"] in _imat):+ w = _imat.get(j["iname"]) if _imat is not None else None+ if w is None or w.size % ne0:+ w = np.ones(ne0, dtype=np.float32)+ imat = w+ rows_per = j["rows_per_expert"]+ r = r0+ while r < r1: # (one expert's rows at a time: an expert's own importance row)+ n = min(r1 - r, rows_per - r % rows_per if rows_per else r1 - r)+ im = None+ if imat is not None:+ e = r // rows_per if rows_per else 0+ im = imat[e * ne0:(e + 1) * ne0] if imat.size >= (e + 1) * ne0 else imat[:ne0]+ im = np.ascontiguousarray(im, dtype=np.float32)+ got = _lib.ggml_quantize_chunk(q, x[(r - r0) * ne0:].ctypes.data, dst[(r - r0) * qrb:].ctypes.data,+ 0, n, ne0, im.ctypes.data if im is not None else None)+ if got != n * qrb:+ raise RuntimeError("%s: ggml_quantize_chunk wrote %d bytes, expected %d" % (j["name"], got, n * qrb))+ r += n+ os.pwrite(out_fd, memoryview(dst), j["out_off"] + r0 * qrb)+ j["n"] = (r1 - r0) * qrb+ j["out_off"] += r0 * qrb+ os.fdatasync(out_fd)+ try:+ os.posix_fadvise(out_fd, j["out_off"], j["n"], os.POSIX_FADV_DONTNEED)+ except (AttributeError, OSError):+ pass+ finally:+ os.close(out_fd)+ return j["name"], j["n"], time.time() - t0+++# ---------------------------------------------------------------- main+def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--src", required=True, help="the source GGUF's shards (a directory)")+ ap.add_argument("--out", required=True, help="the output directory")+ ap.add_argument("--recipe", required=True, help="the recipe (JSON file)")+ ap.add_argument("--lib", default=os.path.expanduser("~/llama.cpp/build/bin/libggml-base.so"),+ help="libggml-base (ggml_quantize_chunk)")+ ap.add_argument("--imatrix", help="llama.cpp's legacy imatrix .dat (default: all ones where a type needs one)")+ ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 2) - 2))+ ap.add_argument("--experts-per-job", type=int, default=4)+ ap.add_argument("--engram-link", action="store_true",+ help="(engram skip) link the source's shards as <out>/Engram/, the shared Engram set")+ ap.add_argument("--dry-run", action="store_true", help="print the plan and the output size")+ ap.add_argument("--only", help="regex: quantize only the matching tensors (the rest zero-filled: tests)")+ ap.add_argument("--first", help="regex: the matching tensors' pieces first (e.g. a few layers for a quick test)")+ ap.add_argument("--resume", action="store_true",+ help="keep the output files of an earlier run (same plan) and skip the pieces already written")+ a = ap.parse_args()++ src = pathlib.Path(a.src).expanduser().resolve()+ out = pathlib.Path(a.out).expanduser()+ recipe = json.loads(pathlib.Path(a.recipe).read_text())+ shards = sorted(src.glob("*.gguf"))+ if not shards:+ raise SystemExit("%s: no .gguf files" % src)+ ss = [Shard(p) for p in shards]++ jobs, plan_sizes, outs = [], {}, []+ for si, sh in enumerate(ss):+ kept = []+ for name, typ, dims, off in sh.tensors:+ q = plan_type(recipe, name, typ, dims)+ if q == -1:+ continue+ kept.append((name, typ, dims, off, typ if q is None else q, q))+ # the output's layout: the KV records (split.tensors.count updated), the tensor infos, the data+ kv = []+ for k, raw in sh.kv_raw:+ if k == "split.tensors.count":+ total = sum(1 for s2 in ss for t in s2.tensors if plan_type(recipe, t[0], t[1], t[2]) != -1)+ raw = kv_int(raw, total)+ kv.append(raw)+ tinfo, data_off = [], 0+ for name, typ, dims, off, otyp, q in kept:+ nb = nbytes_of(otyp, dims)+ tinfo.append((name, dims, otyp, data_off, nb))+ data_off += (nb + sh.align - 1) // sh.align * sh.align+ plan_sizes[GT[otyp][0]] = plan_sizes.get(GT[otyp][0], 0) + nb+ head = struct.pack("<4sIQQ", b"GGUF", sh.version, len(tinfo), len(kv)) + b"".join(kv)+ for name, dims, otyp, doff, nb in tinfo:+ nm = name.encode()+ head += struct.pack("<Q", len(nm)) + nm + struct.pack("<I", len(dims))+ head += struct.pack("<%dQ" % len(dims), *dims) + struct.pack("<IQ", otyp, doff)+ dstart = (len(head) + sh.align - 1) // sh.align * sh.align+ opath = out / sh.path.name.replace("UD-Q8_K_XL", "requant")+ outs.append((opath, head, dstart + data_off))+ for (name, typ, dims, off, otyp, q), (_, _, _, doff, nb) in zip(kept, tinfo):+ if a.only and not re.search(a.only, name):+ continue+ base = {"name": name, "src": str(sh.path), "src_off": off, "out": str(opath), "out_off": dstart + doff}+ if q is None:+ jobs.append(dict(base, q=None, n=nb))+ continue+ ne0 = dims[0]+ rows = 1+ for d in dims[1:]:+ rows *= d+ rpe = dims[1] if len(dims) == 3 else 0 # rows an expert (3-D: [n_exp][rows][ne0])+ step = rpe * a.experts_per_job if rpe else max(1, (64 << 20) // (4 * ne0))+ m = re.match(r"blk\.(\d+)\.(.+)$", name)+ iname = name+ for r0 in range(0, rows, step):+ jobs.append(dict(base, q=q, typ=typ, ne0=ne0, r0=r0, r1=min(rows, r0 + step), rows_per_expert=rpe,+ iname=iname, n=0))+ total = sum(o[2] for o in outs)+ print("plan: %d shards, %.1f GB (%s), %d jobs" % (len(outs), total / 1e9,+ ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(plan_sizes.items())), len(jobs)), flush=True)+ if a.dry_run:+ return 0+ out.mkdir(parents=True, exist_ok=True)+ for opath, head, size in outs:+ if a.resume and opath.is_file() and opath.stat().st_size == size:+ with open(opath, "r+b") as f:+ if f.read(len(head)) != head:+ raise SystemExit("%s: not this plan's file (--resume)" % opath)+ continue+ with open(opath, "wb") as f:+ f.write(head)+ f.truncate(size)+ if a.resume: # a piece is written when the first and last 256 bytes of its output are not all zero+ def written(j):+ n = j["n"] if j["q"] is None else (j["r1"] - j["r0"]) * GT[j["q"]][2] * j["ne0"] // GT[j["q"]][1]+ o = j["out_off"] + (0 if j["q"] is None else j["r0"] * GT[j["q"]][2] * j["ne0"] // GT[j["q"]][1])+ with open(j["out"], "rb") as f:+ f.seek(o)+ h = f.read(min(256, n))+ f.seek(o + max(0, n - 256))+ t = f.read(min(256, n))+ return any(h) and any(t)+ before = len(jobs)+ jobs = [j for j in jobs if not written(j)]+ print("resume: %d of %d pieces already written" % (before - len(jobs), before), flush=True)+ if a.engram_link and recipe.get("engram", "skip") == "skip":+ eg = out / "Engram"+ eg.mkdir(exist_ok=True)+ for p in shards:+ link = eg / p.name+ if not link.exists():+ link.symlink_to(p)+ # the biggest pieces first (the experts), so the last minutes are not one long tensor+ jobs.sort(key=lambda j: -(j.get("r1", 0) - j.get("r0", 0)) * j.get("ne0", 0) if j["q"] is not None else 0)+ if a.first:+ jobs.sort(key=lambda j: 0 if re.search(a.first, j["name"]) else 1)+ t0, done_b, done_j = time.time(), 0, 0+ with mp.get_context("fork").Pool(a.jobs, initializer=_init, initargs=(os.path.expanduser(a.lib), a.imatrix)) as pool:+ for name, nb, dt in pool.imap_unordered(_job, jobs):+ done_b += nb+ done_j += 1+ if done_j % 50 == 0 or done_j == len(jobs):+ el = time.time() - t0+ print("%d / %d jobs, %.1f GB written, %.0f s (%.0f min left)" %+ (done_j, len(jobs), done_b / 1e9, el, el / done_j * (len(jobs) - done_j) / 60), flush=True)+ print("done: %s (%.1f GB) in %.1f min" % (out, total / 1e9, (time.time() - t0) / 60))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/st.py b/tools/dsv41/st.pynew file mode 100644index 000000000..de70f0ba2--- /dev/null+++ b/tools/dsv41/st.py@@ -0,0 +1,64 @@+"""tools/ds4/st.py - every tensor of a DeepSeek-V4.1 model directory, read in place.++ st = SafeTensors(Path("~/models/DSV4").expanduser())+ st.t["layers.0.attn.wq_a.trellis"] -> (file, dtype, shape, absolute offset, bytes)+ st.raw(name), st.f32(name) -> numpy views / float32 copies++The index's own files come first; any other .safetensors in the directory (the Engram shards 47 / 48 of the base+repo, model-0004[78]-of-00048.safetensors) only adds names the index does not have.+"""+from __future__ import annotations++import json+import pathlib+import struct++import numpy as np++DTYPE_BYTES = {"F32": 4, "F16": 2, "BF16": 2, "I32": 4, "I16": 2, "F8_E4M3": 1, "F8_E8M0": 1, "U8": 1, "I8": 1,+ "I64": 8}+++class SafeTensors:+ def __init__(self, d: pathlib.Path):+ self.dir = pathlib.Path(d)+ ix = json.loads((self.dir / "model.safetensors.index.json").read_text())["weight_map"]+ primary = sorted(set(ix.values()))+ others = sorted(p.name for p in self.dir.glob("*.safetensors") if p.name not in primary)+ self.files: list[str] = []+ self.t: dict[str, tuple] = {}+ self.mm: dict[str, np.memmap] = {}+ for fn in primary + others:+ p = self.dir / fn+ with p.open("rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ h = json.loads(f.read(n))+ base = 8 + n+ self.files.append(fn)+ for k, v in h.items():+ if k == "__metadata__" or k in self.t:+ continue+ a, b = v["data_offsets"]+ self.t[k] = (fn, v["dtype"], tuple(v["shape"]), base + a, b - a)+ self.mm[fn] = np.memmap(p, dtype=np.uint8, mode="r")++ def has(self, name: str) -> bool:+ return name in self.t++ def raw(self, name: str) -> np.ndarray:+ fn, _, _, off, nb = self.t[name]+ return self.mm[fn][off:off + nb]++ def f32(self, name: str) -> np.ndarray:+ """The tensor as float32 (exact for F32/F16/BF16), shaped as stored."""+ _, dt, shape, _, _ = self.t[name]+ r = self.raw(name)+ if dt == "F32":+ a = r.view("<f4")+ elif dt == "F16":+ a = r.view("<f2").astype(np.float32)+ elif dt == "BF16":+ a = (r.view("<u2").astype(np.uint32) << 16).view(np.float32)+ else:+ raise ValueError("%s is %s, not a float tensor" % (name, dt))+ return a.reshape(shape)diff --git a/tools/dsv41/tok.py b/tools/dsv41/tok.pynew file mode 100644index 000000000..b26e133bc--- /dev/null+++ b/tools/dsv41/tok.py@@ -0,0 +1,36 @@+"""tools/ds4/tok.py - token ids for maya-ds4's --ids, and text back from its output (DeepSeek-V4.1's tokenizer).++ python tools/ds4/tok.py --model ~/models/DSV4 --chat "Why is the sky blue?" [--think] -> 0,128803,...+ python tools/ds4/tok.py --model ~/models/DSV4 --decode 18033,1 -> text+"""+from __future__ import annotations++import argparse+import pathlib+import sys+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--model", default=str(pathlib.Path("~/models/DSV4").expanduser()))+ ap.add_argument("--chat")+ ap.add_argument("--think", action="store_true", help="thinking mode (the prompt ends with <think>)")+ ap.add_argument("--text")+ ap.add_argument("--decode")+ a = ap.parse_args()+ from tokenizers import Tokenizer+ tok = Tokenizer.from_file(str(pathlib.Path(a.model).expanduser() / "tokenizer.json"))+ if a.decode:+ ids = [int(x) for x in a.decode.replace(" ", "").strip(",").split(",") if x]+ print(tok.decode(ids, skip_special_tokens=False))+ return 0+ if a.chat is not None:+ s = "<|begin▁of▁sentence|><|User|>" + a.chat + "<|Assistant|>" + ("<think>" if a.think else "</think>")+ else:+ s = a.text or ""+ print(",".join(str(i) for i in tok.encode(s, add_special_tokens=False).ids))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/dsv41/vision_pack.py b/tools/dsv41/vision_pack.pynew file mode 100644index 000000000..6144b3501--- /dev/null+++ b/tools/dsv41/vision_pack.py@@ -0,0 +1,208 @@+"""tools/ds4/vision_pack.py - DeepSeek-V4.1's vision tower (the part of the model that reads pictures, which the GGUFs+leave out) for a GGUF install: shard 1 of DeepSeek's own checkpoint (model-00001-of-00048, the tower's vision.* and the+aligner's aligner.* tensors, BF16, nothing else) and config.json (their geometry) from download_gguf.json's "vision"+entry, plus the picture span's three marker rows (image_start, image_newline, image_end: BF16 [5120]), which sit beside+the embedding in shard 2 - only their 30 KB are read from it, with an HTTP range request, never the 1.3 GB file.++ python tools/ds4/vision_pack.py --dir <folder with shard 1 and config.json> -> vision.bin + vision.json there++vision.bin holds every tensor exactly as the checkpoint has it (copied, 256-byte aligned) and then the three marker+rows; it is read back and compared with the shard bit for bit before it is kept. vision.json lists every tensor (name,+dtype, offset, bytes, shape) as tensors.txt does, and the geometry the engine reads (vision.<key> from config.json's+vision_config, image_token_id); pack_gguf.py adds both to the pack.++The marker rows: the shard's header (a range request), then the rows (one range request), each row's sha256 checked+against the manifest's. A server that answers a range request with the whole file (status 200, not 206) is cut off at+once, the request is tried once more, then this stops with the reason - nothing large is ever downloaded here.+--marks-url takes another URL, or a local file (a mirror, the tests).+"""+from __future__ import annotations++import argparse+import hashlib+import json+import pathlib+import struct+import sys+import time+import urllib.request++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+MANIFEST = HERE / "download_gguf.json"+HF = "https://huggingface.co/{repo}/resolve/{revision}/{path}"+ALIGN = 256+MARKS = ("image_start", "image_newline", "image_end") # (the order in vision.bin)+TRIES = 2 # a range request and one retry+++class RangeError(Exception):+ pass+++def manifest_entry(path: pathlib.Path = MANIFEST) -> dict:+ v = json.loads(path.read_text(encoding="utf-8")).get("vision") or {}+ if not v.get("files") or not v.get("marks"):+ raise SystemExit(f"{path}: no \"vision\" entry")+ return v+++def read_range(url: str, start: int, n: int, total: int | None = None, timeout: int = 60) -> bytes:+ """Bytes start .. start + n - 1 of a file: a URL with a range request (exactly those bytes, status 206 - anything+ else is cut off unread: RangeError), or a local file / file:// URL read at the offset."""+ if not url.startswith(("http://", "https://")):+ p = pathlib.Path(url[7:] if url.startswith("file://") else url)+ with p.open("rb") as f:+ f.seek(start)+ b = f.read(n)+ if len(b) != n:+ raise RangeError(f"{p}: {len(b)} of {n} bytes at {start}")+ return b+ end = start + n - 1+ req = urllib.request.Request(url, headers={"User-Agent": "maya-setup", "Range": f"bytes={start}-{end}",+ "Accept-Encoding": "identity"})+ try:+ with urllib.request.urlopen(req, timeout=timeout) as r:+ cr = r.headers.get("Content-Range", "")+ want = f"bytes {start}-{end}/"+ if r.status != 206 or not cr.startswith(want) or (total and cr != f"{want}{total}"):+ # (the whole file, or other bytes: the connection is closed here without reading the body)+ raise RangeError(f"the server answered status {r.status} ({cr or 'no Content-Range'}) to a request for "+ f"bytes {start}-{end}, not just those bytes")+ b = r.read(n + 1)+ except OSError as e:+ raise RangeError(str(e)) from e+ if len(b) != n:+ raise RangeError(f"{len(b)} bytes arrived for a range of {n}")+ return b+++def fetch_marks(url: str, marks: dict, tries: int = TRIES) -> dict:+ """The marker rows {name: bytes} from shard 2 at `url`: its header (where they are), then the rows, every row's+ sha256 checked. Each step is tried `tries` times; then SystemExit with the reason."""+ rows = marks["rows"]+ total = marks.get("bytes")+ why = ""+ for attempt in range(tries):+ try:+ n = struct.unpack("<Q", read_range(url, 0, 8, total))[0]+ if not 0 < n < 1 << 24:+ raise RangeError(f"a safetensors header of {n} bytes")+ head = json.loads(read_range(url, 8, n, total))+ base = 8 + n+ at = {}+ for name, (off, nb, _sha) in rows.items():+ e = head.get(name) or {}+ a, b = e.get("data_offsets") or (None, None)+ if e.get("dtype") != "BF16" or a is None or base + a != off or b - a != nb:+ raise RangeError(f"{name} is not where the manifest says ({e or 'not in the header'})")+ at[name] = (off, nb)+ lo = min(o for o, _ in at.values())+ hi = max(o + nb for o, nb in at.values())+ if hi - lo > 1 << 20:+ raise RangeError(f"the marker rows span {hi - lo} bytes: not read in one piece")+ blob = read_range(url, lo, hi - lo, total)+ out = {}+ for name, (off, nb) in at.items():+ out[name] = blob[off - lo:off - lo + nb]+ if hashlib.sha256(out[name]).hexdigest() != rows[name][2]:+ raise RangeError(f"{name}: the sha256 does not match the manifest's")+ return out+ except (RangeError, ValueError) as e:+ why = str(e)+ if attempt + 1 < tries:+ print(f"vision: the marker rows: {why}; trying once more ...", file=sys.stderr)+ time.sleep(2)+ raise SystemExit(f"vision: the picture markers (30 KB of {marks['file']}) could not be read: {why}. Nothing large "+ "was downloaded. Check the connection and run the setup again (or --no-vision: text only).")+++def shard(path: pathlib.Path):+ raw = np.memmap(path, np.uint8, mode="r")+ n = struct.unpack("<Q", bytes(raw[:8]))[0]+ head = json.loads(bytes(raw[8:8 + n]))+ head.pop("__metadata__", None)+ return raw, head, 8 + n+++def geometry(cfg: dict) -> dict:+ """pack.py's vision keys: vision.<key> for config.json's numeric vision_config entries, and image_token_id."""+ out = {}+ for k, v in (cfg.get("vision_config") or {}).items():+ if isinstance(v, (int, float)) and not isinstance(v, bool):+ out["vision.%s" % k] = v+ if isinstance(cfg.get("image_token_id"), int):+ out["image_token_id"] = cfg["image_token_id"]+ if not out.get("vision.num_hidden_layers") or "image_token_id" not in out:+ raise SystemExit("config.json: no vision_config / image_token_id")+ return out+++def pack(d: pathlib.Path, man: dict, marks_url: str | None = None) -> dict:+ names = [n for n in man["files"] if n.endswith(".safetensors")]+ if len(names) != 1:+ raise SystemExit("the manifest's vision entry: one safetensors shard expected")+ src = d / names[0]+ if not src.is_file() or src.stat().st_size != man["files"][names[0]][0]:+ raise SystemExit(f"{src}: missing or not whole")+ geo = geometry(json.loads((d / "config.json").read_text(encoding="utf-8")))+ mk = man["marks"]+ url = marks_url or HF.format(repo=man["repo"], revision=man["revision"], path=mk["file"])+ rows = fetch_marks(url, mk)+ raw, head, base = shard(src)+ order = sorted(head, key=lambda t: head[t]["data_offsets"][0])+ for t in order:+ if not t.startswith(("vision.", "aligner.")):+ raise SystemExit(f"{src.name}: {t} is not the vision tower's")+ entries, off = [], 0+ part = d / "vision.bin.part"+ with part.open("wb") as f:+ def put(name, dtype, shape, b):+ nonlocal off+ f.write(b)+ entries.append([name, dtype, off, len(b), list(shape)])+ off += len(b)+ pad = (-off) % ALIGN+ f.write(b"\0" * pad)+ off += pad+ for t in order:+ e = head[t]+ a, b = e["data_offsets"]+ put(t, e["dtype"], e["shape"], raw[base + a:base + b].tobytes())+ for name in MARKS:+ put(name, "BF16", [len(rows[name]) // 2], rows[name])+ # read back: every tensor as the checkpoint has it, bit for bit+ back = np.memmap(part, np.uint8, mode="r")+ for name, _dt, o, nb, _shape in entries:+ if name in rows:+ ok = back[o:o + nb].tobytes() == rows[name]+ else:+ a, b = head[name]["data_offsets"]+ ok = np.array_equal(back[o:o + nb], raw[base + a:base + b])+ if not ok:+ raise SystemExit(f"vision.bin: {name} differs from the checkpoint after the copy")+ del back, raw+ part.replace(d / "vision.bin")+ info = {"from": {"repo": man["repo"], "revision": man["revision"], "files": sorted(man["files"]),+ "marks": mk["file"]}, "config": geo, "tensors": entries}+ (d / "vision.json").write_text(json.dumps(info))+ return info+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--dir", required=True, help="the folder with shard 1 and config.json")+ ap.add_argument("--manifest", default=str(MANIFEST))+ ap.add_argument("--marks-url", help="shard 2 elsewhere (a URL or a local file; default: Hugging Face, pinned)")+ a = ap.parse_args()+ d = pathlib.Path(a.dir)+ info = pack(d, manifest_entry(pathlib.Path(a.manifest)), a.marks_url)+ nb = sum(e[3] for e in info["tensors"])+ print(f"vision.bin: {len(info['tensors'])} tensors ({nb / 1e9:.2f} GB), every one equal to the checkpoint's; "+ f"{info['config']['vision.num_hidden_layers']} blocks, image token {info['config']['image_token_id']}")+ return 0+++if __name__ == "__main__":+ sys.exit(main())