Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-dsv41-mac (DeepSeek on Metal) into v0.2.7

Niko12218477811197d9
diff --cc include/strata/dsv41/exl3.hpp
index a5455b75a,9ab2482fd..6fb409e62
--- a/include/strata/dsv41/exl3.hpp
+++ b/include/strata/dsv41/exl3.hpp
@@@ -152,12 -152,10 +152,14 @@@ struct SetArgs
const int* bounds = nullptr;
int row0 = 0;
int max_rows = 0; // the most rows one expert of the set has
+ const int* hbounds = nullptr; // (Metal) the same bounds on the host: [n_exp + 1], absolute rows
+ int total_rows = 0; // (Metal) the set's rows in all
int n_embd = 4096, n_ff = 2048;
float limit = 0.0f;
+ // gate / up: the caller's F16 copy of the activations (row_idx picks the rows). The tiles round F32 to F16 for
+ // their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by
+ // every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".
+ const uint16_t* x16 = nullptr;
};
/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).
void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);
diff --cc metal/include/metal_compat/ds4_metal_host.h
index 000000000,ac298c9db..9eb7c9a5d
mode 000000,100644..100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@@ -1,0 -1,109 +1,111 @@@
+ // metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch.
+ //
+ // GENERATED from metal/src/engine/ds4_host.cpp and ds4_finish.cpp, which are the source of truth: do not hand-edit the
+ // declaration block, regenerate it (see RIGS/m1002.md).
+ #pragma once
+
+ #include <cstddef>
+ #include <cstdint>
+ #include <vector>
+
+ namespace strata::gpu {
+ struct StreamImpl;
+ using Stream = StreamImpl*;
+ } // namespace strata::gpu
+
+ namespace strata::dsv41::metal_host {
+
+ // ds4_kernels.cu:1894 finish_kernel<false> - the engine's Finish, as ds4_kernels.cuh declares it
+ struct FinishIn {
+ const float* w;
+ float eps;
+ int rd, pos, inverse, vpp, pos_step;
+ const float* freqs;
+ int quant;
+ const uint16_t* dst;
+ long long dst_stride;
+ uint8_t* pdst;
+ long long pdst_stride;
+ };
+
+ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);
+ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);
+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);
+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);
+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);
+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);
+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);
+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);
+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);
++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post, float* comb);
+ bool init_pre(float* pre, int T, void*);
+ bool iq_mv(const void* jobs, int nj);
+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);
+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);
+ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
+ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+ bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);
+ uint64_t zc_map(const void* p, size_t bytes); // zero-copy weights: the GPU address of host memory, 0 if not mappable
+ bool q8_quant_multi(const void* specs, int n); // {const float* x; uint64 ldx; uint32 rows, k; int8* q; float* d} each
+ bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);
+ bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);
+ bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);
+ bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha, float beta);
+ bool swiglu(const float* g, const float* u, int n, float lim, float* h);
+ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
+
+ // generated from ds4_host.cpp and ds4_finish.cpp
+ bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);
+ bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);
+ bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);
+ bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out);
+ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
+ bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc);
+ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);
+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
+ bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);
+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);
+ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);
+ bool gather_rows(const float* x, const void* g, int n, int cols, float* out);
+ bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);
+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);
+ bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);
+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);
+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);
+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);
+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);
+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);
+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);
++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post, float* comb);
+ bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand, int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt);
+ bool init_pre(float* pre, int T, void*);
+ bool iota_rows(int* out, int T, int n, int stride);
+ bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int type, int groups);
+ bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int type, int groups);
+ bool iq_mv(const void* jobs, int nj);
+ bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R);
+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);
+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);
+ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride, const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd);
+ bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int groups);
+ bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int groups);
+ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);
+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);
+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);
+ bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w, const float* bias_vl, const void* vl);
+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
+ bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);
+ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
+ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows, const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt);
+ bool swiglu(const float* g, const float* u, int n, float lim, float* h);
+ bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);
+ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
+
+ } // namespace strata::dsv41::metal_host
diff --cc metal/kernels/moe_mm.metal
index fb2f6f056,ff28897ac..5207a5f2d
--- a/metal/kernels/moe_mm.metal
+++ b/metal/kernels/moe_mm.metal
@@@ -635,12 -633,12 +635,13 @@@ template <> struct W8<T_IQ2_XXS>
device const uchar* bl = row + (k >> 8) * 66;
const int ib = (k & 255) >> 5, l = (k & 31) >> 3, hh = (k & 7) >> 2;
const uint a0 = rd32(bl + 2 + 8 * ib), a1 = rd32(bl + 2 + 8 * ib + 4);
-- threadgroup const uchar* ks = (threadgroup const uchar*)(tab + 512);
const float sc = rdh(bl) * (0.5f + float(a1 >> 28)) * 0.25f;
const uint i0 = (a0 >> (8 * l)) & 255, i1 = (a0 >> (8 * (l + 2))) & 255;
-- const uint s0 = ks[(a1 >> (7 * l)) & 127], s1 = ks[(a1 >> (7 * (l + 2))) & 127];
-- a = w4_signed(tab[2 * i0 + hh], hh ? s0 >> 4 : s0, sc);
-- b = w4_signed(tab[2 * i1 + hh], hh ? s1 >> 4 : s1, sc);
++ // through the accessors, so QUANT_CTAB (the constant grids, no threadgroup copy) gives the same values: the
++ // direct tab[] reads below skipped the ctab branch and read an unfilled table
++ const uint s0 = ksg(tab, 512, (a1 >> (7 * l)) & 127), s1 = ksg(tab, 512, (a1 >> (7 * (l + 2))) & 127);
++ a = w4_signed(g_iq2xxs(tab, i0, hh), hh ? s0 >> 4 : s0, sc);
++ b = w4_signed(g_iq2xxs(tab, i1, hh), hh ? s1 >> 4 : s1, sc);
}
};
template <> struct W8<T_IQ2_XS> {
@@@ -649,14 -647,14 +650,14 @@@
const int ib = (k & 255) >> 5, l = (k & 31) >> 3, hh = (k & 7) >> 2;
device const ushort* q = (device const ushort*)(bl + 2) + 4 * ib;
const uint v0 = q[l], v1 = q[l + 2];
-- threadgroup const uchar* ks = (threadgroup const uchar*)(tab + 1024);
const uint scb = bl[66 + ib];
const float d = rdh(bl);
const float sc0 = d * (0.5f + float((scb >> (4 * (l >> 1))) & 15)) * 0.25f;
const float sc1 = d * (0.5f + float((scb >> (4 * ((l + 2) >> 1))) & 15)) * 0.25f;
-- const uint s0 = ks[v0 >> 9], s1 = ks[v1 >> 9];
-- a = w4_signed(tab[2 * (v0 & 511) + hh], hh ? s0 >> 4 : s0, sc0);
-- b = w4_signed(tab[2 * (v1 & 511) + hh], hh ? s1 >> 4 : s1, sc1);
++ // through the accessors: the direct tab[]/ks[] reads ignored QUANT_CTAB, whose tables are then never filled
++ const uint s0 = ksg(tab, 1024, v0 >> 9), s1 = ksg(tab, 1024, v1 >> 9);
++ a = w4_signed(g_iq2xs(tab, v0 & 511, hh), hh ? s0 >> 4 : s0, sc0);
++ b = w4_signed(g_iq2xs(tab, v1 & 511, hh), hh ? s1 >> 4 : s1, sc1);
}
};
template <> struct W8<T_IQ2_S> {
@@@ -675,6 -673,6 +676,42 @@@
b = w4_signed(g_iq2s(tab, i1, hh), hh ? s1 >> 4 : s1, sc1);
}
};
++
++// IQ1_M (the 2.00 bpw pack's 1.75-bit type), ggml's dequantize_row_iq1_m: the block's F16 scale sits in the top
++// nibbles of its four u16 scales; a 32-value sub-block has two 3-bit scales (groups 0-1 and 2-3, eight values each),
++// and each eight-value group carries a +-0.125 delta picked by a bit of its qh byte. The tile's two halves are the
++// four values at k and the four at k + 16, each decoded on its own (k + 16 crosses into the next sub-block at l >= 2).
++template <typename P>
++static inline half4 iq1m_w4(P row, int k) {
++ const P bl = row + (k >> 8) * 56;
++ const int ib = (k & 255) >> 5, g = (k & 31) >> 3, hh = (k & 7) >> 2;
++ const uint s0 = uint(bl[48]) | (uint(bl[49]) << 8), s1 = uint(bl[50]) | (uint(bl[51]) << 8);
++ const uint s2 = uint(bl[52]) | (uint(bl[53]) << 8), s3 = uint(bl[54]) | (uint(bl[55]) << 8);
++ const float d = float(as_type<half>(ushort((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u))));
++ const uint sc = uint(((const device ushort*) (bl + 48))[ib >> 1]);
++ const float dl = d * float(2u * ((sc >> (6u * uint(ib & 1) + (g < 2 ? 0u : 3u))) & 7u) + 1u);
++ const uint qh = (g < 2) ? uint(bl[32 + 2 * ib]) : uint(bl[33 + 2 * ib]);
++ const uint idx = uint(bl[4 * ib + g]) | (((g & 1) != 0) ? ((qh << 4u) & 0x700u) : ((qh << 8u) & 0x700u));
++ const bool neg = ((g & 1) != 0) ? ((qh & 0x80u) != 0u) : ((qh & 0x08u) != 0u);
++ const ulong gr = iq1s_grid[idx];
++ const float delta = neg ? -0.125f : 0.125f;
++ float v[4];
++ for (int j = 0; j < 4; ++j) {
++ const uint by = uint((gr >> (8u * uint(4 * hh + j))) & 0xffu);
++ const int c = (by & 0x80u) != 0u ? int(by) - 256 : int(by);
++ v[j] = dl * (float(c) + delta);
++ }
++ return half4(v[0], v[1], v[2], v[3]);
++}
++template <> struct W8<T_IQ1_M> {
++ template <typename P>
++ static inline void run(P row, int k, threadgroup const uint* tab, thread half4& a, thread half4& b) {
++ (void) tab;
++ a = iq1m_w4(row, k);
++ b = iq1m_w4(row, k + 16);
++ }
++};
++
template <> struct W8<T_IQ3_S> {
template <typename P>
static inline void run(P row, int k, threadgroup const uint* tab, thread half4& a, thread half4& b) {
diff --cc metal/kernels/quant.metal
index 14df14eef,51fecca7c..cbed727cc
--- a/metal/kernels/quant.metal
+++ b/metal/kernels/quant.metal
@@@ -25,6 -25,6 +25,7 @@@
#define T_IQ3_XXS 18
#define T_IQ4_NL 20
#define T_IQ3_S 21
++#define T_IQ1_M 29 // the 2.00 bpw pack's 1.75-bit type
#define T_IQ2_S 22
#define T_IQ4_XS 23
#define T_Q2_0 42
@@@ -39,6 -36,6 +40,7 @@@ template <> struct fmt<T_Q2_0> { enu
template <> struct fmt<T_Q2_0R> { enum { QK = 64, BB = 18, UNIT = 16, TAB = 1 }; };
template <> struct fmt<T_IQ3_XXS> { enum { QK = 256, BB = 98, UNIT = 16, TAB = 256 + 32 }; };
template <> struct fmt<T_IQ2_XXS> { enum { QK = 256, BB = 66, UNIT = 16, TAB = 512 + 32 }; };
++template <> struct fmt<T_IQ1_M> { enum { QK = 256, BB = 56, UNIT = 16, TAB = 1 }; };
template <> struct fmt<T_IQ2_XS> { enum { QK = 256, BB = 74, UNIT = 16, TAB = 1024 + 32 }; };
template <> struct fmt<T_IQ2_S> { enum { QK = 256, BB = 82, UNIT = 16, TAB = 2048 + 32 }; };
template <> struct fmt<T_IQ3_S> { enum { QK = 256, BB = 110, UNIT = 16, TAB = 512 + 32 }; };
@@@ -389,6 -112,6 +391,8 @@@ template <> inline void load_tab<T_IQ2_
for (int i = tid; i < 512; i += nth) { tab[2 * i] = uint(iq2xs_grid[i]); tab[2 * i + 1] = uint(iq2xs_grid[i] >> 32); }
load_signs(tab, 1024, tid, nth);
}
++// IQ1_M: no threadgroup table - the tile reads iq1s_grid from the constants (the w4 helper in moe_mm.metal)
++template <> inline void load_tab<T_IQ1_M>(threadgroup uint*, ushort, ushort) {}
template <> inline void load_tab<T_IQ2_S>(threadgroup uint* tab, ushort tid, ushort nth) {
if (ctab) return;
for (int i = tid; i < 1024; i += nth) { tab[2 * i] = uint(iq2s_grid[i]); tab[2 * i + 1] = uint(iq2s_grid[i] >> 32); }
diff --cc metal/src/engine/ds4_finish.cpp
index 000000000,c773eeed5..0bc8f4ad7
mode 000000,100644..100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@@ -1,0 -1,2599 +1,2730 @@@
+ // metal/src/engine/ds4_finish.cpp - the host side of ds4_finish (ds4_kernels.cu:1894, finish_kernel<false>).
+ //
+ // The kernel takes the engine's own device pointer for x as a bound buffer plus an offset, and the Ds4Finish struct as a constant
+ // buffer whose pointer fields are the engine's gpuAddresses (Metal lets a kernel dereference them - ss the repo's note that
+ // "kernels may take raw gpuAddress pointers"). Strides and counts travel as plain values.
+ //
+ // The mrope (rd > 0) and rounding (quant != 0) branches are not written yet, and neither is the KV pack (pdst): this function
+ // returns false for those, so the engine reports the launch line instead of a step being skipped.
+ #include "mtl_rt.h"
+
+ #include <strata/gpu/runtime.hpp>
+ #include <cuda_runtime.h>
+
+ #include <cstdio>
+ #include <cstdint>
+ #include <cstring>
+ #include <algorithm>
+ #include <map>
+ #include <vector>
+
+ // called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
+ extern "C" void ds4_metal_flush();
+ extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);
+ extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off);
+ extern "C" void ds4_gpu_sync_all_streams();
+ extern "C" unsigned long long ds4_dispatch_seq();
+ static unsigned long long g_q8_deduped = 0;
+
+ #define DSV41_PROBE(fmt, ...) do { static bool once_ = false; if (!once_) { once_ = true; std::fprintf(stderr, "[probe] " fmt, __VA_ARGS__); } } while (0)
+
+ namespace strata::dsv41::metal_host {
+
+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Finish (Metal has no 'long long': 64-bit is 'long')
+ struct GpuFinish {
+ uint64_t w; // device float* (the engine's gpuAddress, 0 = none)
+ float eps;
+ int rd, pos, inverse, vpp, pos_step;
+ uint64_t freqs; // device float*
+ int quant;
+ uint64_t dst; // device ushort*
+ int64_t dst_stride;
+ uint64_t pdst; // device uchar*
+ int64_t pdst_stride;
+ };
+
+ namespace {
+ mtl::Context* g_cx = nullptr;
+ int g_lib = -1;
+ int g_lib_iq = -1;
+
+ bool ensure() {
+ if (std::getenv("DS4_DBG_MEM")) { // does a write through host_view read back through cudaMemcpy?
+ void* p = nullptr;
+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
+ if (cudaMalloc(&p, tsz) == cudaSuccess) {
+ void* v = strata::gpu::host_view(p);
+ if (v) {
+ ((float*) v)[0] = 42.0f;
+ float out = 0.0f;
+ cudaMemcpy(&out, p, 4, cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG mem: gpu=%p view=%p wrote 42 through the view, cudaMemcpy gives %g\n", p, v, out);
+ } else {
+ std::fprintf(stderr, "DBG mem: gpu=%p has no host view\n", p);
+ }
+ cudaFree(p);
+ }
+ }
+ if (g_cx) return g_lib >= 0;
+ g_cx = new mtl::Context();
+ g_cx->start_keepalive();
+ g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
+ if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
+ g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);
+ if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");
+ if (g_lib >= 0 && std::getenv("DS4_DBG_BIND")) { // write a constant by each route, copy it back
+ void* p = nullptr;
+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
+ if (cudaMalloc(&p, tsz) == cudaSuccess) {
+ void* v = strata::gpu::host_view(p);
+ float got[4] = {0, 0, 0, 0};
+ {
+ mtl::Buffer b = g_cx->wrap((void*) (((uintptr_t) v) & ~(uintptr_t) 16383), 16384);
+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write");
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(b, ((uintptr_t) v) & 16383), mtl::Arg::V((unsigned) 4)};
+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->flush_runs();
+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG bind A (wrap host_view + Arg::B): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
+ }
+ {
+ uint64_t pp = (uint64_t) (uintptr_t) p;
+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write_v");
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::V(pp), mtl::Arg::V((unsigned) 4)};
+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->flush_runs();
+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG bind B (device pointer by value): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
+ }
+ cudaFree(p);
+ }
+ }
+ return g_lib >= 0;
+ }
+ } // namespace
+
+ // the engine's Finish, as ds4_kernels.cuh declares it
+ struct FinishIn {
+ const float* w;
+ float eps;
+ int rd, pos, inverse, vpp, pos_step;
+ const float* freqs;
+ int quant;
+ const uint16_t* dst;
+ long long dst_stride;
+ uint8_t* pdst;
+ long long pdst_stride;
+ };
+
+ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {
+ if (n_vec <= 0 || len <= 0) return true;
+ // finish_kernel<true> (the KV pack: the row quantised to codes + group scales) or <false>; the <false> rounding
+ // branch (quant without pdst) is the one still unwritten
+ const bool pack = f.pdst != nullptr;
+ if (!ensure()) return false;
+ void* hx = strata::gpu::host_view(x);
+ if (!hx) { std::fprintf(stderr, "dsv41 metal: finish has no host view\n"); return false; }
+ const size_t page = 16384;
+ const size_t bytes = (size_t) (n_vec - 1) * (size_t) stride * sizeof(float) + (size_t) len * sizeof(float);
+ const size_t base = ((uintptr_t) hx) & ~(uintptr_t) (page - 1);
+ const size_t off = ((uintptr_t) hx) - base;
+ mtl::Buffer bx = g_cx->wrap((void*) base, ((bytes + off + page - 1) / page) * page);
+ if (!bx.handle) { std::fprintf(stderr, "dsv41 metal: finish wrap failed\n"); return false; }
+
+ GpuFinish gf{};
+ gf.w = (uint64_t) (uintptr_t) f.w;
+ gf.eps = f.eps;
+ gf.rd = f.rd; gf.pos = f.pos; gf.inverse = f.inverse; gf.vpp = f.vpp; gf.pos_step = f.pos_step;
+ gf.freqs = (uint64_t) (uintptr_t) f.freqs;
+ gf.quant = f.quant;
+ gf.dst = (uint64_t) (uintptr_t) f.dst;
+ gf.dst_stride = f.dst_stride;
+ gf.pdst = (uint64_t) (uintptr_t) f.pdst;
+ gf.pdst_stride = f.pdst_stride;
+
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib, pack ? "ds4_finish_pack" : "ds4_finish");
+ } catch (...) {
+ return false;
+ }
+ // one dispatch, one threadgroup a vector (the kernel's blockIdx.x is threadgroup_position_in_grid)
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bx, off), mtl::Arg::V((long) stride), mtl::Arg::V((unsigned) len), mtl::Arg::V(gf)};
+ d.grid[0] = (uint32_t) n_vec;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256;
+ d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ // The fp8-rounding finishes (the KV rows, the attention output) are read stale by the next dispatch when it shares their
+ // command buffer: end the buffer after them (committed, not waited for: the queue orders the buffers).
+ // (it once ended the command buffer after every fp8-rounding finish, 2 a layer, against a stale read; each extra buffer costs the GPU ~0.3 ms idle, a third of a decode token.
+ // The decode-path score is bit-identical without it, the dispatch barrier after every dispatch orders the readers. DS4_FINISH_SPLIT=1 brings it back.)
+ static const bool split_after = std::getenv("DS4_FINISH_SPLIT") != nullptr;
+ if (f.quant && split_after) g_cx->split_runs();
+ if (const char* fm2 = std::getenv("DS4_FLUSH_FINISH_CTX")) { if (f.quant) { if (std::atoi(fm2) == 2) g_cx->split_runs(); else g_cx->flush_runs(); } }
+ if (const char* fm = std::getenv("DS4_FLUSH_FINISH")) { // debugging: which finish calls need a drain after them
+ const int m = std::atoi(fm);
+ if ((m == 1 && f.dst) || (m == 2 && f.rd > 0) || (m == 3 && f.w) || (m == 4 && f.quant) || (m == 5 && !f.dst) ||
+ (m == 6 && f.dst == nullptr && f.rd == 0) || (m == 7 && f.rd > 0 && f.inverse) || (m == 8 && f.rd > 0 && !f.inverse && !f.dst) || (m == 9 && f.rd > 0 && !f.dst) || (m == 10 && (f.dst || f.inverse)) || (m == 11 && f.rd > 0 && !f.inverse))
+ ds4_metal_flush();
+ }
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1677 - hc_post_inplace: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]
+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T) {
+ if (!ensure()) return false;
+ void* hy = strata::gpu::host_view((void*) y);
+ void* hh = strata::gpu::host_view(h);
+ if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
+ if (!by.handle || !bh.handle) return false;
+ // post is [T][8], comb is [T][64]: device operands like the others (reached through their host views)
+ void* hpost = strata::gpu::host_view((void*) post);
+ void* hcomb = strata::gpu::host_view((void*) comb);
+ if (!hpost || !hcomb) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view for post / comb\n"); return false; }
+ mtl::Buffer bp = wrap(hpost, (size_t) T * 8 * 4);
+ mtl::Buffer bc = wrap(hcomb, (size_t) T * 64 * 4);
+ if (!bp.handle || !bc.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp, off(hpost)), mtl::Arg::B(bc, off(hcomb)),
+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
+ d.grid[0] = (uint32_t) ((dim + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ const bool dbg = std::getenv("DS4_DBG_SG") != nullptr;
+ if (dbg) {
+ g_cx->flush_runs();
+ const float* fy = (const float*) hy; const float* fh = (const float*) hh; const float* fp = (const float*) hpost; const float* fc = (const float*) hcomb;
+ std::fprintf(stderr, "hc_post in: y %g %g h %g %g | %g %g post %g %g %g %g comb %g %g %g %g\n", fy[0], fy[1], fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1], fp[0], fp[1], fp[2], fp[3], fc[0], fc[1], fc[2], fc[3]);
+ }
+ g_cx->run({d});
+ if (dbg) {
+ g_cx->flush_runs();
+ const float* fh = (const float*) hh;
+ std::fprintf(stderr, "hc_post out: h %g %g | %g %g\n", fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1]);
+ }
+ g_cx->free(bp);
+ g_cx->free(bc);
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1692 - hc_mix_scale: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps)
+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes) {
+ if (!ensure()) return false;
+ void* hh = strata::gpu::host_view((void*) h);
+ void* hm = strata::gpu::host_view((void*) mixes);
+ if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
+ if (!bh.handle || !bm.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::V((unsigned) hcdim),
+ mtl::Arg::V((unsigned) n_mix), mtl::Arg::V(eps)};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fc = (const float*) hm;
+ std::fprintf(stderr, "hc_mix_scale T %d hcdim %d n_mix %d eps %g -> %g %g %g\n", T, hcdim, n_mix, eps, fc[0], fc[1], fc[2]);
+ }
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1901 - hc_mix: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps)
+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes) {
+ if (!ensure()) return false;
+ void* hh = strata::gpu::host_view((void*) h);
+ void* hf = strata::gpu::host_view((void*) fn);
+ void* hm = strata::gpu::host_view((void*) mixes);
+ if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );
+ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
+ if (!bh.handle || !bf.handle || !bm.handle) return false;
+ mtl::Pipeline pl;
+ const bool all = T >= 16 && n_mix <= 32 && !std::getenv("DS4_NO_HC_ALL"); // a prompt chunk: a threadgroup a token, 4 mixes a pass
+ try { pl = g_cx->pipeline(g_lib, all ? "ds4_hc_mix_all" : "ds4_hc_mix"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),
+ mtl::Arg::V((unsigned) hcdim), mtl::Arg::V(eps)};
+ if (all) d.args.push_back(mtl::Arg::V((unsigned) n_mix));
+ d.grid[0] = all ? (uint32_t) T : (uint32_t) n_mix;
+ d.grid[1] = all ? 1u : (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1908 - hc_split: the sinkhorn normalisation (scalar path; see the MSL comment)
+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
+ float* pre, float* post, float* comb) {
+ if (!ensure()) return false;
+ void* hm = strata::gpu::host_view((void*) mixes);
+ void* hp = strata::gpu::host_view((void*) pre);
+ void* ho = strata::gpu::host_view((void*) post);
+ void* hc_ = strata::gpu::host_view((void*) comb);
+ if (!hm || !hp || !ho || !hc_) { std::fprintf(stderr, "dsv41 metal: hc_split has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 );
+ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 );
+ if (!bm.handle || !bp.handle || !bo.handle || !bc.handle) return false;
+ // scale and base come from the engine like every other operand: reach them through host_view, not as raw host pointers
+ void* hs = strata::gpu::host_view((void*) scale);
+ void* hb = strata::gpu::host_view((void*) base);
+ if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }
+ const size_t nbase = (size_t) (2 * hc + hc * hc);
+ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));
+ mtl::Buffer bb = wrap(hb, nbase * sizeof(float));
+ if (!bs.handle || !bb.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, hc == 4 && !std::getenv("DS4_NO_HCS4") ? "ds4_hc_split4" : "ds4_hc_split"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),
+ mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),
+ mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fm = (const float*) hm; const float* fs = (const float*) hs; const float* fb = (const float*) hb;
+ std::fprintf(stderr, "hc_split in: mix %g %g %g scale %g %g %g base %g %g\n", fm[0], fm[1], fm[2], fs[0], fs[1], fs[2], fb[0], fb[1]);
+ }
+ g_cx->run({d});
+ if (std::getenv("DS4_DBG_SG")) {
+ g_cx->flush_runs();
+ const float* fp = (const float*) hp;
+ std::fprintf(stderr, "hc_split out: pre %g %g %g %g\n", fp[0], fp[1], fp[2], fp[3]);
+ }
+ return true;
+ }
+
+
++// the hyper-connection mix and split in one dispatch (ds4_hc_mix_split): the engine used to issue hc_mix and hc_split
++// back to back for the same token, and the kernel does both in one threadgroup, bit for bit as before.
++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,
++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
++ float* comb) {
++ if (!ensure()) return false;
++ void* hh = strata::gpu::host_view((void*) h);
++ void* hf = strata::gpu::host_view((void*) fn);
++ void* hm = strata::gpu::host_view((void*) mixes);
++ void* hs = strata::gpu::host_view((void*) scale);
++ void* hb = strata::gpu::host_view((void*) base);
++ void* hp = strata::gpu::host_view((void*) pre);
++ void* ho = strata::gpu::host_view((void*) post);
++ void* hc_ = strata::gpu::host_view((void*) comb);
++ if (!hh || !hf || !hm || !hs || !hb || !hp || !ho || !hc_) {
++ std::fprintf(stderr, "dsv41 metal: hc_mix_split has no host view\n");
++ return false;
++ }
++ const size_t page = 16384;
++ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
++ const size_t base_ = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
++ const size_t end = (size_t) ((uintptr_t) host) + bytes;
++ return g_cx->wrap((void*) base_, ((end - base_ + page - 1) / page) * page);
++ };
++ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(size_t) (page - 1))); };
++ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4);
++ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4);
++ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4);
++ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));
++ mtl::Buffer bb = wrap(hb, (size_t) (2 * hc + hc * hc) * sizeof(float));
++ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4);
++ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4);
++ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4);
++ if (!bh.handle || !bf.handle || !bm.handle || !bs.handle || !bb.handle || !bp.handle || !bo.handle || !bc.handle)
++ return false;
++ mtl::Pipeline pl;
++ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_split"); } catch (...) { return false; }
++ mtl::Dispatch d;
++ d.p = &pl;
++ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),
++ mtl::Arg::V((unsigned) hcdim), mtl::Arg::V((float) eps), mtl::Arg::V((unsigned) n_mix),
++ mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),
++ mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),
++ mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};
++ d.grid[0] = (uint32_t) T;
++ d.grid[1] = d.grid[2] = 1;
++ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
++ g_cx->run({d});
++ return true;
++}
++
++
+ // ds4_kernels.cu:1938 - moe_sort: count each virtual slot's picks, prefix-sum the counts, fill in token order.
+ // Three dispatches on one command buffer, so their order is the order of the calls (serial dispatch).
+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos) {
+ if (!ensure()) return false;
+ void* hi = strata::gpu::host_view((void*) idx);
+ void* hv = strata::gpu::host_view((void*) vs);
+ void* hb = strata::gpu::host_view((void*) bounds);
+ void* hr = strata::gpu::host_view((void*) row_tok);
+ void* hp = strata::gpu::host_view((void*) pos);
+ if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t t_bytes = (size_t) T * (size_t) k * 4;
+ mtl::Buffer bi = wrap(hi, t_bytes);
+ mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );
+ mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );
+ mtl::Buffer br = wrap(hr, t_bytes);
+ mtl::Buffer bp = wrap(hp, t_bytes);
+ if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;
+ mtl::Pipeline pc, pf, ps;
+ try {
+ pc = g_cx->pipeline(g_lib, "ds4_moe_sort_count");
+ pf = g_cx->pipeline(g_lib, "ds4_moe_sort_fill");
+ ps = g_cx->pipeline(g_lib, "ds4_bounds_scan");
+ } catch (...) { return false; }
+ // 1. counts
+ mtl::Dispatch d1;
+ d1.p = &pc;
+ d1.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb))};
+ d1.grid[0] = (uint32_t) n_vs;
+ d1.grid[1] = d1.grid[2] = 1;
+ d1.tg[0] = 256; d1.tg[1] = d1.tg[2] = 1;
+ g_cx->run({d1});
+ // 2. the exclusive prefix over the slots
+ mtl::Dispatch d2;
+ d2.p = &ps;
+ d2.args = {mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n_vs)};
+ d2.grid[0] = 1;
+ d2.grid[1] = d2.grid[2] = 1;
+ d2.tg[0] = 32; d2.tg[1] = d2.tg[2] = 1;
+ g_cx->run({d2});
+ // 3. the fill
+ mtl::Dispatch d3;
+ d3.p = &pf;
+ d3.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp))};
+ d3.grid[0] = (uint32_t) n_vs;
+ d3.grid[1] = d3.grid[2] = 1;
+ d3.tg[0] = 256; d3.tg[1] = d3.tg[2] = 1;
+ g_cx->run({d3});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1690 - the Engram row decoder (fmt 2 = IQ4_NL, this model's tables)
+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt) {
+ if (T <= 0) return true;
+ if (!ensure()) return false;
+ void* hs = strata::gpu::host_view((void*) src);
+ void* ho = strata::gpu::host_view((void*) out);
+ if (!hs || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant_tok has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t rows = (size_t) T * (size_t) cols;
+ mtl::Buffer bs = wrap(hs, (size_t) T * per );
+ mtl::Buffer bo = wrap(ho, rows * (size_t) hd * 4 );
+ if (!bs.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((unsigned long) per), mtl::Arg::V((unsigned) cols),
+ mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((int) fmt)};
+ d.grid[0] = (uint32_t) rows;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:341 - hc_pre_norm: a = sum_s mix[s]*h[s][e]; x = a * rsqrt(mean(a^2) + eps) (x w when given)
+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x) {
+ if (!ensure()) return false;
+ void* hh = strata::gpu::host_view((void*) h);
+ void* hm = strata::gpu::host_view((void*) mix);
+ void* hx = strata::gpu::host_view((void*) x);
+ if (!hh || !hm || !hx) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );
+ mtl::Buffer bm = wrap(hm, (size_t) T * 8 * 4 );
+ mtl::Buffer bx = wrap(hx, (size_t) T * dim * 4 );
+ if (!bh.handle || !bm.handle || !bx.handle) return false;
+ mtl::Buffer bw;
+ size_t w_off = 0;
+ if (w) {
+ void* hw = strata::gpu::host_view((void*) w);
+ if (!hw) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm weight has no host view\n"); return false; }
+ bw = wrap(hw, (size_t) dim * 4 );
+ w_off = off(hw);
+ if (!bw.handle) return false;
+ } else {
+ bw = bx; // never read when has_w == 0
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_pre_norm"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bw, w_off),
+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc), mtl::Arg::V(eps), mtl::Arg::B(bx, off(hx)),
+ mtl::Arg::V((unsigned) (w ? 1 : 0))};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_gguf.cu:186 - the activation quantiser the expert matmuls consume (Q8_0: 32 values, one scale)
+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
+ if (rows <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ void* hx = strata::gpu::host_view((void*) x);
+ void* hq = strata::gpu::host_view((void*) q);
+ void* hd = strata::gpu::host_view((void*) d);
+ if (!hx || !hq || !hd) { std::fprintf(stderr, "dsv41 metal: q8_quant has no host view\n"); return false; }
+ const size_t page = 16384;
+ // The pages the buffer itself spans: [base, base + off + bytes). A fixed slack (the +65536 the call sites used to
+ // carry) walks off the end of the mapping when the buffer is small and sits at an arena chunk's edge, and Metal then
+ // refuses the no-copy wrap ("mtl: wrap 81920 failed" - the hd scales here, a few pages).
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bx = wrap(hx, (size_t) rows * ldx * 4);
+ mtl::Buffer bq = wrap(hq, (size_t) rows * k);
+ mtl::Buffer bd = wrap(hd, (size_t) rows * (k / 32) * 4);
+ if (!bx.handle || !bq.handle || !bd.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bd, off(hd))};
+ const uint groups = (uint32_t) ((k / 32 + 7) / 8);
+ dsp.grid[0] = groups;
+ dsp.grid[1] = (uint32_t) rows;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+
+ // the GPU address of host weight memory (see mtl::Context::zc_map); 0: not available
+ uint64_t zc_map(const void* p, size_t bytes) {
+ if (!ensure()) return 0;
+ return g_cx->zc_map(p, bytes);
+ }
+
+ // Several ds4_q8_quant calls in one dispatch (specs: {x, ldx, rows, k, q, d} with the engine's device pointers, which are the
+ // kernels' own addresses, as iq_mv's jobs carry them)
+ namespace {
+ struct HostQ8Spec {
+ const float* x;
+ uint64_t ldx;
+ uint32_t rows, k;
+ int8_t* q;
+ float* d;
+ };
+ } // namespace
+ bool q8_quant_multi(const void* specs, int n) {
+ if (n <= 0) return true;
+ if (!ensure()) return false;
+ const HostQ8Spec* sp = (const HostQ8Spec*) specs;
+ uint32_t max_rows = 1, max_k = 32;
+ for (int i = 0; i < n; ++i) {
+ max_rows = std::max(max_rows, sp[i].rows);
+ max_k = std::max(max_k, sp[i].k);
+ }
+ mtl::Buffer bj = g_cx->alloc((size_t) n * sizeof(HostQ8Spec));
+ std::memcpy(bj.ptr, sp, (size_t) n * sizeof(HostQ8Spec));
+ static mtl::Pipeline pl;
+ static bool have = false;
+ if (!have) {
+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant_multi"); } catch (...) { g_cx->free(bj); return false; }
+ have = true;
+ }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (max_k / 32 + 7) / 8;
+ d.grid[1] = max_rows;
+ d.grid[2] = (uint32_t) n;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+
+
+ // ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so
+ // the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.
+ namespace {
+ // mirrors ds4_gguf.cu's BfJob (and the kernels' Ds4MxJob)
+ struct HostBfJob {
+ const void* w;
+ uint64_t ld;
+ const float* x;
+ uint64_t xcs;
+ float* y;
+ uint64_t ycs;
+ const float* bias;
+ float alpha;
+ int k, n, ncol;
+ const int8_t* xq;
+ const float* xd;
+ };
+ } // namespace
+
+ bool mx_mv(const void* jobs_in, int nj_in, int max_n, size_t job_bytes, int fmt) {
+ if (nj_in <= 0 || max_n <= 0) return true;
+ if (!ensure()) return false;
+ // The matvec kernels keep 8 accumulators (the T=8 window): a job of more activations (a 16-token window) is cut into
+ // jobs of at most 8, each reading its own activations and writing its own rows of y. (Left whole, the columns past the
+ // eighth were never computed: rows of zeros in every projection of a prompt window.)
+ std::vector<HostBfJob> cut;
+ if (job_bytes == sizeof(HostBfJob)) {
+ const HostBfJob* src = (const HostBfJob*) jobs_in;
+ for (int i = 0; i < nj_in; ++i) {
+ const HostBfJob& q = src[i];
+ const int ncol = q.ncol > 0 ? q.ncol : 1;
+ for (int c0 = 0; c0 < ncol; c0 += 8) {
+ HostBfJob b = q;
+ b.ncol = std::min(8, ncol - c0);
+ if (c0 != 0) {
+ if (q.x) b.x = q.x + (size_t) c0 * (size_t) q.xcs;
+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;
+ if (q.xq) b.xq = q.xq + (size_t) c0 * (size_t) q.k;
+ if (q.xd) b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
+ }
+ cut.push_back(b);
+ }
+ }
+ }
+ const void* jobs = cut.empty() ? jobs_in : (const void*) cut.data();
+ const int nj = cut.empty() ? nj_in : (int) cut.size();
+ mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);
+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }
+ std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);
+ mtl::Pipeline pl;
+ // fmt 1 = MXFP4, 2 = Q8_0, 0 = BF16 (a dense matrix: the engram's wkv, ds4_gguf.cu:1064)
+ const char* fn = fmt == 2 ? "ds4_q8_mv" : (fmt == 1 ? "ds4_mx_mv" : "ds4_bf16_mv");
+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);
+ d.grid[1] = (uint32_t) nj;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+
+
+ // ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they
+ // travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.
+ static inline void* cpu_addr(void* p);
+ namespace {
+ // mirrors ds4_iq.cuh's MvJob (and ds4_iq.metal's Ds4MvJob): 80 bytes on arm64
+ struct Ds4IqMvJob {
+ const uint8_t* w;
+ uint64_t ld;
+ int type, k, n, ncol;
+ const int8_t* xq;
+ const float* xd;
+ float* y;
+ uint64_t ycs;
+ const float* bias;
+ float alpha;
+ };
+ static_assert(sizeof(Ds4IqMvJob) == 80, "Ds4IqMvJob layout must match ds4_iq.cuh's MvJob");
+ } // namespace
+
+ bool iq_mv(const void* jobs, int nj) {
+ if (nj <= 0) return true;
+ if (!ensure()) return false;
+ const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
+ // Each job goes to the kernel of its type and column count: NC = 1, 2, 4 or 8 activations decode the row's weights once
+ // (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.
+ std::map<std::string, std::vector<Ds4IqMvJob>> groups;
+ int max_n = 0;
+ if (std::getenv("DS4_DBG_JOBS")) {
+ std::fprintf(stderr, "iq_mv call:");
+ for (int i = 0; i < nj; ++i) std::fprintf(stderr, " [t%d n%d k%d c%d]", src[i].type, src[i].n, src[i].k, src[i].ncol);
+ std::fprintf(stderr, "\n");
+ }
+ static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;
+ for (int i = 0; i < nj; ++i) {
+ const Ds4IqMvJob& q = src[i];
+ { static const char* sk = std::getenv("DS4_SKIP_TYPES"); if (sk != nullptr && std::strstr(sk, ("," + std::to_string(q.type) + ",").c_str()) != nullptr) continue; } // (timing only)
+ if (q.n > max_n) max_n = q.n;
+ const int ncol = q.ncol > 0 ? q.ncol : 1;
+ const char* tn = nullptr;
+ switch (q.type) {
+ case 13: tn = "ds4_q5k_mvn"; break;
+ case 16: tn = "ds4_iq2xxs_mvn"; break;
+ case 17: tn = "ds4_iq2xs_mvn"; break;
+ case 18: tn = "ds4_iq3xxs_mvn"; break;
+ case 22: tn = "ds4_iq2s_mvn"; break;
++ case 29: tn = "ds4_iq1m_mvn"; break; // (the 2.00 bpw pack's IQ1_M: ds4_iq.metal's ds4_w32_iq1m)
+ default: break;
+ }
+ const bool grid = tn != nullptr && !old_only;
+ const int step = grid ? 8 : (ncol > 16 ? 16 : ncol);
+ for (int c0 = 0; c0 < ncol; c0 += step) {
+ Ds4IqMvJob b = q;
+ b.ncol = (c0 + step < ncol) ? step : ncol - c0;
+ if (c0 != 0) {
+ b.xq = q.xq + (size_t) c0 * (size_t) q.k;
+ b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;
+ }
+ std::string fn;
+ if (grid) {
+ const int nc = b.ncol <= 1 ? 1 : (b.ncol <= 2 ? 2 : (b.ncol <= 4 ? 4 : 8));
+ fn = std::string(tn) + std::to_string(nc);
+ } else {
+ fn = (b.type == 13 || b.type == 14) ? "ds4_iq_mv_k" : "ds4_iq_mv";
+ }
+ groups[fn].push_back(b);
+ }
+ }
+ if (max_n <= 0) return true;
+ const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group
+ for (auto& kv : groups) {
+ std::vector<Ds4IqMvJob>& v = kv.second;
+ mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib_iq, kv.first);
+ } catch (...) {
+ g_cx->free(bj);
+ return false;
+ }
+ std::vector<mtl::Dispatch> ds;
+ for (size_t base = 0; base < v.size(); base += kBatch) {
+ const size_t cnt = std::min(kBatch, v.size() - base);
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj, base * sizeof(Ds4IqMvJob))};
+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);
+ d.grid[1] = (uint32_t) cnt;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ ds.push_back(d);
+ }
+ g_cx->run(ds);
+ g_cx->free(bj);
+ }
+ return true;
+ }
+
+
+ // ds4_iq.cu:536 gate_up_kernel<T, NE> / :588 down_kernel<T, NE> - the prompt / window path's expert GEMM (the launches at
+ // :958 launch_gu / :960 launch_dn). A threadgroup of 256 is 8 warps: a warp RW rows (gate / up 1, down 2), a lane the
+ // row's sub-blocks, the group's entries four at a pass. The group's blob and its entries are read the way the CUDA kernel
+ // reads them, so the plan / window pointer arrays travel as they are: every pointer here is a CPU address (unified memory)
+ // and a kernel may take raw gpuAddresses (as ds4_finish's f.w does).
+ namespace {
+ // (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4IqJob (Metal has no 'long long': 64-bit is 'long')
+ struct IqJob {
+ const void* plan_ptr;
+ const void* grp_ptr;
+ const void* grp_start;
+ const void* grp_ent;
+ const void* n_groups;
+ const void* xq;
+ const void* xd;
+ const void* hq;
+ const void* hd;
+ void* h;
+ void* part;
+ int64_t tr0, tr1, tr2;
+ int n_embd, n_ff, type;
+ float limit;
+ };
+
+ static bool iq_gemm(const IqJob& j, const char* fn, int grid_x, int groups) {
+ if (!ensure()) return false;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) grid_x;
+ d.grid[1] = (uint32_t) groups;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+ } // namespace
+
+ // gate / up of the group's entries: h = the clamped swiglu (ds4_iq.cuh's Args: tr[0] gate, tr[1] up, tr[2] down)
+ bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
+ const int8_t* xq, const float* xd, float* h, int type, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 256 != 0 || n_ff % 8 != 0) return false;
+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ IqJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.xq = xq;
+ j.xd = xd;
+ j.h = h;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.type = type;
+ j.limit = limit;
+ return iq_gemm(j, "ds4_iq_gu", n_ff / 8, groups);
+ }
+
+ // down of the group's entries against their int8 h: part (a row of n_embd an entry)
+ bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
+ const float* hd, float* part, int type, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 256 != 0 || n_embd % 16 != 0) return false;
+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ IqJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.hq = hq;
+ j.hd = hd;
+ j.part = part;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.type = type;
+ j.limit = 0.0f;
+ return iq_gemm(j, "ds4_iq_dn", n_embd / 16, groups);
+ }
+
+
+ // ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel - the MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down /
+ // win_experts). The same frame as iq_gu / iq_dn above: a threadgroup of 256 is 8 warps, a warp a row, a lane strides the
+ // 32-value blocks, the group's entries sixteen at a pass. The job struct is the kernel's own (metal/src/engine/msl/
+ // ds4_iq.metal's Ds4MxJob); the plan / window pointer arrays travel as they are: every pointer here is a CPU address
+ // (unified memory) and a kernel may take raw gpuAddresses.
+ namespace {
+ // (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4MxJob (Metal
+ // has no 'long long': 64-bit is 'long')
+ struct MxJob {
+ const void* plan_ptr;
+ const void* grp_ptr;
+ const void* grp_start;
+ const void* grp_ent;
+ const void* n_groups;
+ const void* xq;
+ const void* xd;
+ const void* hq;
+ const void* hd;
+ void* h;
+ void* part;
+ int64_t tr0, tr1, tr2; // ('long' there, so 8 a field and the float ends the struct on the same byte)
+ int n_embd, n_ff;
+ float limit;
+ };
+ static_assert(sizeof(MxJob) == 128, "Ds4MxJob's layout");
+
+ static bool mx_gemm(const MxJob& j, const char* fn, int grid_x, int groups) {
+ if (!ensure()) return false;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) grid_x;
+ d.grid[1] = (uint32_t) groups;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+ } // namespace
+
+ // gate / up of the group's experts: h = the clamped swiglu (ds4_gguf.cu's MxArgs: tr[0] gate, tr[1] up, tr[2] down)
+ bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,
+ const int8_t* xq, const float* xd, float* h, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 32 != 0 || n_ff % 8 != 0) return false;
+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ MxJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.xq = xq;
+ j.xd = xd;
+ j.h = h;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ j.limit = limit;
+ return mx_gemm(j, "ds4_mx_gu", (n_ff + 7) / 8, groups);
+ }
+
+ // down of the group's experts against their int8 h: part (a row of n_embd an entry)
+ bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,
+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,
+ const float* hd, float* part, int groups) {
+ if (groups <= 0) return true;
+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 32 != 0 || n_embd % 8 != 0) return false;
+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;
+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;
+ MxJob j{};
+ j.plan_ptr = plan_ptr;
+ j.grp_ptr = grp_ptr;
+ j.grp_start = grp_start;
+ j.grp_ent = grp_ent;
+ j.n_groups = n_groups;
+ j.hq = hq;
+ j.hd = hd;
+ j.part = part;
+ j.tr0 = (int64_t) tr[0];
+ j.tr1 = (int64_t) tr[1];
+ j.tr2 = (int64_t) tr[2];
+ j.n_embd = n_embd;
+ j.n_ff = n_ff;
+ return mx_gemm(j, "ds4_mx_dn", (n_embd + 7) / 8, groups);
+ }
+
+ // ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),
+ // the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block
+ namespace {
+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Comb (a 'long' there is int64_t here)
+ struct MxCombJob {
+ const void* plan_ptr;
+ const void* plan_w;
+ const void* part;
+ const void* cpu_part;
+ const int* cpu_flag;
+ const void* sh_out;
+ void* out;
+ int64_t cpu_stride, out_stride;
+ int k, n_embd;
+ };
+ } // namespace
+
+ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride,
+ const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd) {
+ if (T <= 0 || n_embd <= 0) return true;
+ if (k <= 0 || plan_ptr == nullptr || plan_w == nullptr || part == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ MxCombJob j{};
+ j.plan_ptr = plan_ptr;
+ j.plan_w = plan_w;
+ j.part = part;
+ j.cpu_part = cpu_part;
+ j.cpu_flag = cpu_flag;
+ j.sh_out = sh_out;
+ j.out = out;
+ j.cpu_stride = (int64_t) cpu_stride;
+ j.out_stride = (int64_t) out_stride;
+ j.k = k;
+ j.n_embd = n_embd;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: mx_combine job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_mx_combine"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((n_embd + 127) / 128);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+
+ // ds4_kernels.cu:1092 - the KV ring row to BF16 (E4M3 code x E8M0 group scale)
+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes) {
+ if (n <= 0 || d <= 0) return true;
+ if (!ensure()) return false;
+ void* hr = strata::gpu::host_view((void*) ring);
+ void* he = strata::gpu::host_view(ext);
+ if (!hr || !he) { std::fprintf(stderr, "dsv41 metal: ring_to_ext has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
+ mtl::Buffer be = wrap(he, (size_t) n * d * 2 );
+ if (!br.handle || !be.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_ring_to_ext"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::B(be, off(he)), mtl::Arg::V((long) p0),
+ mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};
+ dsp.grid[0] = (uint32_t) n;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1104 - the window back into the ring (E4M3 codes + E8M0 group scales, optional backup)
+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup) {
+ if (T <= 0 || n <= 0 || d <= 0) return true;
+ if (!ensure()) return false;
+ void* he = strata::gpu::host_view((void*) ext);
+ void* hr = strata::gpu::host_view(ring);
+ if (!he || !hr) { std::fprintf(stderr, "dsv41 metal: ext_to_ring has no host view\n"); return false; }
+ void* hb = backup ? strata::gpu::host_view(backup) : nullptr;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer be = wrap(he, (size_t) (n + T) * d * 2 );
+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );
+ if (!be.handle || !br.handle) return false;
+ mtl::Buffer bb = br; // never written when backup is null
+ size_t b_off = 0;
+ if (hb) { bb = wrap(hb, (size_t) T * row_bytes ); b_off = off(hb); if (!bb.handle) return false; }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_ext_to_ring"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),
+ mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),
+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes), mtl::Arg::V((unsigned) (hb ? 1 : 0))};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)
+
+ namespace {
+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4AttnRows (ds4_kernels.cuh:149 AttnRows, kWin = 16)
+ struct Ds4AttnRowsJob {
+ int win0[16], nwin[16], nsel[16];
+ int gen, t0, win, R, kcap;
+ int64_t p0;
+ };
+ static_assert(sizeof(Ds4AttnRowsJob) == 224, "Ds4AttnRowsJob layout must match the MSL struct");
+ } // namespace
+
+ // ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false> - the prompt path's sparse attention.
+ // The packed kernel stages rows once for G heads of a query; the BF16 kernel handles rows already in BF16 (a DSpark block).
+ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows,
+ const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt) {
+ if (T <= 0 || heads <= 0) return true;
+ if (d != 512 || rows == nullptr || (ckv_fmt != 0 && ckv_fmt != 2)) { std::fprintf(stderr, "dsv41 metal: sparse_attn unsupported\n"); return false; }
+ Ds4AttnRowsJob ar{};
++ ar.t0 = (int) T; // the two-query kernel reads the row count here on the gen == 0 path
+ std::memcpy(&ar, rows, sizeof(ar));
+ if (q == nullptr || out == nullptr) return false;
+ if (ar.gen == 0 && T > 16) { std::fprintf(stderr, "dsv41 metal: sparse_attn gen0 T>16\n"); return false; }
+ if (!ensure()) return false;
+
+ void* hq = cpu_addr((void*) q);
+ void* hw = cpu_addr((void*) win);
+ void* hc = cpu_addr((void*) ckv);
+ void* hs = cpu_addr((void*) sel);
+ void* hk = cpu_addr((void*) sink);
+ void* ho = cpu_addr((void*) out);
+ if (!hq || !ho) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+
+ int wmax = 0;
+ if (ar.gen != 0) {
+ wmax = ar.win + T + 1;
+ } else {
+ for (int t = 0; t < T; ++t) wmax = ar.win0[t] + ar.nwin[t] > wmax ? ar.win0[t] + ar.nwin[t] : wmax;
+ wmax += 1;
+ }
+ if (wmax > 0 && hw == nullptr) { std::fprintf(stderr, "dsv41 metal: sparse_attn win null (wmax %d)\n", wmax); return false; }
+ const size_t wrows = (size_t) (wmax > 0 ? wmax : 1);
+ size_t crows = 0;
+ if (ckv_fmt == 2) {
+ if (ar.gen != 0 && ar.R > 0) {
+ crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position
+ } else {
+ // (the rows the selection names are on the device and not read back here: an earlier build read them on the
+ // host while the kernel that wrote them was still queued, and sized the binding from stale zeros)
+ const size_t rem = ds4_gpu_alloc_remaining(hc);
+ crows = rem > 0 ? rem / 288 : (size_t) (ar.nsel[0] > 0 ? ar.nsel[0] : 1);
+ }
+ if (crows < 1) crows = 1;
+ } else {
+ crows = (size_t) (T > 16 ? T : 16);
+ }
+ const size_t qbytes = (size_t) T * (size_t) heads * 512 * 4;
+ const size_t wbytes = wrows * 512 * 2;
+ const size_t cbytes = crows * (ckv_fmt == 2 ? 288 : 1024);
+ const size_t obytes = (size_t) T * (size_t) heads * 512 * 4;
+ const size_t sbytes = (size_t) T * (size_t) (sel_stride > 0 ? sel_stride : 1) * 4;
+ mtl::Buffer bq = wrap(hq, qbytes);
+ mtl::Buffer bw = hw != nullptr ? wrap(hw, wbytes) : g_cx->alloc(64);
+ mtl::Buffer bc = hc != nullptr ? wrap(hc, cbytes) : g_cx->alloc(64);
+ mtl::Buffer bs = hs != nullptr ? wrap(hs, sbytes) : g_cx->alloc(4);
+ mtl::Buffer bk = hk != nullptr ? wrap(hk, (size_t) heads * 4) : g_cx->alloc((size_t) heads * 4);
+ mtl::Buffer bo = wrap(ho, obytes);
+ if (!bq.handle || !bw.handle || !bc.handle || !bs.handle || !bk.handle || !bo.handle) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn wrap failed\n");
+ return false;
+ }
+
+ int G = 8;
+ if (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);
+ while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)
+
+ static const int attn_mode = std::getenv("DS4_ATTN_MX") ? std::atoi(std::getenv("DS4_ATTN_MX")) : 1; // 1 (default): matrix-unit kernel; 0: the scalar one; 2: both, compare
+ const bool use_mx = ckv_fmt == 2 && G == 8 && attn_mode != 0;
- const char* fn = use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"
++ // DS4_ATTN2Q: two queries a threadgroup, sharing the staged keys (the staging is ~20 % of a 32K prompt and is per
++ // key, so sharing halves it). Verified bit-exact in tools/ds4/attn_harness, including the causal shape where the
++ // two rows have different window lengths.
++ static const bool attn2q_env = std::getenv("DS4_ATTN2Q") != nullptr && use_mx;
++ // Two queries a threadgroup need a shared window base and selection list (the kernel stages one key set for both,
++ // and the causal shape gives the second query a superset only when those agree). The rows carry their own
++ // win0/nwin/nsel, so the pairing is decided here, per dispatch, and never assumed.
++ bool attn2q = attn2q_env;
++ if (attn2q) {
++ for (int t = 1; t < T; ++t) {
++ if (ar.win0[t] != ar.win0[0] || ar.nsel[t] != ar.nsel[0] || ar.nwin[t] < ar.nwin[t - 1]) { attn2q = false; break; }
++ }
++ }
++ const int qpt = attn2q ? 2 : 1;
++ const char* fn = attn2q ? "ds4_sparse_attn_mx2" : use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"
+ : G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1")
+ : "ds4_sparse_attn_bf16";
+ mtl::Pipeline pl;
+ try {
- pl = g_cx->pipeline(g_lib, fn);
++ // DS4_ATTN_PROBE=N: the attention timing probe (see ATTN_PROBE in ds4_init.metal): 1 skips P.V, 2 skips QK.
++ // The numbers it produces are wrong on purpose.
++ const char* ap = std::getenv("DS4_ATTN_PROBE");
++ if (ap != nullptr && std::atoi(ap) != 0) {
++ mtl::Constants cs;
++ cs.i(25, std::atoi(ap));
++ pl = g_cx->pipeline_uncached(g_lib, fn, cs);
++ } else {
++ pl = g_cx->pipeline(g_lib, fn);
++ }
+ } catch (...) {
+ std::fprintf(stderr, "dsv41 metal: sparse_attn pipeline %s unavailable\n", fn);
+ return false;
+ }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, (hw != nullptr ? off(hw) : (size_t) 0)),
+ mtl::Arg::B(bc, (hc != nullptr ? off(hc) : (size_t) 0)),
+ mtl::Arg::B(bs, (hs != nullptr ? off(hs) : (size_t) 0)),
+ mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),
+ mtl::Arg::B(bk, (hk != nullptr ? off(hk) : (size_t) 0)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),
+ mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);
- dsp.grid[1] = (uint32_t) T;
++ dsp.grid[1] = (uint32_t) ((T + qpt - 1) / qpt);
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256;
+ dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the difference
+ mtl::Pipeline ps;
+ try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }
+ mtl::Buffer bo2 = g_cx->alloc(obytes);
+ mtl::Dispatch d2 = dsp;
+ d2.p = &ps;
+ d2.args.back() = mtl::Arg::B(bo2, 0);
+ g_cx->run({d2});
+ g_cx->flush_runs();
+ const float* a = (const float*) ho;
+ const float* c = (const float*) bo2.ptr;
+ double mx_abs = 0, mx_ref = 0, sum_sq = 0, sum_ref = 0;
+ for (size_t i = 0; i < obytes / 4; ++i) {
+ const double df = std::fabs((double) a[i] - (double) c[i]);
+ mx_abs = std::max(mx_abs, df);
+ mx_ref = std::max(mx_ref, (double) std::fabs(c[i]));
+ sum_sq += df * df;
+ sum_ref += (double) c[i] * (double) c[i];
+ }
+ static double worst_rel = 0; static long calls = 0;
+ const double rel = std::sqrt(sum_sq / (sum_ref + 1e-30));
+ worst_rel = std::max(worst_rel, rel);
+ if (++calls % 50 == 1) std::fprintf(stderr, "attn mx vs scalar: call %ld T %d max abs %.3g (max |ref| %.3g) rms rel %.3g worst rms rel %.3g\n", calls, T, mx_abs, mx_ref, rel, worst_rel);
+ g_cx->free(bo2);
+ }
+ return true;
+ }
+
+
+ // ds4_kernels.cu:461 route_kernel - sqrt(softplus(logits)) top k with bias, then the unbiased weights. vl is a VlBits
+ // (1024 bytes) or null; bias_vl applies to the rows its bits select.
+ bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,
+ const float* bias_vl, const void* vl) {
+ if (T <= 0) return true;
+ if (logits == nullptr || bias == nullptr || idx == nullptr || w == nullptr || n <= 0 || n > 1024 || k <= 0 || k > 8)
+ return false;
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* hb = cpu_addr((void*) bias);
+ void* hi = cpu_addr((void*) idx);
+ void* hw = cpu_addr((void*) w);
+ void* hv = bias_vl != nullptr ? cpu_addr((void*) bias_vl) : nullptr;
+ if (!hl || !hb || !hi || !hw) {
+ std::fprintf(stderr, "dsv41 metal: route has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4);
+ mtl::Buffer bb = wrap(hb, (size_t) n * 4);
+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4);
+ mtl::Buffer bv = hv != nullptr ? wrap(hv, (size_t) n * 4) : g_cx->alloc(4);
+ mtl::Buffer bbv = g_cx->alloc(1024);
+ if (!bl.handle || !bb.handle || !bi.handle || !bw.handle || !bv.handle || !bbv.ptr) return false;
+ const bool has_vl = vl != nullptr;
+ if (has_vl) std::memcpy(bbv.ptr, vl, 1024);
+ mtl::Pipeline pl;
+ try {
+ pl = g_cx->pipeline(g_lib, "ds4_route");
+ } catch (...) {
+ std::fprintf(stderr, "dsv41 metal: route pipeline unavailable\n");
+ return false;
+ }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
+ mtl::Arg::V(route_scale), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::B(bv, hv != nullptr ? off(hv) : 0), mtl::Arg::V(has_vl ? 1u : 0u), mtl::Arg::B(bbv)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 512;
+ dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(bbv);
+ return true;
+ }
+
+
+ namespace {
+ struct Ds4GatherJob {
+ int src[16 * 8];
+ };
+ struct Ds4CombineJob {
+ int n[16];
+ int row[16][8];
+ float w[16][8];
+ int cpu[16];
+ };
+ static_assert(sizeof(Ds4GatherJob) == 512, "Ds4GatherJob layout must match the MSL struct");
+ static_assert(sizeof(Ds4CombineJob) == 1152, "Ds4CombineJob layout must match the MSL struct");
+ } // namespace
+
+ // ds4_kernels.cu:1336 gather_kernel - out[r] = x[g.src[r]] for n rows of cols floats
+ bool gather_rows(const float* x, const void* g, int n, int cols, float* out) {
+ if (n <= 0 || cols <= 0) return true;
+ if (x == nullptr || g == nullptr || out == nullptr || n > 16 * 8) return false;
+ if (!ensure()) return false;
+ void* hx = cpu_addr((void*) x);
+ void* ho = cpu_addr((void*) out);
+ if (!hx || !ho) return false;
+ Ds4GatherJob gj{};
+ std::memcpy(&gj, g, sizeof(gj));
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ int max_src = 0;
+ for (int i = 0; i < n; ++i) max_src = gj.src[i] > max_src ? gj.src[i] : max_src;
+ mtl::Buffer bx = wrap(hx, (size_t) (max_src + 1) * cols * 4); // src rows may be beyond the n output rows
+ mtl::Buffer bo = wrap(ho, (size_t) n * cols * 4);
+ if (!bx.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gather"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V(gj), mtl::Arg::V((unsigned) cols), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) ((cols + 255) / 256);
+ dsp.grid[1] = (uint32_t) n;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+ // ds4_kernels.cu:1342 combine_kernel - out[t] = sum_j w[t][j] ed[row[t][j]] + cpu[t] (when set) + sh[t]
+ bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out) {
+ if (T <= 0 || dim <= 0) return true;
+ if (ed == nullptr || c == nullptr || out == nullptr || T > 16) return false;
+ if (!ensure()) return false;
+ void* he = cpu_addr((void*) ed);
+ void* hc = cpu != nullptr ? cpu_addr((void*) cpu) : nullptr;
+ void* hs = sh != nullptr ? cpu_addr((void*) sh) : nullptr;
+ void* ho = cpu_addr((void*) out);
+ if (!he || !ho) return false;
+ Ds4CombineJob cj{};
+ std::memcpy(&cj, c, sizeof(cj));
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ int max_row = 0;
+ for (int t = 0; t < T; ++t)
+ for (int j = 0; j < cj.n[t]; ++j) max_row = cj.row[t][j] > max_row ? cj.row[t][j] : max_row;
+ mtl::Buffer be = wrap(he, (size_t) (max_row + 1) * dim * 4); // rows may be beyond T * kPicks
+ mtl::Buffer bc = hc != nullptr ? wrap(hc, (size_t) T * dim * 4) : g_cx->alloc(4);
+ mtl::Buffer bs = hs != nullptr ? wrap(hs, (size_t) T * dim * 4) : g_cx->alloc(4);
+ mtl::Buffer bo = wrap(ho, (size_t) T * dim * 4);
+ if (!be.handle || !bc.handle || !bs.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_combine"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(bc, hc != nullptr ? off(hc) : 0),
+ mtl::Arg::B(bs, hs != nullptr ? off(hs) : 0), mtl::Arg::V(cj), mtl::Arg::V((unsigned) dim),
+ mtl::Arg::V(hc != nullptr ? 1u : 0u), mtl::Arg::V(hs != nullptr ? 1u : 0u), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+
+ // On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about
+ // (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.
+ static inline void* cpu_addr(void* p) {
+ void* h = strata::gpu::host_view(p);
+ return h ? h : p;
+ }
+
+ // ds4_kernels.cu:1286 engram_dequant_kernel - the Engram rows (fmt 2 = IQ4_NL, the tables this model's pack carries; the
+ // other formats are left to the CUDA launch, which reports the line)
+ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt) {
+ (void) scales; // (fmt 2 keeps every block's scale in its own 18 bytes)
+ if (n <= 0 || hd <= 0) return true;
+ if (fmt != 2 || rows == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ void* hr = cpu_addr((void*) rows);
+ void* ho = cpu_addr((void*) out);
+ if (!hr || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ // (the kernel strides rows by hd bytes, as the CUDA kernel does, though an IQ4_NL row only fills hd/32*18 of them:
+ // wrapping n * 18/32 * hd left the last rows outside the buffer - garbage scales, measured as inf at row 257 of 288)
+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);
+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);
+ if (!br.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant_rows"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(fmt)};
+ d.grid[0] = (uint32_t) n;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+ // ds4_kernels.cu:1311 engram_gate_kernel - the Engram gate over a (token, copy) row: h += sigmoid(normalised dot) * val
+ bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps) {
+ if (T <= 0 || hc <= 0 || dim <= 0) return true;
+ if (h == nullptr || kv == nullptr || w == nullptr) return false;
+ if (!ensure()) return false;
+ void* hh = cpu_addr((void*) h);
+ void* hk = cpu_addr((void*) kv);
+ void* hw = cpu_addr((void*) w);
+ if (!hh || !hk || !hw) { std::fprintf(stderr, "dsv41 metal: engram_gate has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
+ mtl::Buffer bk = wrap(hk, (size_t) T * (size_t) (hc + 1) * (size_t) dim * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) hc * (size_t) dim * 4);
+ if (!bh.handle || !bk.handle || !bw.handle) {
+ std::fprintf(stderr, "dsv41 metal: engram_gate wrap failed (h %p kv %p w %p; %p %p %p)\n", hh, hk, hw,
+ bh.handle, bk.handle, bw.handle);
+ return false;
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_gate"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bk, off(hk)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::V((unsigned) hc), mtl::Arg::V((unsigned) dim), mtl::Arg::V(eps)};
+ d.grid[0] = (uint32_t) hc;
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+ namespace {
+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Rows (ds4_kernels.cuh:115 Rows, kWin = 16)
+ struct Ds4RowsJob {
+ int n[16];
+ int k[16];
+ int gen, R, kcap, ccap, blk;
+ int64_t p0;
+ };
+ } // namespace
+
+ // ds4_kernels.cu:524 compress_kernel - the window compressor (ratio > 1): each token's kv / score row into its slot, and
+ // every ratio'th token pooled by a per-value softmax over the slots, scaled by the row RMS and the norm weights
+ bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,
+ int d, const float* w, float eps, float* latent) {
+ if (T <= 0 || d <= 0 || ratio <= 0) return true;
+ if (d > 2 * 512) return false; // (the kernel holds d <= 2 * blockDim)
+ if (kv == nullptr || score == nullptr || kv_state == nullptr || score_state == nullptr || w == nullptr ||
+ latent == nullptr)
+ return false;
+ if (!ensure()) return false;
+ void* hkv = cpu_addr((void*) kv);
+ void* hsc = cpu_addr((void*) score);
+ void* hks = cpu_addr((void*) kv_state);
+ void* hss = cpu_addr((void*) score_state);
+ void* hw = cpu_addr((void*) w);
+ void* hla = cpu_addr((void*) latent);
+ if (!hkv || !hsc || !hks || !hss || !hw || !hla) {
+ std::fprintf(stderr, "dsv41 metal: compress_win has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bkv = wrap(hkv, (size_t) T * (size_t) d * 4);
+ mtl::Buffer bsc = wrap(hsc, (size_t) T * (size_t) d * 4);
+ mtl::Buffer bks = wrap(hks, (size_t) ratio * (size_t) d * 4);
+ mtl::Buffer bss = wrap(hss, (size_t) ratio * (size_t) d * 4);
+ mtl::Buffer bw = wrap(hw, (size_t) d * 4);
+ mtl::Buffer bla = wrap(hla, (size_t) T * (size_t) d * 4);
+ if (!bkv.handle || !bsc.handle || !bks.handle || !bss.handle || !bw.handle || !bla.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_compress"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bkv, off(hkv)), mtl::Arg::B(bsc, off(hsc)), mtl::Arg::B(bks, off(hks)),
+ mtl::Arg::B(bss, off(hss)), mtl::Arg::V(ratio), mtl::Arg::V(p0), mtl::Arg::V(T),
+ mtl::Arg::V((unsigned) d), mtl::Arg::B(bw, off(hw)), mtl::Arg::V(eps), mtl::Arg::B(bla, off(hla))};
+ dsp.grid[0] = 1;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 512; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+ // ds4_kernels.cu:582 index_score_kernel<128, kfmt> - the indexer's scores: a position's key row against every head's
+ // query, relu, the heads' weighted sum (rows = ds4_kernels.cuh's Rows, by value there and copied here)
+ bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand,
+ int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt) {
+ if (n <= 0 || T <= 0) return true;
+ if (hd != 128 || heads <= 0 || heads > 32 || (kfmt != 3 && kfmt != 0)) return false;
+ if (q == nullptr || wq == nullptr || kc == nullptr || out == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hq = cpu_addr((void*) q);
+ void* hwq = cpu_addr((void*) wq);
+ void* hkc = cpu_addr((void*) kc);
+ void* hout = cpu_addr((void*) out);
+ if (!hq || !hwq || !hkc || !hout) {
+ std::fprintf(stderr, "dsv41 metal: index_score has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t kb = kfmt == 3 ? (size_t) ((hd / 2 + hd / 32 + 15) / 16 * 16) : (size_t) hd * 2;
+ mtl::Buffer bq = wrap(hq, (size_t) T * (size_t) heads * (size_t) hd * 4);
+ mtl::Buffer bwq = wrap(hwq, (size_t) T * 64 * 4);
+ mtl::Buffer bkc = wrap(hkc, (size_t) n * kb);
+ mtl::Buffer bo = wrap(hout, ((size_t) (T - 1) * (size_t) out_stride + (size_t) n) * 4);
+ if (!bq.handle || !bwq.handle || !bkc.handle || !bo.handle) return false;
+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
+ if (!bc.handle) return false;
+ // Rows is a host struct: it travels as a copy (no gpu address)
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: index_score rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_index_score"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),
+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::B(bo, off(hout)),
+ mtl::Arg::B(br), mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V(cand_stride), mtl::Arg::V(blk),
+ mtl::Arg::V((long) out_stride), mtl::Arg::V(kfmt), mtl::Arg::V(cand != nullptr ? 1 : 0)};
++ // DS4_IDX_MMA: the matrix-unit indexer (ds4_index_score_mma). It stages 16 fp4 keys a threadgroup and runs the
++ // 32 x 128 x 16 head-dot product through simdgroup_matrix - 3.17-6.10x the scalar loop in the standalone harness,
++ // with every output matching to half precision. Opt-in until the gate and KL say the rounding is acceptable.
++ // On by default: measured +7.7 to +12 % on the 32K prompt and +2 % at 3K, with the KL unchanged on both packs
++ // (0.11018 and 0.17087). DS4_NO_IDX_MMA=1 restores the scalar indexer; on the 2.00 pack the score hash moves
++ // (e81282a3 -> 0800179d) because half rounding breaks token ties, while KL and top-1 are identical.
++ static const bool idx_mma = std::getenv("DS4_NO_IDX_MMA") == nullptr;
++ if (idx_mma && cand == nullptr && kfmt == 3) {
++ mtl::Pipeline pm;
++ try { pm = g_cx->pipeline(g_lib, "ds4_index_score_mma"); } catch (...) { pm = mtl::Pipeline{}; }
++ if (pm.handle) {
++ mtl::Dispatch dm;
++ dm.p = &pm;
++ dm.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),
++ mtl::Arg::B(bc, 0), mtl::Arg::B(bo, off(hout)), mtl::Arg::B(br),
++ mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V((int) T), mtl::Arg::V((long) out_stride)};
++ dm.grid[0] = (uint32_t) ((n + 15) / 16);
++ dm.grid[1] = (uint32_t) T;
++ dm.grid[2] = 1;
++ dm.tg[0] = 256; dm.tg[1] = dm.tg[2] = 1;
++ g_cx->run({dm});
++ g_cx->free(br);
++ if (bc.handle && cand != nullptr) g_cx->free(bc);
++ return true;
++ }
++ }
+ dsp.grid[0] = (uint32_t) ((n + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ if (cand == nullptr) g_cx->free(bc);
+ return true;
+ }
+
+ // ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations, a warp a row of the matrix and 2 tokens
+ // a block (the indexer's wproj: ds4_model.cu:7743)
+ bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy) {
+ if (T <= 0 || n <= 0 || k <= 0) return true;
+ if (k % 8 != 0 || W == nullptr || x == nullptr || y == nullptr) return false;
+ if (!ensure()) return false;
+ void* hw = cpu_addr((void*) W);
+ void* hx = cpu_addr((void*) x);
+ void* hy = cpu_addr((void*) y);
+ if (!hw || !hx || !hy) {
+ std::fprintf(stderr, "dsv41 metal: gemv_f16_rows has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bw = wrap(hw, (size_t) n * (size_t) k * 2);
+ mtl::Buffer bx = wrap(hx, ((size_t) (T - 1) * ldx + (size_t) k) * 4);
+ mtl::Buffer by = wrap(hy, ((size_t) (T - 1) * ldy + (size_t) n) * 4);
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f16_rows"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V(n),
+ mtl::Arg::V(k), mtl::Arg::V(T), mtl::Arg::V(alpha), mtl::Arg::B(by, off(hy)),
+ mtl::Arg::V((unsigned long) ldy)};
+ dsp.grid[0] = (uint32_t) ((n + 7) / 8);
+ dsp.grid[1] = (uint32_t) ((T + 1) / 2);
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+ // ds4_kernels.cu:650 topk_kernel - a token's k largest indexer scores (four 8-bit radix passes, then the picks in index
+ // order); Rows by value there and copied here
+ bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out,
+ int out_stride) {
+ if (T <= 0) return true;
+ if (vals == nullptr || out == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hv = cpu_addr((void*) vals);
+ void* ho = cpu_addr((void*) out);
+ if (!hv || !ho) {
+ std::fprintf(stderr, "dsv41 metal: topk has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bv = wrap(hv, ((size_t) (T - 1) * (size_t) stride + 1) * 4);
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + 1) * 4);
+ if (!bv.handle || !bo.handle) return false;
+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;
+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);
+ if (!bc.handle) return false;
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: topk rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_topk"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bv, off(hv)), mtl::Arg::V((long) stride), mtl::Arg::B(br),
+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::V(cand_stride),
+ mtl::Arg::V(blk), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(out_stride),
+ mtl::Arg::V(cand != nullptr ? 1 : 0)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ if (cand == nullptr) g_cx->free(bc);
+ return true;
+ }
+
+ // ds4_kernels.cu:751 block_max_kernel - a block of the indexer's positions and its largest score (the newest pinned at +inf)
+ bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride,
+ int nb_max) {
+ if (T <= 0 || nb_max <= 0 || blk <= 0) return true;
+ if (score == nullptr || bm == nullptr || rows == nullptr) return false;
+ if (!ensure()) return false;
+ void* hs = cpu_addr((void*) score);
+ void* hb = cpu_addr((void*) bm);
+ if (!hs || !hb) {
+ std::fprintf(stderr, "dsv41 metal: block_max has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bs = wrap(hs, ((size_t) (T - 1) * (size_t) stride + (size_t) nb_max * (size_t) blk) * 4);
+ mtl::Buffer bb = wrap(hb, ((size_t) (T - 1) * (size_t) bm_stride + (size_t) nb_max) * 4);
+ if (!bs.handle || !bb.handle) return false;
+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));
+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: block_max rows buffer failed\n"); return false; }
+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_block_max"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((long) stride), mtl::Arg::B(br), mtl::Arg::V(blk),
+ mtl::Arg::B(bb, off(hb)), mtl::Arg::V((long) bm_stride)};
+ dsp.grid[0] = (uint32_t) ((nb_max + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ g_cx->free(br);
+ return true;
+ }
+
+ // ds4_kernels.cu:1141 hc_mean_kernel - a token's mean over its hc streams (grid (ceil(dim / 256), T), 256 threads)
+ bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride) {
+ if (T <= 0 || hc <= 0 || dim <= 0) return true;
+ if (h == nullptr || out == nullptr) return false;
+ if (!ensure()) return false;
+ void* hh = cpu_addr((void*) h);
+ void* ho = cpu_addr((void*) out);
+ if (!hh || !ho) {
+ std::fprintf(stderr, "dsv41 metal: hc_mean has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + (size_t) dim) * 4);
+ if (!bh.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mean"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::V(hc), mtl::Arg::V(dim), mtl::Arg::B(bo, off(ho)),
+ mtl::Arg::V((long) out_stride)};
+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);
+ dsp.grid[1] = (uint32_t) T;
+ dsp.grid[2] = 1;
+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+ // ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i
+ // ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel - the drafter's head. markov_add is a warp a vocabulary
+ // row (the previous token's E row staged in the threadgroup, then E . H[v] added to logits[v]); confidence a threadgroup
+ // a token. Both jobs are the kernels' own structs (metal/src/engine/msl/ds4_init.metal's Ds4MarkovJob / Ds4ConfJob).
+ namespace {
+ // (not an entry point: the generator takes only the ^bool lines) mirrors ds4_init.metal's Ds4MarkovJob / Ds4ConfJob
+ struct MarkovJob {
+ void* logits;
+ const void* E;
+ const void* H;
+ const void* tok;
+ int V, R;
+ };
+ static_assert(sizeof(MarkovJob) == 40, "Ds4MarkovJob's layout");
+
+ struct ConfJob {
+ const void* x;
+ const void* E;
+ const void* tok;
+ const void* w;
+ void* out;
+ int dim, R;
+ };
+ static_assert(sizeof(ConfJob) == 48, "Ds4ConfJob's layout");
+ } // namespace
+
+ bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R) {
+ if (logits == nullptr || E == nullptr || H == nullptr || tok == nullptr) return false;
+ if (V <= 0 || R <= 0 || R > 1024) return false; // (the kernel's staged E row holds 1024 floats)
+ if (!ensure()) return false;
+ MarkovJob j{};
+ j.logits = const_cast<float*>(logits);
+ j.E = E;
+ j.H = H;
+ j.tok = tok;
+ j.V = V;
+ j.R = R;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: markov_add job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_markov_add"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) ((V + 7) / 8);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+
+ bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out) {
+ if (x == nullptr || E == nullptr || tok == nullptr || w == nullptr || out == nullptr) return false;
+ if (T <= 0 || dim <= 0 || R <= 0) return false;
+ if (!ensure()) return false;
+ ConfJob j{};
+ j.x = x;
+ j.E = E;
+ j.tok = tok;
+ j.w = w;
+ j.out = out;
+ j.dim = dim;
+ j.R = R;
+ mtl::Buffer bj = g_cx->alloc(sizeof(j));
+ if (!bj.ptr) {
+ std::fprintf(stderr, "dsv41 metal: confidence job buffer failed\n");
+ return false;
+ }
+ std::memcpy(bj.ptr, &j, sizeof(j));
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_confidence"); } catch (...) { g_cx->free(bj); return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+ }
+
+ bool iota_rows(int* out, int T, int n, int stride) {
+ if (T <= 0 || n <= 0) return true;
+ if (out == nullptr) return false;
+ if (!ensure()) return false;
+ void* ho = cpu_addr((void*) out);
+ if (!ho) {
+ std::fprintf(stderr, "dsv41 metal: iota_rows has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) stride + (size_t) n) * 4);
+ if (!bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_iota_rows"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bo, off(ho)), mtl::Arg::V(n), mtl::Arg::V(stride)};
+ dsp.grid[0] = (uint32_t) T;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 64; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ return true;
+ }
+
+ // vendor/sampler.cu:93 sampler_greedy_kernel - the plain argmax over the vocabulary, one block (1024) a token, the
+ // repetition penalties from the history window's bitmap (the caller takes this path when greedy or temperature <= 0)
+ bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n,
+ float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out) {
+ if (n_tokens <= 0 || n_vocab <= 0) return true;
+ if (logits == nullptr || out == nullptr) return false;
+ if ((n_vocab + 31) / 32 > 4096) return false; // (the kernel's threadgroup bitmap)
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* ho = cpu_addr((void*) out);
+ if (!hl || !ho) {
+ std::fprintf(stderr, "dsv41 metal: sample_greedy has no host view\n");
+ return false;
+ }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) n_tokens * (size_t) n_vocab * 4);
+ mtl::Buffer bo = wrap(ho, (size_t) n_tokens * 4);
+ if (!bl.handle || !bo.handle) return false;
+ const size_t hbytes = (size_t) n_tokens * (size_t) (history_len > 0 ? history_len : 1) * 4;
+ mtl::Buffer bh = history != nullptr ? wrap(cpu_addr((void*) history), hbytes) : g_cx->alloc(4);
+ mtl::Buffer bb = logit_bias != nullptr ? wrap(cpu_addr((void*) logit_bias), (size_t) n_vocab * 4) : g_cx->alloc(4);
+ if (!bh.handle || !bb.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_sampler_greedy"); } catch (...) { return false; }
+ mtl::Dispatch dsp;
+ dsp.p = &pl;
+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::V(n_vocab),
+ mtl::Arg::B(bh, history != nullptr ? off(cpu_addr((void*) history)) : 0),
+ mtl::Arg::V(history_len), mtl::Arg::V(penalty_last_n), mtl::Arg::V(penalty_repeat),
+ mtl::Arg::V(penalty_freq), mtl::Arg::V(penalty_present),
+ mtl::Arg::B(bb, logit_bias != nullptr ? off(cpu_addr((void*) logit_bias)) : 0),
+ mtl::Arg::V(logit_bias != nullptr ? 1 : 0), mtl::Arg::B(bo, off(ho))};
+ dsp.grid[0] = (uint32_t) n_tokens;
+ dsp.grid[1] = dsp.grid[2] = 1;
+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;
+ g_cx->run({dsp});
+ if (history == nullptr) g_cx->free(bh);
+ if (logit_bias == nullptr) g_cx->free(bb);
+ return true;
+ }
+
+ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap,
+ const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes) {
+ if (T <= 0 || sel_stride <= 0 || nsel_count <= 0) return true;
+ // nothing selected yet (a prompt at position 0): the scratch stays untouched, which is what the CUDA launch would do with
+ // no rows to write
+ if (ckv == nullptr || sel == nullptr || out == nullptr) return true;
+ if (!ensure()) return false;
+ void* hc = cpu_addr((void*) ckv);
+ void* hs = cpu_addr((void*) sel);
+ void* ho = cpu_addr((void*) out);
+ if (!hc || !hs || !ho) { std::fprintf(stderr, "dsv41 metal: unpack_sel has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ int maxrow = -1;
+ if (gen != 0 && R > 0) {
+ maxrow = (int) ((p0 + t0 + T) / R) + 1;
+ } else if (gen == 0 && nsel != nullptr) {
+ for (int t = 0; t < T && t < nsel_count; ++t)
+ for (int i = 0; i < nsel[t]; ++i) {
+ const int r = sel[(size_t) t * (size_t) sel_stride + (size_t) i];
+ if (r > maxrow) maxrow = r;
+ }
+ }
+ mtl::Buffer bc = wrap(hc, (size_t) (maxrow + 1 > 0 ? maxrow + 1 : 1) * row_bytes);
+ mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );
+ if (!bc.handle || !bsl.handle || !bo.handle) return false;
+ mtl::Buffer bn = g_cx->alloc((size_t) nsel_count * 4);
+ if (!bn.ptr) return false;
+ std::memcpy(bn.ptr, nsel, (size_t) nsel_count * 4);
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_unpack_sel"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bc, off(hc)), mtl::Arg::B(bsl, off(hs)), mtl::Arg::V((unsigned) sel_stride),
+ mtl::Arg::V((int) gen), mtl::Arg::V((long) p0), mtl::Arg::V((int) t0), mtl::Arg::V((int) R),
+ mtl::Arg::V((int) kcap), mtl::Arg::B(bn), mtl::Arg::V((unsigned) cap), mtl::Arg::B(bo, off(ho)),
+ mtl::Arg::V((unsigned long) row_bytes)};
+ d.grid[0] = (uint32_t) ((sel_stride + 15) / 16);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bn);
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1772 - the expert outputs back into the token rows
+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim) {
+ if (T <= 0 || dim <= 0) return true;
+ if (!ensure()) return false;
+ void* ha = cpu_addr(acc);
+ void* hy = cpu_addr((void*) y);
+ void* hp = cpu_addr((void*) pos);
+ void* hw = cpu_addr((void*) w_);
+ if (!ha || !hy || !hp || !hw) { std::fprintf(stderr, "dsv41 metal: moe_combine has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer ba = wrap(ha, (size_t) T * dim * 4 );
+ mtl::Buffer by = wrap(hy, (size_t) (r1 > r0 ? r1 - r0 : 1) * dim * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
+ if (!ba.handle || !by.handle || !bp.handle || !bw.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_moe_combine"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(by, off(hy)), mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::V((int) k), mtl::Arg::V((int) r0), mtl::Arg::V((int) r1), mtl::Arg::V((unsigned) dim)};
+ d.grid[0] = (uint32_t) ((dim + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:362 - hc_post: out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out) {
+ if (T <= 0 || dim <= 0) return true;
+ if (!ensure()) return false;
+ void* hy = cpu_addr((void*) y);
+ void* hr = cpu_addr((void*) res);
+ void* hp = cpu_addr((void*) post);
+ void* hco = cpu_addr((void*) comb);
+ void* ho = cpu_addr((void*) out);
+ if (!hy || !hr || !hp || !hco || !ho) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );
+ mtl::Buffer br = wrap(hr, (size_t) T * hc * dim * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );
+ mtl::Buffer bc = wrap(hco, (size_t) T * 64 * 4 );
+ mtl::Buffer bo = wrap(ho, (size_t) T * hc * dim * 4 );
+ if (!by.handle || !br.handle || !bp.handle || !bc.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post_v"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp)),
+ mtl::Arg::B(bc, off(hco)), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((unsigned) dim),
+ mtl::Arg::V((unsigned) hc)};
+ d.grid[0] = (uint32_t) ((dim * hc + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:441 - the F32 GEMV (residual / dense projections)
+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy) {
+ if (T <= 0 || n <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ void* hw = cpu_addr((void*) W);
+ void* hx = cpu_addr((void*) x);
+ void* hy = cpu_addr((void*) y);
+ if (!hw || !hx || !hy) { std::fprintf(stderr, "dsv41 metal: gemv_f32 has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bw = wrap(hw, (size_t) n * k * 4 );
+ mtl::Buffer bx = wrap(hx, (size_t) T * k * 4 );
+ mtl::Buffer by = wrap(hy, (size_t) T * (ldy > 0 ? ldy : n) * 4 );
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f32"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned) n),
+ mtl::Arg::V((unsigned) k), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned) ldy)};
+ d.grid[0] = (uint32_t) ((n + 7) / 8);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu:1353 - the router's selection (idx, w, plan_ptr). The prefetch requests stay unpublished for now.
+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram,
+ int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl) {
+ if (T <= 0 || n <= 0) return true;
+ if (!ensure()) return false;
+ void* hl = cpu_addr((void*) logits);
+ void* hb = cpu_addr((void*) bias);
+ void* hv = cpu_addr((void*) vram);
+ void* hi = cpu_addr((void*) idx);
+ void* hw = cpu_addr((void*) w);
+ void* hp = cpu_addr((void*) plan_ptr);
+ if (!hl || !hb || !hv || !hi || !hw || !hp) { std::fprintf(stderr, "dsv41 metal: route_publish has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4 );
+ mtl::Buffer bb = wrap(hb, (size_t) n * 4 );
+ mtl::Buffer bv = wrap(hv, (size_t) n * 8 );
+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4 );
+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );
+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 8 );
+ if (!bl.handle || !bb.handle || !bv.handle || !bi.handle || !bw.handle || !bp.handle) return false;
+ mtl::Buffer bvl = bb;
+ size_t vl_off = 0;
+ if (bias_vl) {
+ void* hvl = cpu_addr((void*) bias_vl);
+ if (!hvl) return false;
+ bvl = wrap(hvl, (size_t) n * 4 );
+ vl_off = off(hvl);
+ }
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_route_publish"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),
+ mtl::Arg::V(route_scale), mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),
+ mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bvl, vl_off), mtl::Arg::V((unsigned) vl),
+ mtl::Arg::V((unsigned) (bias_vl ? 1 : 0))};
+ d.grid[0] = (uint32_t) T;
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 512; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // ds4_kernels.cu swiglu: h = silu(g) * u (clamped when lim > 0)
+ bool swiglu(const float* g, const float* u, int n, float lim, float* h) {
+ if (n <= 0) return true;
+ if (!ensure()) return false;
+ void* hg = cpu_addr((void*) g);
+ void* hu = cpu_addr((void*) u);
+ void* hh = cpu_addr((void*) h);
+ if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + (size_t) n * 4;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);
+ if (!bg.handle || !bu.handle || !bh.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_swiglu"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bg, off(hg)), mtl::Arg::B(bu, off(hu)), mtl::Arg::V((unsigned) n), mtl::Arg::V(lim),
+ mtl::Arg::B(bh, off(hh))};
+ d.grid[0] = (uint32_t) ((n + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+
+ // vendor/moe_win.cu:49 moe_win_group - the expert -> group index for a window. One thread in CUDA; on the Mac every device
+ // pointer is a CPU address, so the same 25-line loop runs on the host, which keeps the first-appearance order the kernel has and
+ // avoids mirroring the 17-field MoeWin struct through a constant buffer.
+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k) {
+ if (T <= 0 || k <= 0) return true;
+ if (T * 8 > 4096) { std::fprintf(stderr, "dsv41 metal: moe_win_group window of %d too wide\n", T); return false; }
+ const unsigned long long* plan = (const unsigned long long*) cpu_addr((void*) plan_ptr);
+ unsigned long long* gptr = (unsigned long long*) cpu_addr(grp_ptr);
+ int* gstart = (int*) cpu_addr(grp_start);
+ int* gent = (int*) cpu_addr(grp_ent);
+ int* ng_out = (int*) cpu_addr(n_groups);
+ if (!plan || !gptr || !gstart || !gent || !ng_out) return false;
+ static unsigned long long gp[4096];
+ static int cnt[4096], fill[4097], gof[4096];
+ int ng = 0;
+ for (int t = 0; t < T; ++t) {
+ for (int i = 0; i < 8; ++i) {
+ const int e = t * 8 + i;
+ gof[e] = -1;
+ const unsigned long long p = i < k ? plan[e] : 0ull;
+ if (p == 0ull) continue;
+ int g = 0;
+ while (g < ng && gp[g] != p) ++g;
+ if (g == ng) {
+ gp[ng] = p;
+ cnt[ng++] = 0;
+ }
+ ++cnt[g];
+ gof[e] = g;
+ }
+ }
+ fill[0] = 0;
+ for (int g = 0; g < ng; ++g) {
+ gptr[g] = gp[g];
+ gstart[g] = fill[g];
+ fill[g + 1] = fill[g] + cnt[g];
+ }
+ gstart[ng] = fill[ng];
+ for (int e = 0; e < T * 8; ++e) {
+ if (gof[e] >= 0) gent[fill[gof[e]]++] = e;
+ }
+ *ng_out = ng;
+ return true;
+ }
+
+
+
+
+ // ---- the prompt chunk's dense GEMM: float rows -> half, a quantized matrix -> half rows, MPP matmul2d (kernels/dense_mm.metal)
+ namespace {
+ mtl::Buffer g_gm_xs, g_gm_yt; // grow-only scratch: packed activations, the contiguous result
+ int g_lib_mm = -2;
+ mtl::Buffer scratch(mtl::Buffer& b, size_t bytes) {
+ if (!b.handle || b.bytes < bytes) {
+ if (b.handle) g_cx->free(b);
+ b = g_cx->alloc(bytes + (bytes >> 2));
+ }
+ return b;
+ }
+ } // namespace
+
+ bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out) {
+ if (nrows <= 0 || ncols <= 0) return true;
+ if (!ensure()) return false;
+ void* hx = strata::gpu::host_view((void*) x);
+ void* ho = strata::gpu::host_view(out);
+ void* hi = idx ? strata::gpu::host_view((void*) idx) : nullptr;
+ if (!hx || !ho) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bx = wrap(hx, ((size_t) (nrows - 1) * (size_t) ldx + (size_t) ncols) * 4);
+ mtl::Buffer bo = wrap(ho, (size_t) nrows * (size_t) ncols * 2);
+ mtl::Buffer bi = hi ? wrap(hi, (size_t) nrows * 4) : bx;
+ if (!bx.handle || !bo.handle || !bi.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_rows_f16"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bi, hi ? off(hi) : off(hx)),
+ mtl::Arg::V((unsigned) (hi ? 1 : 0)), mtl::Arg::V((unsigned) ncols), mtl::Arg::B(bo, off(ho))};
+ d.grid[0] = (uint32_t) ((ncols + 255) / 256);
+ d.grid[1] = (uint32_t) nrows;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+ // kind: 0 BF16 values, 8 Q8_0, 13 Q5_K. n rows of k values, ld the row stride (bytes; values for BF16), out half rows.
+ bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out) {
+ if (n <= 0 || k <= 0) return true;
+ if (!ensure()) return false;
+ const char* fn = kind == 13 ? "ds4_dq_q5k" : kind == 8 ? "ds4_dq_q8" : kind == 0 ? "ds4_dq_bf16" : nullptr;
+ if (!fn) return false;
+ if ((kind == 13 && k % 256 != 0) || (kind == 8 && k % 32 != 0)) return false;
+ void* hw = strata::gpu::host_view((void*) raw);
+ void* ho = strata::gpu::host_view(out);
+ if (!hw || !ho) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t wbytes = (size_t) (n - 1) * ld * (kind == 0 ? 2 : 1) + (kind == 0 ? (size_t) k * 2 : (kind == 13 ? (size_t) (k / 256) * 176 : (size_t) (k / 32) * 34));
+ if (!std::getenv("DS4_NO_TOUCH")) { // (the matrix may be file-backed pages not read yet: the GPU does not page them in, it stalls)
+ volatile unsigned char sink = 0;
+ for (size_t o = 0; o < wbytes; o += 4096) sink = sink + ((const volatile unsigned char*) hw)[o];
+ (void) sink;
+ }
+ mtl::Buffer bw = wrap(hw, wbytes);
+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) k * 2);
+ if (!bw.handle || !bo.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }
+ size_t items = kind == 13 ? (size_t) n * (k / 256) * 8 : kind == 8 ? (size_t) n * (k / 32) : (size_t) n * k;
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::V((unsigned long) ld), mtl::Arg::V((unsigned) k), mtl::Arg::V((unsigned) n),
+ mtl::Arg::B(bo, off(ho))};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((items + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+
+ // Y[r][c] = alpha * (X16[r] . W16[c]) + beta * Y[r][c]: X16 rows x K (stride ldx halves), W16 N x K contiguous, Y rows x N float
+ // (stride ldy). The cuBLAS call's operands in row-major terms; the MPP tile kernel wants contiguous operands and result.
+ bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha,
+ float beta) {
+ if (rows <= 0 || N <= 0) return true;
+ if (!ensure()) return false;
+ if (g_lib_mm == -2) {
+ g_lib_mm = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal"});
+ if (g_lib_mm < 0) std::fprintf(stderr, "dsv41 metal: compiling dense_mm.metal failed\n");
+ }
+ if (g_lib_mm < 0) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ void* hw = strata::gpu::host_view((void*) W16);
+ void* hx = strata::gpu::host_view((void*) X16);
+ void* hy = strata::gpu::host_view(Y);
+ if (!hw || !hx || !hy) return false;
+ mtl::Buffer bw = wrap(hw, (size_t) N * K * 2);
+ mtl::Buffer bx = wrap(hx, ((size_t) (rows - 1) * (size_t) ldx + (size_t) K) * 2);
+ mtl::Buffer by = wrap(hy, ((size_t) (rows - 1) * (size_t) ldy + (size_t) N) * 4);
+ if (!bw.handle || !bx.handle || !by.handle) return false;
+ // activations contiguous
+ mtl::Buffer ax = bx;
+ size_t ax_off = off(hx);
+ if (ldx != K) {
+ ax = scratch(g_gm_xs, (size_t) rows * K * 2);
+ ax_off = 0;
+ mtl::Pipeline pp;
+ try { pp = g_cx->pipeline(g_lib, "ds4_pack_h16"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pp;
+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) K), mtl::Arg::B(ax)};
+ d.grid[0] = (uint32_t) ((K + 255) / 256);
+ d.grid[1] = (uint32_t) rows;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ }
+ const bool direct = ldy == N && alpha == 1.0f && beta == 0.0f;
+ mtl::Buffer ty = by;
+ size_t ty_off = off(hy);
+ if (!direct) {
+ ty = scratch(g_gm_yt, (size_t) rows * N * 4);
+ ty_off = 0;
+ }
+ const int tm = 64, tn = N <= 64 ? 32 : 64;
+ const std::string fn = "gemm_f16_" + std::to_string(tm) + "_" + std::to_string(tn) + "_4";
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib_mm, fn); } catch (...) { return false; }
+ struct GemmArgs { int M, N, K, pad; } ga{rows, N, K, 0};
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(ax, ax_off), mtl::Arg::B(bw, off(hw)), mtl::Arg::B(ty, ty_off), mtl::Arg::V(ga)};
+ d.grid[0] = (uint32_t) ((N + tn - 1) / tn);
+ d.grid[1] = (uint32_t) ((rows + tm - 1) / tm);
+ d.grid[2] = 1;
+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ if (!direct) {
+ mtl::Pipeline pa;
+ try { pa = g_cx->pipeline(g_lib, "ds4_axpby_rows"); } catch (...) { return false; }
+ mtl::Dispatch e;
+ e.p = &pa;
+ e.args = {mtl::Arg::B(ty), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned long) ldy), mtl::Arg::V((unsigned) N),
+ mtl::Arg::V(alpha), mtl::Arg::V(beta)};
+ e.grid[0] = (uint32_t) ((N + 255) / 256);
+ e.grid[1] = (uint32_t) rows;
+ e.grid[2] = 1;
+ e.tg[0] = 256; e.tg[1] = e.tg[2] = 1;
+ g_cx->run({e});
+ }
+ return true;
+ }
+
+
+ // ---- the prompt chunk's expert GEMMs (ds4_mm.metal): a set = n_exp consecutive expert slots, their rows sorted by expert ----
+ namespace {
+ int g_lib_moe = -2;
+ mtl::Buffer g_mm_xs, g_mm_hs, g_mm_ys; // grow-only scratch: the sorted rows' half activations; the gate/up's swiglu rows (the down's input)
+ bool g_mm_hs_ok = false;
-size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : 0; }
++// bytes of a 256-value block per type: the tile kernels are instantiated for exactly these (DS4_MM_INST)
++size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : type == 29 ? 56 : 0; }
+ struct DsMmArgsH {
+ unsigned long gate_off, up_off, down_off, stride;
+ unsigned gu_row, d_row, n_embd, n_ff;
+ float limit;
+ unsigned pad0;
+ };
+ bool mm_prepare() {
+ if (!ensure()) return false;
+ if (g_lib_moe == -2) {
+ g_lib_moe = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal", "../src/engine/msl/ds4_mm.metal"});
+ if (g_lib_moe < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_mm.metal failed\n");
+ }
+ return g_lib_moe >= 0;
+ }
+ } // namespace
+
+ static bool mm_set(bool gate_up, const uint8_t* wbase, size_t stride, const size_t* tr, const int* hb, int n_exp, int total_rows,
+ int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, const uint16_t* hin, uint16_t* hout,
+ int ldh, float* yout, int ldy, int tg, int td) {
+ const int type = gate_up ? tg : td;
+ if (n_exp <= 0 || total_rows <= 0) return true;
+ if (iq_bs(tg) == 0 || iq_bs(td) == 0 || n_embd % 256 != 0 || n_ff % 256 != 0) return false;
+ if (ldh != n_ff || (!gate_up && ldy != n_embd)) return false;
+ if (gate_up ? (n_ff % 128 != 0) : (n_embd % 128 != 0)) return false;
+ if (!mm_prepare()) return false;
- constexpr int TM = 16;
++ constexpr int TM = 32; // (the tile kernels' TM: ds4_mm.metal DS4_MM_INST)
++ constexpr int SG = 4, TN = 32; // the tile kernels' simdgroups and columns each:
++ constexpr int COLS = SG * TN; // ds4_mm.metal DS4_MM_INST must match these
+ // the tile list: (slot, first row, valid rows) a TM-row piece of one expert
+ std::vector<int> tiles;
+ for (int e = 0; e < n_exp; ++e) {
+ const int lo = hb[e] - hb[0], hi = hb[e + 1] - hb[0];
+ for (int r = lo; r < hi; r += TM) { tiles.push_back(e); tiles.push_back(r); tiles.push_back(std::min(TM, hi - r)); tiles.push_back(0); }
+ }
+ const int ntiles = (int) (tiles.size() / 4);
+ if (ntiles == 0) return true;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ const size_t len = ((end - base + page - 1) / page) * page;
+ if (std::getenv("DS4_MM_DBG")) { size_t o2 = 0; std::fprintf(stderr, "mm wrap %p len %zu own=%d rem=%zu\n", (void*) base, len, ds4_gpu_buffer_for_cpu((void*) base, len, &o2) != nullptr, ds4_gpu_alloc_remaining((void*) base)); }
+ return g_cx->wrap((void*) base, len);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ void* hw = strata::gpu::host_view((void*) wbase);
+ if (!hw) return false;
+ mtl::Buffer bw = wrap(hw, (size_t) n_exp * stride);
+ mtl::Buffer bt = g_cx->alloc(tiles.size() * 4);
+ std::memcpy(bt.ptr, tiles.data(), tiles.size() * 4);
+ DsMmArgsH a{tr[0], tr[1], tr[2], (unsigned long) stride, (unsigned) ((size_t) n_embd / 256 * iq_bs(tg)), (unsigned) ((size_t) n_ff / 256 * iq_bs(td)),
+ (unsigned) n_embd, (unsigned) n_ff, limit, std::getenv("DS4_MM_ABL") ? (unsigned) std::atoi(std::getenv("DS4_MM_ABL")) : 0u};
+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "mm %s embd %d ff %d tg %d td %d lim %g stride %zu tr %zu %zu %zu gu_row %u d_row %u\n", gate_up ? "gu" : "dn", n_embd, n_ff, tg, td, limit, stride, tr[0], tr[1], tr[2], a.gu_row, a.d_row);
+ mtl::Pipeline pl;
- try { pl = g_cx->pipeline(g_lib_moe, std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type)); } catch (...) { g_cx->free(bt); return false; }
++ // DS4_MM_CTAB: QUANT_CTAB (function constant 23) - the i-quant grids and the sign table read from constant
++ // memory instead of a threadgroup copy. Same values; the decode's two lookups per 8 weights go through the
++ // constant cache instead of threadgroup memory.
++ static const bool mm_ctab = std::getenv("DS4_MM_CTAB") != nullptr;
++ try {
++ const std::string fn = std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type);
++ if (mm_ctab) {
++ mtl::Constants cs;
++ cs.b(23, true);
++ pl = g_cx->pipeline_uncached(g_lib_moe, fn, cs);
++ } else {
++ pl = g_cx->pipeline(g_lib_moe, fn);
++ }
++ } catch (...) { g_cx->free(bt); return false; }
+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "pipe %s max_threads %d tgmem %d width %d\n", pl.name.c_str(), pl.max_threads, pl.static_tg_mem, pl.simd_width);
+ mtl::Dispatch d;
+ d.p = &pl;
- d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
++ d.tg[0] = SG * 32; d.tg[1] = d.tg[2] = 1;
+ d.grid[2] = 1;
+ d.grid[1] = (uint32_t) ntiles;
+ if (gate_up) {
+ // the sorted rows' activations, half
+ const size_t xs_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 2;
+ if (!g_mm_xs.handle || g_mm_xs.bytes < xs_bytes) {
+ if (g_mm_xs.handle) g_cx->free(g_mm_xs);
+ g_mm_xs = g_cx->alloc(xs_bytes + (xs_bytes >> 2));
+ }
+ void* hx = strata::gpu::host_view((void*) xf);
+ void* hi = strata::gpu::host_view((void*) idx);
+ void* ho = strata::gpu::host_view(hout);
+ if (!hx || !hi || !ho) { g_cx->free(bt); return false; }
+ // (the float rows: the whole token slab the engine's c.x holds; its extent is the allocation's)
+ const size_t xrem = ds4_gpu_alloc_remaining(hx);
+ mtl::Buffer bxf = wrap(hx, xrem > 0 ? xrem : (size_t) ldx * 4);
+ mtl::Buffer bix = wrap(hi, (size_t) total_rows * 4);
+ mtl::Pipeline pg;
+ try { pg = g_cx->pipeline(g_lib_moe, "ds4_mm_gather"); } catch (...) { g_cx->free(bt); return false; }
+ mtl::Dispatch dg;
+ dg.p = &pg;
+ dg.args = {mtl::Arg::B(bxf, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bix, off(hi)), mtl::Arg::V((unsigned) n_embd), mtl::Arg::B(g_mm_xs)};
+ dg.grid[0] = (uint32_t) ((n_embd + 255) / 256);
+ dg.grid[1] = (uint32_t) total_rows;
+ dg.grid[2] = 1;
+ dg.tg[0] = 256; dg.tg[1] = dg.tg[2] = 1;
+ g_cx->run({dg});
+ const size_t hs_bytes = ((size_t) total_rows + TM) * (size_t) n_ff * 2;
+ if (!g_mm_hs.handle || g_mm_hs.bytes < hs_bytes) {
+ if (g_mm_hs.handle) g_cx->free(g_mm_hs);
+ g_mm_hs = g_cx->alloc(std::max<size_t>(hs_bytes + (hs_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ g_mm_hs_ok = true;
+ { // (the down's result rows are made here too)
+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);
+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ }
+ (void) ho;
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_xs), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
- d.grid[0] = (uint32_t) (n_ff / 128);
++ d.grid[0] = (uint32_t) (n_ff / COLS);
+ } else {
+ void* hh = strata::gpu::host_view((void*) hin);
+ void* hy = strata::gpu::host_view(yout);
+ void* hy_real = nullptr;
+ if (!hh || !hy) { g_cx->free(bt); return false; }
+ const size_t hrem = ds4_gpu_alloc_remaining(hh);
+ if (!g_mm_hs_ok) { g_cx->free(bt); return false; }
+ g_mm_hs_ok = false;
+ (void) hrem;
+ mtl::Buffer& bh = g_mm_hs;
+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;
+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {
+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);
+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));
+ }
+ mtl::Buffer by = g_mm_ys;
+ hy_real = hy;
+ hy = by.ptr;
+ if (std::getenv("DS4_MM_YALT")) { hy = hh; by = wrap(hh, (size_t) total_rows * (size_t) n_ff * 2); }
+ if (std::getenv("DS4_MM_EXP1")) { by = g_cx->alloc((size_t) total_rows * (size_t) n_embd * 4 + 4096); hy = by.ptr; }
+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_ys), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};
- d.grid[0] = (uint32_t) (n_embd / 128);
++ d.grid[0] = (uint32_t) (n_embd / COLS);
+ }
+ if (!gate_up && std::getenv("DS4_MM_ASGU")) { // test: the gate/up kernel on the down's buffers
+ static mtl::Pipeline pgu;
+ pgu = g_cx->pipeline(g_lib_moe, "ds4_mm_gu_" + std::to_string(tg));
+ d.p = &pgu;
+ d.args[1] = mtl::Arg::B(g_mm_xs);
+ d.args[2] = mtl::Arg::B(g_mm_ys);
- if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / 128);
++ if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / COLS);
+ }
+ if (!gate_up && std::getenv("DS4_MM_DROP")) {
+ const char* dk = std::getenv("DS4_MM_DROP");
+ static mtl::Buffer dummy;
+ if (!dummy.handle) dummy = g_cx->alloc(1 << 26);
+ static mtl::Buffer dummy2;
+ if (!dummy2.handle) dummy2 = g_cx->alloc(1 << 26);
+ for (int k = 0; k < 4; ++k) if (std::strchr(dk, '0' + k)) d.args[k] = mtl::Arg::B(std::getenv("DS4_MM_D2") && k == std::atoi(std::getenv("DS4_MM_D2")) ? dummy2 : dummy);
+ }
+ if (!gate_up && std::getenv("DS4_MM_ONE")) { d.grid[1] = 1; if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "dn tile0 %d %d %d ntiles %d x %u rows %d\n", tiles[0], tiles[1], tiles[2], ntiles, d.grid[0], total_rows); }
+ if (std::getenv("DS4_MM_DBG")) for (size_t q = 0; q < d.args.size(); ++q) if (d.args[q].buf) std::fprintf(stderr, "arg %s %zu bytes %zu bias %zu off %zu ptr %p\n", gate_up ? "gu" : "dn", q, d.args[q].buf->bytes, (size_t) d.args[q].buf->bias, (size_t) d.args[q].off, d.args[q].buf->ptr);
+ if (std::getenv(gate_up ? "DS4_MM_NOGU" : "DS4_MM_NODN")) { g_cx->free(bt); return true; }
+ g_cx->run({d});
+ if (!std::getenv("DS4_MM_LEAK")) g_cx->free(bt);
+ if (!gate_up) { // the result into the engine's rows
+ void* hyr = strata::gpu::host_view(yout);
+ const size_t n4 = (size_t) total_rows * (size_t) n_embd;
+ const size_t base = (size_t) ((uintptr_t) hyr) & ~(size_t) (page - 1);
+ mtl::Buffer bd = g_cx->wrap((void*) base, ((((size_t) (uintptr_t) hyr + n4 * 4 - base) + page - 1) / page) * page);
+ mtl::Pipeline pc;
+ try { pc = g_cx->pipeline(g_lib, "ds4_copy4"); } catch (...) { return false; }
+ mtl::Dispatch c;
+ c.p = &pc;
+ c.args = {mtl::Arg::B(g_mm_ys), mtl::Arg::B(bd, (size_t) ((uintptr_t) hyr - base)), mtl::Arg::V((unsigned long) n4)};
+ c.tg[0] = 256; c.tg[1] = c.tg[2] = 1;
+ c.grid[0] = (uint32_t) ((n4 + 255) / 256);
+ c.grid[1] = c.grid[2] = 1;
+ g_cx->run({c});
+ }
+ return true;
+ }
+
+ bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
+ float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td) {
+ return mm_set(true, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, xf, ldx, idx, nullptr, h, ldh, nullptr, 0, tg, td);
+ }
+ bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,
+ float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td) {
+ return mm_set(false, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, nullptr, 0, nullptr, h16, nullptr, ldh, y, ldy, tg, td);
+ }
+
+ // ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------
+ // strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched
+ // command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).
++// DS4_COPY_LOG: how much host time the copies take (they wrap host memory into Metal buffers per call)
++namespace { struct CopyLog { unsigned long long n = 0, bytes = 0, us = 0;
++ ~CopyLog() { if (n) std::fprintf(stderr, "copy log: %llu copies, %.1f MB, %.1f ms host time (%.1f us a copy)\n",
++ n, bytes / 1e6, us / 1e3, n ? (double) us / (double) n : 0.0); } };
++ CopyLog g_copy_us; }
++static inline unsigned long long copy_now_us() {
++ return (unsigned long long) std::chrono::duration_cast<std::chrono::microseconds>(
++ std::chrono::steady_clock::now().time_since_epoch()).count();
++}
+ extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) {
+ if (n == 0) return true;
++ const bool copy_log = std::getenv("DS4_COPY_LOG") != nullptr;
++ const unsigned long long copy_t0 = copy_log ? copy_now_us() : 0;
++ struct CopyDone { bool on; unsigned long long t0; size_t n; ~CopyDone() { if (on) { g_copy_us.n++; g_copy_us.bytes += n; g_copy_us.us += copy_now_us() - t0; } } } copy_done{copy_log, copy_t0, n};
+ if (!ensure()) return false;
+ void* hd = strata::gpu::host_view(dst);
+ void* hs = strata::gpu::host_view((void*) src);
+ if (!hd || !hs) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bs = wrap(hs, n);
+ mtl::Buffer bd = wrap(hd, n);
+ if (!bs.handle || !bd.handle) return false;
+ const bool w4 = (((uintptr_t) hd | (uintptr_t) hs | n) & 3u) == 0;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, w4 ? "ds4_copy4" : "ds4_copy1"); } catch (...) { return false; }
+ const size_t items = w4 ? n / 4 : n;
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bd, off(hd)), mtl::Arg::V((unsigned long) items)};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((items + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+ extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) {
+ if (n == 0) return true;
+ if (!ensure()) return false;
+ void* hd = strata::gpu::host_view(dst);
+ if (!hd) return false;
+ const size_t page = 16384;
+ const size_t base = (size_t) ((uintptr_t) hd) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) hd) + n;
+ mtl::Buffer bd = g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ if (!bd.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_fill1"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bd, (size_t) ((uintptr_t) hd - base)), mtl::Arg::V((unsigned long) n), mtl::Arg::V((unsigned) (value & 0xff))};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((n + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+ }
+ extern "C" void* ds4_ctx_mark() {
+ if (!g_cx) return nullptr;
+ return g_cx->mark();
+ }
+ extern "C" void ds4_ctx_wait(void* token) {
+ static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;
+ if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)
+ if (g_cx && token) g_cx->wait_mark(token);
+ }
+ // commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)
+ extern "C" void ds4_metal_kick() {
+ if (g_cx) g_cx->split_runs();
+ }
+
+ extern "C" void ds4_metal_flush() {
+ // The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default
+ // stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves the
+ // other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so
+ // embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,
+ // the readbacks are encoded after this returns.
+ ds4_gpu_sync_all_streams();
+ if (g_cx) g_cx->flush_runs();
+ }
+
+
+ // the one Context every dsv41 kernel dispatches through: two of them means two Metal queues and no ordering between their work
+ mtl::Context& shared_cx() {
+ ensure();
+ return *g_cx;
+ }
+ mtl::Context& finish_cx() {
+ ensure();
+ return *g_cx;
+ }
+
+ int shared_lib() {
+ ensure();
+ return g_lib;
+ }
+
+ } // namespace strata::dsv41::metal_host
diff --cc metal/src/engine/msl/ds4_init.metal
index 000000000,70761a4ae..ed98143a6
mode 000000,100644..100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@@ -1,0 -1,2408 +1,2770 @@@
+ // metal/src/engine/msl/ds4_init.metal - the first kernels the engine asks for, ported from ds4_kernels.cu.
+ //
+ // ds4_kernels.cu:1477 __global__ void init_pre_kernel(float* pre, int T) {
+ // const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ // if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;
+ // }
+ #include <metal_stdlib>
+
+ using namespace metal;
+
+ // the grid is threadgroups of 128; the flat id is what CUDA's blockIdx*blockDim+threadIdx was
+ kernel void ds4_init_pre(device float* pre [[buffer(0)]], constant uint& T [[buffer(1)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (gid < T * 8u) pre[gid] = (gid % 8u) == 0u ? 1.0f : 0.0f;
+ }
+
+ // ds4_kernels.cu:130 embed_kernel: v = bf16(rows[t*dim + e]); h[(t*hc + s)*dim + e] = v for each of hc copies
+ kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [[buffer(1)]],
+ constant uint& dim [[buffer(2)]], constant uint& hc [[buffer(3)]],
+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + uint(lane);
+ if (e >= dim) return;
+ const uint t = tgp.y;
+ const uint bits = uint(rows[(size_t) t * dim + e]) << 16; // BF16 -> F32 is the top half
+ const float v = as_type<float>(bits);
+ for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;
+ }
+
+ // ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, n_mix, T, hcd, ...) - the mHC mixes: mixes[t][m] = h[t] . fn[m]
+ // one thread an output element; both operands F32, alpha = 1, beta = 0 (every call site seen)
+ kernel void ds4_sgemm_tn(device const float* A [[buffer(0)]], device const float* B [[buffer(1)]],
+ device float* C [[buffer(2)]], constant uint& m [[buffer(3)]], constant uint& nt [[buffer(4)]],
+ constant uint& k [[buffer(5)]], constant uint& lda [[buffer(6)]], constant uint& ldb [[buffer(7)]],
+ constant uint& ldc [[buffer(8)]], uint gid [[thread_position_in_grid]]) {
+ const uint total = m * nt;
+ if (gid >= total) return;
+ const uint t = gid / m, mm = gid % m;
+ const device float* a = A + (size_t) mm * lda; // row mm of A
+ const device float* b = B + (size_t) t * ldb; // row t of B
+ float acc = 0.0f;
+ for (uint i = 0; i < k; ++i) acc += a[i] * b[i];
+ C[(size_t) t * ldc + mm] = acc;
+ }
+
+ // ds4_kernels.cu:1894 finish_kernel<false> - RMS norm (f.w), mrope (f.rd), rounding (f.quant), then store to x or dst.
+ // The mrope and rounding branches are not written yet; the host wrapper will decline until they are.
+ struct Ds4Finish {
+ device float* w;
+ float eps;
+ int rd, pos, inverse, vpp, pos_step;
+ device float* freqs;
+ int quant;
+ device ushort* dst;
+ long dst_stride;
+ device uchar* pdst;
+ long pdst_stride;
+ };
+
+ static inline float ds4_warp_sum(float v) {
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ return v;
+ }
+ static inline float ds4_block_sum(float v, threadgroup float* red, ushort lane, ushort w) {
+ v = ds4_warp_sum(v);
+ if (lane == 0) red[w] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (w == 0) {
+ float t = lane < 8 ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t);
+ if (lane == 0) red[0] = t;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return red[0];
+ }
+
+ // ds4_kernels.cu:65 pow2_ceil / :88 fp8_rt / :90 e2m1_rt - the KV quantiser's scale and roundings, shared by finish_pack
+ // (the packed rows) and ds4_finish's rounding branch
+ static inline float ds4_pow2_ceil(float x) {
+ const uint b = as_type<uint>(x);
+ const int e = int((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);
+ return as_type<float>(uint(e + 127) << 23);
+ }
+
+ static inline float ds4_e4m3_round(float v) { // E4M3, nearest even (the input is clamped)
+ const float a = fabs(v);
+ if (!(a == a)) return v;
+ float r;
+ if (a >= 448.0f) r = 448.0f;
+ else {
+ const int e = int((as_type<uint>(a) >> 23) & 0xffu) - 127;
+ const float step = e < -6 ? 0.001953125f : as_type<float>(uint(e - 3 + 127) << 23);
+ r = a == 0.0f ? 0.0f : rint(a / step) * step;
+ r = fmin(r, 448.0f);
+ }
+ return copysign(r, v);
+ }
+
+ static inline float ds4_e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even
+ const float a = fabs(v);
+ float r;
+ if (a <= 0.25f) r = 0.0f;
+ else if (a < 0.75f) r = 0.5f;
+ else if (a <= 1.25f) r = 1.0f;
+ else if (a < 1.75f) r = 1.5f;
+ else if (a <= 2.5f) r = 2.0f;
+ else if (a < 3.5f) r = 3.0f;
+ else if (a <= 5.0f) r = 4.0f;
+ else r = 6.0f;
+ return copysign(r, v);
+ }
+
+ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {
+ threadgroup float red[8];
+ device float* v = x + (long) b * stride;
+ const uint nj = (len + 255u) / 256u;
+ float r[32];
+ float ss = 0.0f;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ r[j] = e < len ? v[e] : 0.0f;
+ ss += r[j] * r[j];
+ }
+ if (f.w != nullptr) {
+ const float tot = ds4_block_sum(ss, red, lane, w);
+ const float inv = rsqrt(tot / float(len) + f.eps);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e < len) r[j] = r[j] * inv * f.w[e];
+ }
+ }
+ if (f.rd > 0) {
+ // mrope: the last rd values move in (re, im) pairs a lane apart (ds4_kernels.cu:161): the partner comes from a
+ // lane xor 1, the angle from the token's position and the frequency of the pair
+ const uint r0 = len - uint(f.rd);
+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ const float other = simd_shuffle_xor(r[j], 1);
+ if (e >= r0 && e < len) {
+ const uint i = (e - r0) >> 1;
+ const float ang = float(pos) * f.freqs[i];
+ const float c = cos(ang);
+ const float sn = f.inverse ? -sin(ang) : sin(ang);
+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
+ }
+ }
+ }
+ if (f.quant != 0) { // the KV rounding without the pack: the group's scale, then the value rounded to the format
+ const uint g = f.quant == 2 ? 16u : 32u;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ float a = e < len ? fabs(r[j]) : 0.0f;
+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
+ if (e >= len) continue;
+ if (f.quant == 1) {
+ const float s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
+ r[j] = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f)) * s;
+ } else if (f.quant == 2) {
+ const float s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
+ } else {
+ const float s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;
+ }
+ }
+ }
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e >= len) continue;
+ if (f.dst != nullptr) {
+ const uint bits = as_type<uint>(r[j]);
+ f.dst[(long) b * f.dst_stride + (long) e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even, the vector's own row
+ } else {
+ v[e] = r[j];
+ }
+ }
+ }
+
+ // ds4_kernels.cu:1677 hc_post_inplace_kernel: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]
+ kernel void ds4_hc_post(device const float* y [[buffer(0)]], device float* h [[buffer(1)]],
+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],
+ constant uint& dim [[buffer(4)]], constant uint& hc [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + uint(lane);
+ if (e >= dim) return;
+ const uint t = tgp.y;
+ device float* r = h + (size_t) t * hc * dim;
+ float rv[8];
+ for (uint j = 0; j < 8u; ++j) if (j < hc) rv[j] = r[(size_t) j * dim + e];
+ const float yv = y[(size_t) t * dim + e];
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ float a = post[(size_t) t * 8u + k] * yv;
+ for (uint j = 0; j < 8u; ++j) if (j < hc) a += comb[(size_t) t * 64u + j * hc + k] * rv[j];
+ r[(size_t) k * dim + e] = a;
+ }
+ }
+
+ // ds4_kernels.cu:1692 hc_mix_scale_kernel: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps), one threadgroup a token
+ kernel void ds4_hc_mix_scale(device const float* h [[buffer(0)]], device float* mixes [[buffer(1)]],
+ constant uint& hcdim [[buffer(2)]], constant uint& n_mix [[buffer(3)]],
+ constant float& eps [[buffer(4)]], uint b [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[8];
+ const device float* x = h + (size_t) b * hcdim;
+ float ss = 0.0f;
+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
+ const float4 hv = *(const device float4*) (x + k);
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ ss = ds4_block_sum(ss, red, lane, w);
+ const float inv = rsqrt(ss / float(hcdim) + eps);
+ if (tid < n_mix) mixes[(size_t) b * 32u + tid] *= inv;
+ }
+
+ // ds4_kernels.cu:1017 hc_mix_kernel: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps), one threadgroup a (m, t)
+ kernel void ds4_hc_mix(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
+ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
+ constant float& eps [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[8];
+ const uint m = tgp.x, t = tgp.y;
+ const device float* x = h + (size_t) t * hcdim;
+ const device float* f = fn + (size_t) m * hcdim;
+ float acc = 0.0f, ss = 0.0f;
+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
+ const float4 hv = *(const device float4*) (x + k);
+ const float4 fv = *(const device float4*) (f + k);
+ acc += hv.x * fv.x + hv.y * fv.y + hv.z * fv.z + hv.w * fv.w;
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ acc = ds4_block_sum(acc, red, lane, w);
+ ss = ds4_block_sum(ss, red, lane, w);
+ if (tid == 0) mixes[(size_t) t * 32u + m] = acc * rsqrt(ss / float(hcdim) + eps);
+ }
+
+ // hc_mix for a prompt chunk: one threadgroup a token for four mixes a pass (the token's row is read once per four mixes instead of once per
+ // mix). Every mix's partial sums keep the single-mix kernel's order, so the result is bit-identical.
+ kernel void ds4_hc_mix_all(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
+ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
+ constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],
+ uint t [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[8];
+ const device float* x = h + (size_t) t * hcdim;
+ float ss = 0.0f;
+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
+ const float4 hv = *(const device float4*) (x + k);
+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
+ }
+ ss = ds4_block_sum(ss, red, lane, w);
+ const float inv = rsqrt(ss / float(hcdim) + eps);
+ for (uint m0 = 0; m0 < n_mix; m0 += 4u) {
+ const uint nm = min(4u, n_mix - m0);
+ float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
+ const device float* f0 = fn + (size_t) m0 * hcdim;
+ const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);
+ const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);
+ const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);
+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
+ const float4 hv = *(const device float4*) (x + k);
+ const float4 v0 = *(const device float4*) (f0 + k);
+ const float4 v1 = *(const device float4*) (f1 + k);
+ const float4 v2 = *(const device float4*) (f2 + k);
+ const float4 v3 = *(const device float4*) (f3 + k);
+ a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;
+ a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;
+ a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;
+ a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;
+ }
+ a0 = ds4_block_sum(a0, red, lane, w);
+ a1 = ds4_block_sum(a1, red, lane, w);
+ a2 = ds4_block_sum(a2, red, lane, w);
+ a3 = ds4_block_sum(a3, red, lane, w);
+ if (tid == 0) {
+ mixes[(size_t) t * 32u + m0] = a0 * inv;
+ if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;
+ if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;
+ if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;
+ }
+ }
+ }
+
++// hc mix + split in one dispatch. The mix is ds4_hc_mix_all's (four mixes a pass, every one keeping the single-mix
++// kernel's summation order, so the values are bit-identical), then the same threadgroup runs the sinkhorn split of the
++// 24 mixes into pre / post / comb. Two dispatches become one, and nothing else changes.
++kernel void ds4_hc_mix_split(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],
++ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],
++ constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],
++ device const float* scale [[buffer(6)]], device const float* base [[buffer(7)]],
++ device float* pre [[buffer(8)]], device float* post [[buffer(9)]],
++ device float* comb [[buffer(10)]], constant uint& hc [[buffer(11)]],
++ constant uint& iters [[buffer(12)]], constant float& hc_eps [[buffer(13)]],
++ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
++ ushort lane [[thread_index_in_simdgroup]], ushort wid [[simdgroup_index_in_threadgroup]]) {
++ threadgroup float red[8];
++ const device float* x = h + (size_t) t * hcdim;
++ float ss = 0.0f;
++ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
++ const float4 hv = *(const device float4*) (x + k);
++ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;
++ }
++ ss = ds4_block_sum(ss, red, lane, wid);
++ const float inv = rsqrt(ss / float(hcdim) + eps);
++ for (uint m0 = 0; m0 < n_mix; m0 += 4u) {
++ const uint nm = min(4u, n_mix - m0);
++ float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
++ const device float* f0 = fn + (size_t) m0 * hcdim;
++ const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);
++ const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);
++ const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);
++ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {
++ const float4 hv = *(const device float4*) (x + k);
++ const float4 v0 = *(const device float4*) (f0 + k);
++ const float4 v1 = *(const device float4*) (f1 + k);
++ const float4 v2 = *(const device float4*) (f2 + k);
++ const float4 v3 = *(const device float4*) (f3 + k);
++ a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;
++ a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;
++ a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;
++ a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;
++ }
++ a0 = ds4_block_sum(a0, red, lane, wid);
++ a1 = ds4_block_sum(a1, red, lane, wid);
++ a2 = ds4_block_sum(a2, red, lane, wid);
++ a3 = ds4_block_sum(a3, red, lane, wid);
++ if (tid == 0) {
++ mixes[(size_t) t * 32u + m0] = a0 * inv;
++ if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;
++ if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;
++ if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ // the split: ds4_hc_split4's math, in the first warp (every lane of it takes the same path)
++ const device float* mx = mixes + (size_t) t * 32u;
++ if (tid < 4u) {
++ pre[(size_t) t * 8u + tid] = 1.0f / (1.0f + exp(-(mx[tid] * scale[0] + base[tid]))) + hc_eps;
++ post[(size_t) t * 8u + tid] = 2.0f / (1.0f + exp(-(mx[4u + tid] * scale[1] + base[4u + tid])));
++ }
++ if (tid < 32u) {
++ const bool on = lane < 16u;
++ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;
++ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));
++ m = fmax(m, simd_shuffle_xor(m, ushort(2)));
++ c = on ? exp(c - m) : 0.0f;
++ float rs = c + simd_shuffle_xor(c, ushort(1));
++ rs += simd_shuffle_xor(rs, ushort(2));
++ c = c / rs + hc_eps;
++ c = on ? c : 0.0f;
++ float cs = c + simd_shuffle_xor(c, ushort(4));
++ cs += simd_shuffle_xor(cs, ushort(8));
++ c = c / (cs + hc_eps);
++ for (uint it = 1; it < iters; ++it) {
++ float r = c + simd_shuffle_xor(c, ushort(1));
++ r += simd_shuffle_xor(r, ushort(2));
++ c = c / (r + hc_eps);
++ float q = c + simd_shuffle_xor(c, ushort(4));
++ q += simd_shuffle_xor(q, ushort(8));
++ c = c / (q + hc_eps);
++ }
++ if (on) comb[(size_t) t * 64u + lane] = c;
++ }
++}
++
+ // ds4_kernels.cu:268 hc_split_kernel - the sinkhorn normalisation of the hyper-connection mix.
+ // This is the SCALAR path of the CUDA kernel (lane 0 of the threadgroup does the token, a local 8x8 matrix), applied for every
+ // hc including 4: it computes the same mathematics as the warp-lane version, only without the shuffles. Correctness first -
+ // the lane-parallel hc == 4 version can be added behind a profile.
+ kernel void ds4_hc_split(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],
+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],
+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],
+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],
+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],
+ uint lane [[thread_index_in_threadgroup]]) {
+ if (lane != 0u) return;
+ const device float* mx = mixes + (size_t) b * 32u;
+ device float* pr = pre + (size_t) b * 8u;
+ device float* po = post + (size_t) b * 8u;
+ device float* cb = comb + (size_t) b * 64u;
+ for (uint i = 0; i < 8u; ++i) {
+ if (i >= hc) break;
+ pr[i] = 1.0f / (1.0f + exp(-(mx[i] * scale[0] + base[i]))) + eps;
+ po[i] = 2.0f / (1.0f + exp(-(mx[hc + i] * scale[1] + base[hc + i])));
+ }
+ float c[8][8];
+ for (uint j = 0; j < 8u; ++j) {
+ if (j >= hc) break;
+ float m = -3.402823466e+38f;
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ c[j][k] = mx[2u * hc + j * hc + k] * scale[2] + base[2u * hc + j * hc + k];
+ m = fmax(m, c[j][k]);
+ }
+ float s = 0.0f;
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ c[j][k] = exp(c[j][k] - m);
+ s += c[j][k];
+ }
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ c[j][k] = c[j][k] / s + eps;
+ }
+ }
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ float s = 0.0f;
+ for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];
+ for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;
+ }
+ for (uint it = 1; it < iters; ++it) {
+ for (uint j = 0; j < 8u; ++j) {
+ if (j >= hc) break;
+ float s = 0.0f;
+ for (uint k = 0; k < 8u; ++k) if (k < hc) s += c[j][k];
+ for (uint k = 0; k < 8u; ++k) if (k < hc) c[j][k] /= s + eps;
+ }
+ for (uint k = 0; k < 8u; ++k) {
+ if (k >= hc) break;
+ float s = 0.0f;
+ for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];
+ for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;
+ }
+ }
+ for (uint j = 0; j < 8u; ++j) {
+ if (j >= hc) break;
+ for (uint k = 0; k < 8u; ++k) { if (k >= hc) break; cb[j * hc + k] = c[j][k]; }
+ }
+ }
+
+ // ds4_kernels.cu:1938 moe_sort - count each virtual slot's picks, prefix-sum the counts, then fill in token order.
+ // kPicks = 8 (ds4_kernels.cuh:19): idx is [T][8], so q -> (t = q/8, i = q%8).
+ kernel void ds4_bounds_scan(device int* bounds [[buffer(0)]], constant uint& n_vs [[buffer(1)]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ if (tid != 0u) return;
+ bounds[0] = 0;
+ for (uint s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];
+ }
+
+ // simd_ballot returns a simd_vote, which does not cast to an integer in this toolchain; the same counts come from the
+ // prefix-sum and sum primitives, which give a lane its offset inside its simdgroup and the group its total.
+ static inline int ds4_warp_prefix(bool hit) { return simd_prefix_exclusive_sum(hit ? 1 : 0); }
+ static inline int ds4_warp_total(bool hit) { return simd_sum(hit ? 1 : 0); }
+
+ kernel void ds4_moe_sort_count(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
+ device int* bounds [[buffer(4)]], uint slot [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup int warp_n[8];
+ threadgroup int base_s;
+ const uint wp = tid / 32u, lane = tid % 32u;
+ if (tid == 0u) base_s = 0;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint n = T * kk;
+ for (uint q = tid; q < n; q += 256u) {
+ const uint t = q / kk, i = q - t * kk;
+ const bool hit = vs[idx[t * 8u + i]] == int(slot);
+ // one vote a warp: the running offset needs each warp's count, accumulated in slot order
+ const int wtot = ds4_warp_total(hit);
+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int total = 0;
+ for (uint w = 0; w < 8u; ++w) total += warp_n[w];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) base_s += total;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) bounds[slot + 1] = base_s;
+ }
+
+ kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
+ device int* bounds [[buffer(4)]], device int* row_tok [[buffer(5)]],
+ device int* pos [[buffer(6)]], uint slot [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup int warp_n[8];
+ threadgroup int base_s;
+ const uint wp = tid / 32u, lane = tid % 32u;
+ if (tid == 0u) base_s = bounds[slot];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint n = T * kk;
+ for (uint q = tid; q < n; q += 256u) {
+ const uint t = q / kk, i = q - t * kk;
+ const bool hit = vs[idx[t * 8u + i]] == int(slot);
+ const int wpre = ds4_warp_prefix(hit), wtot = ds4_warp_total(hit);
+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int before = 0, total = 0;
+ for (uint w = 0; w < 8u; ++w) { if (w < wp) before += warp_n[w]; total += warp_n[w]; }
+ if (hit) {
+ const int r = base_s + before + wpre;
+ row_tok[r] = int(t);
+ pos[t * 8u + i] = r;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) base_s += total;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+
+ // ds4_kernels.cu:1690 engram_dequant_tok_kernel - one hash-table row a threadgroup: IQ4_NL blocks to F32.
+ // fmt 2 (this model's Engram): an 18-byte block is fp16 d then 16 packed nibbles; the low nibble of q[j] is value j and the
+ // high nibble value j + 16, each an index into ggml-common.h's kvalues_iq4nl (ds4_kernels.cu:1280).
+ constant int ds4_kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};
+
+ kernel void ds4_engram_dequant(device const uchar* src [[buffer(0)]], constant ulong& per [[buffer(1)]],
+ constant uint& cols [[buffer(2)]], constant uint& hd [[buffer(3)]],
+ device float* out [[buffer(4)]], constant int& fmt [[buffer(5)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = r / cols, c = r - t * cols;
+ const device uchar* rows = src + (size_t) t * per + (size_t) c * hd;
+ if (fmt == 2) {
+ for (uint e = tid; e < hd; e += 256u) {
+ const device uchar* b = rows + (size_t) (e / 32u) * 18u;
+ const uint q = b[2u + (e & 15u)];
+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
+ out[(size_t) r * hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
+ }
+ return;
+ }
+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: write NaN, so a wrong path shows as NaN in the output rather than
+ // as a silent zero.
+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * hd + e] = NAN;
+ }
+
+ // ds4_kernels.cu:341 hc_pre_norm_kernel - per token: a = sum_s mix[s]*h[token][s][e], one RMS over dim, then x = a*inv
+ // (times w when given). One threadgroup a token, 1024 threads; a thread holds at most 8 values (dim <= 8*1024 = 8192).
+ kernel void ds4_hc_pre_norm(device const float* h [[buffer(0)]], device const float* mix [[buffer(1)]],
+ device const float* w [[buffer(2)]], constant uint& dim [[buffer(3)]],
+ constant uint& hc [[buffer(4)]], constant float& eps [[buffer(5)]],
+ device float* x [[buffer(6)]], constant uint& has_w [[buffer(7)]],
+ uint b [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const device float* hh = h + (size_t) b * hc * dim;
+ const device float* mm = mix + (size_t) b * 8u;
+ device float* xx = x + (size_t) b * dim;
+ float r[8];
+ float ss = 0.0f;
+ uint n = 0;
+ for (uint e = tid; e < dim; e += 1024u, ++n) {
+ float a = 0.0f;
+ for (uint s = 0; s < 8u; ++s) if (s < hc) a += mm[s] * hh[(size_t) s * dim + e];
+ if (n < 8u) r[n] = a;
+ ss += a * a;
+ }
+ // a 1024-thread reduction: 32 simdgroups, each into red[], then the first simdgroup sums those
+ float v = ds4_warp_sum(ss);
+ if (lane == 0u) red[wg] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wg == 0u) {
+ float t = lane < 32u ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t);
+ if (lane == 0u) red[0] = t;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const float inv = rsqrt(red[0] / float(dim) + eps);
+ n = 0;
+ for (uint e = tid; e < dim; e += 1024u, ++n) {
+ if (n >= 8u) break;
+ xx[e] = has_w != 0u ? r[n] * inv * w[e] : r[n];
+ }
+ }
+
+ // ds4_gguf.cu:186 q8_quant_kernel: a warp a 32-group - amax across the warp, sc = amax/127, q = rint(v/sc), d = sc.
+ // grid = (ceil((k/32) / 8 warps), rows), threadgroup = 256 (8 warps of 32).
+ kernel void ds4_q8_quant(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
+ constant uint& k [[buffer(2)]], device char* q [[buffer(3)]],
+ device float* d [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint warp = tid / 32u, lane = tid % 32u;
+ const uint b = tgp.x * 8u + warp; // the 32-group index
+ const uint r = tgp.y;
+ if ((size_t) b * 32u >= (size_t) k) return;
+ const device float* xr = x + (size_t) r * ldx;
+ const float v = xr[(size_t) b * 32u + lane];
+ const float am = simd_max(fabs(v));
+ const float sc = am / 127.0f;
+ q[(size_t) r * k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
+ if (lane == 0u) d[(size_t) r * (k / 32u) + b] = sc;
+ }
+
+ // ds4_gguf.cu:371 mx_mv_kernel - the MXFP4 expert matrix-vector.
+ // grid = (ceil(n / 8 warps), jobs), threadgroup = 256. A warp a row; a lane strides the 32-value blocks; NC columns of
+ // activations share each decoded block (the T=8 window), then a warp sum and y[c][row] = alpha*v + bias[row].
+ // Block layout (kMxBytes = 17 a 32 values): the E8M0 scale byte, then 16 bytes whose low nibble is value j and high nibble
+ // value j+16, each an index into the 16-entry codebook {0,1,2,3,4,6,8,12} and its negatives.
+ struct Ds4MxJob {
+ device const void* w;
+ ulong ld; // bytes a row
+ device const float* x;
+ ulong xcs;
+ device float* y;
+ ulong ycs;
+ device const float* bias;
+ float alpha;
+ int k, n, ncol;
+ device const char* xq;
+ device const float* xd;
+ };
+
+ static inline float ds4_e8m0(uint e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF
+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));
+ }
+
+ static inline void ds4_mx_block(device const uchar* p, threadgroup const float* lut, thread float* w) {
+ const float d = ds4_e8m0(uint(p[0]));
+ for (uint j = 0; j < 16u; ++j) {
+ const uint q = uint(p[1u + j]);
+ w[j] = lut[q & 15u] * d;
+ w[j + 16u] = lut[q >> 4] * d;
+ }
+ }
+
+ kernel void ds4_mx_mv(device const uchar* jobs [[buffer(0)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]]) {
+ threadgroup float lut[16];
+ if (tid < 16u) {
+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};
+ lut[tid] = tid < 8u ? v[tid] : -v[tid - 8u];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
+ thread float acc[8];
+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ const uint nblk = uint(b.k) / 32u;
+ for (uint kb = lane; kb < nblk; kb += 32u) {
+ float wv[32];
+ ds4_mx_block(w + (size_t) kb * 17u, lut, wv);
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device float* x = b.x + c * b.xcs + (size_t) kb * 32u;
+ float a = acc[c];
+ for (uint j = 0; j < 32u; ++j) a = fma(wv[j], x[j], a);
+ acc[c] = a;
+ }
+ }
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ // ds4_gguf.cu:320 q8_mv_kernel - the Q8_0 matvec against quantised activations.
+ // grid = (ceil(n / 8), jobs), threadgroup 256: a warp a row, a lane strides the 34-byte blocks (fp16 d + 32 int8), the dot is
+ // integer (CUDA's __dp4a in four-value groups; here the same sum written out), then acc += (dw * xd[c]) * sum.
+ // The staged-prefetch (ST) and vector variants of the CUDA kernel are prefetch strategies only - this reads the blocks directly.
+ kernel void ds4_q8_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;
+ const uint nb = uint(b.k) / 32u;
+ float acc[8];
+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;
+ for (uint kb = lane; kb < nb; kb += 32u) {
+ const device uchar* p = w + (size_t) kb * 34u;
+ const ushort dh = ushort(p[0]) | (ushort(p[1]) << 8);
+ const float dw = float(as_type<half>(dh));
+ int wv[32];
+ for (uint j = 0; j < 32u; ++j) wv[j] = int((char) p[2u + j]);
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + (size_t) kb * 32u;
+ int sum = 0;
+ for (uint j = 0; j < 32u; ++j) sum += wv[j] * int(xq[j]);
+ acc[c] = fma(dw * b.xd[(size_t) c * nb + kb], float(sum), acc[c]);
+ }
+ }
+ for (uint c = 0; c < 8u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ // ds4_kernels.cu:1092 ring_to_ext_kernel - one ring row of the KV cache to BF16: each value an E4M3 code times its group's
+ // E8M0 power of two (row[e] and row[d + e/32]); rows before the window start write zero.
+ static inline float ds4_e4m3(uint b) { // E4M3 (bias 7, no infinities, max 448): sign, 4-bit exp, 3-bit mantissa
+ const float s = (b & 0x80u) ? -1.0f : 1.0f;
+ const uint ex = (b >> 3) & 0xfu, m = b & 7u;
+ if (ex == 0u) return s * float(m) * (1.0f / 8.0f) * (1.0f / 64.0f);
+ if (ex == 15u && m == 7u) return NAN;
+ return s * exp2(float(ex) - 7.0f) * (1.0f + float(m) * 0.125f);
+ }
+ static inline float ds4_ue8m0(uint b) { return as_type<float>(b << 23); } // 2^(b - 127)
+ static inline ushort ds4_f2bf16(float f) {
+ const uint u = as_type<uint>(f);
+ return ushort((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);
+ }
+
+
+ // ds4_kernels.cu:1104 ext_to_ring_kernel - the window's BF16 rows packed back into the E4M3 ring, each 32-value group at the
+ // smallest power-of-two scale it fits under (k), so v / 2^k is an E4M3 value times a power of two.
+ static inline uint ds4_e4m3_code(float v, int k) {
+ const uint sign = (v < 0.0f) ? 0x80u : 0u;
+ float a = fabs(v) * exp2(-float(k));
+ if (a == 0.0f) return sign;
+ int e = int(floor(log2(a)));
+ int mant = int(round((a / exp2(float(e)) - 1.0f) * 8.0f));
+ int ex = e + 7;
+ if (mant >= 8) { mant = 0; ++ex; }
+ if (ex <= 0) { mant = 0; ex = 0; }
+ if (ex > 15) { mant = 7; ex = 15; }
+ return sign | (uint(ex) << 3) | uint(mant & 7);
+ }
+
+ kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],
+ device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],
+ constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],
+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]], constant uint& has_backup [[buffer(8)]],
+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ if (T - t > n) return; // overwritten by a later row of the window anyway
+ device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;
+ if (has_backup != 0u) { // (a bound buffer is never a null pointer here: the host says whether there is a backup)
+ for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const device ushort* src = ext + (size_t) (n + t) * d;
+ for (uint e0 = 0; e0 < d; e0 += 256u) {
+ const uint e = e0 + tid;
+ const float v = e < d ? as_type<float>(uint(src[e]) << 16) : 0.0f; // (BF16 bits, not half)
+ uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits
+ for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));
+ const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);
+ if (e >= d) continue;
+ row[e] = uchar(ds4_e4m3_code(v, k));
+ if ((e & 31u) == 0u) row[d + e / 32u] = uchar(k + 127);
+ }
+ }
+
+
+ // ds4_kernels.cu:916 unpack32 - 16 threads expand one MXFP4 KV row (288 B at d=512) into 32 BF16 values a thread:
+ // 16 code bytes at part*16, two E4M3 scale bytes at 256 + part*2, and the 4-bit code is itself an FP4 bit pattern
+ // (e2m1_val14: sign and exponent/mantissa bits spliced into a half).
+ static inline float ds4_e2m1_val14(uint c) {
+ const ushort h = (ushort) (((c & 8u) << 12) | ((c & 7u) << 9));
+ return float(as_type<half>(h));
+ }
+
+ // ds4_kernels.cu:937 unpack_sel_kernel - the selected rows of the compressed cache into the attention scratch, one row a
+ // group of 16 threads. n_sel comes either from the generated schedule (gen: min(kcap, (pos+1)/R)) or from nsel[t].
+ kernel void ds4_unpack_sel(device const uchar* ckv [[buffer(0)]], device const int* sel [[buffer(1)]],
+ constant uint& sel_stride [[buffer(2)]], constant int& gen [[buffer(3)]],
+ constant long& p0 [[buffer(4)]], constant int& t0 [[buffer(5)]],
+ constant int& R [[buffer(6)]], constant int& kcap [[buffer(7)]],
+ device const int* nsel [[buffer(8)]], constant uint& cap [[buffer(9)]],
+ device ushort* out [[buffer(10)]], constant ulong& rb [[buffer(11)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = tgp.y;
+ int n_sel;
+ if (gen != 0) {
+ const long pa = p0 + long(t0) + long(t);
+ n_sel = R > 0 ? int(min(long(kcap), (pa + 1) / long(R))) : 0;
+ } else {
+ n_sel = nsel[t];
+ }
+ const uint i = tgp.x * 16u + (tid >> 4u), part = tid & 15u;
+ if (int(i) >= n_sel) return;
+ const size_t r = (size_t) t * cap + i;
+ const device uchar* rr = ckv + (size_t) sel[(size_t) t * sel_stride + i] * rb;
+ const uint4 cw = *(const device uint4*) (rr + part * 16u);
+ const float k0 = ds4_e4m3(uint(rr[256u + part * 2u])) * 16384.0f;
+ const float k1 = ds4_e4m3(uint(rr[256u + part * 2u + 1u])) * 16384.0f;
+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
+ uint o[16];
+ for (uint k = 0; k < 16u; ++k) {
+ const uint b = wv[k >> 2] >> (8u * (k & 3u));
+ const float ks = k < 8u ? k0 : k1;
+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
+ }
+ device uint* dst = (device uint*) (out + r * 512u + part * 32u);
+ for (uint q = 0; q < 4u; ++q) {
+ dst[q * 4u + 0u] = o[4u * q + 0u];
+ dst[q * 4u + 1u] = o[4u * q + 1u];
+ dst[q * 4u + 2u] = o[4u * q + 2u];
+ dst[q * 4u + 3u] = o[4u * q + 3u];
+ }
+ }
+
+
+ // ds4_kernels.cu:1772 moe_combine_kernel - a token's expert outputs back into its row: for each pick whose routed row is in
+ // [r0, r1), acc += w[t][i] * y[r - r0][e]; untouched when no pick falls in range.
+ kernel void ds4_moe_combine(device float* acc [[buffer(0)]], device const float* y [[buffer(1)]],
+ device const int* pos [[buffer(2)]], device const float* w [[buffer(3)]],
+ constant int& kk [[buffer(4)]], constant int& r0 [[buffer(5)]],
+ constant int& r1 [[buffer(6)]], constant uint& dim [[buffer(7)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + tid;
+ if (e >= dim) return;
+ const uint t = tgp.y;
+ float a = acc[(size_t) t * dim + e];
+ bool any = false;
+ for (int i = 0; i < 8; ++i) {
+ if (i >= kk) break;
+ const int r = pos[(size_t) t * 8u + i];
+ if (r < r0 || r >= r1) continue;
+ a = fma(w[(size_t) t * 8u + i], y[(size_t) (r - r0) * dim + e], a);
+ any = true;
+ }
+ if (any) acc[(size_t) t * dim + e] = a;
+ }
+
+
+ // ds4_kernels.cu:362 hc_post_kernel - like hc_post_inplace, but reads a separate residual and writes a separate output:
+ // out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]
+ kernel void ds4_hc_post_v(device const float* y [[buffer(0)]], device const float* res [[buffer(1)]],
+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],
+ device float* out [[buffer(4)]], constant uint& dim [[buffer(5)]],
+ constant uint& hc [[buffer(6)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint i = tgp.x * 256u + tid;
+ if (i >= dim * hc) return;
+ const uint t = tgp.y, k = i / dim, e = i - k * dim;
+ const device float* r = res + (size_t) t * hc * dim;
+ float a = post[(size_t) t * 8u + k] * y[(size_t) t * dim + e];
+ for (uint j = 0; j < 8u; ++j) {
+ if (j >= hc) break;
+ a += comb[(size_t) t * 64u + j * hc + k] * r[(size_t) j * dim + e];
+ }
+ out[(size_t) t * hc * dim + i] = a;
+ }
+
+
+ // ds4_kernels.cu:441 gemv_f32_kernel - a warp a row of F32 weights: float4 loads of W[row] and x[t], four FMAs a step,
+ // lane-strided over k/4, then the warp sum into y[t][row].
+ kernel void ds4_gemv_f32(device const float* W [[buffer(0)]], device const float* x [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
+ device float* y [[buffer(4)]], constant uint& ldy [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]]) {
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (row >= n) return;
+ const uint t = tgp.y;
+ const device float4* w = (const device float4*) (W + (size_t) row * kk);
+ const device float4* xv = (const device float4*) (x + (size_t) t * kk);
+ float acc = 0.0f;
+ for (uint c = lane; c < kk / 4u; c += 32u) {
+ const float4 a = w[c], b = xv[c];
+ acc = fma(a.x, b.x, acc);
+ acc = fma(a.y, b.y, acc);
+ acc = fma(a.z, b.z, acc);
+ acc = fma(a.w, b.w, acc);
+ }
+ acc += simd_shuffle_xor(acc, 16);
+ acc += simd_shuffle_xor(acc, 8);
+ acc += simd_shuffle_xor(acc, 4);
+ acc += simd_shuffle_xor(acc, 2);
+ acc += simd_shuffle_xor(acc, 1);
+ if (lane == 0u) y[(size_t) t * ldy + row] = acc;
+ }
+
+
+ // ds4_kernels.cu:1353 route_publish_kernel - the MoE router: sc = sqrt(softplus(logit)), bs = sc + bias, then k rounds of a
+ // block-wide argmax (a strided max, a warp reduction, a 16-warp pass), the picks' sc summed into the weights, and each pick's
+ // device address published to plan_ptr. The prefetch requests (req) are NOT published here yet: that is the schedule, not the
+ // answer, and it can be turned on separately.
+ kernel void ds4_route_publish(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],
+ constant float& route_scale [[buffer(4)]], device const ulong* vram [[buffer(5)]],
+ device int* idx [[buffer(6)]], device float* w [[buffer(7)]],
+ device ulong* plan_ptr [[buffer(8)]], device const float* bias_vl [[buffer(9)]],
+ constant uint& vl [[buffer(10)]], constant uint& has_vl [[buffer(11)]],
+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup float sc[1024];
+ threadgroup float bs[1024];
+ threadgroup float bv[16];
+ threadgroup int bi[16];
+ threadgroup int pick[8];
+ const device float* lg = logits + (size_t) t * 1024u;
+ const device float* b = (has_vl != 0u && ((vl >> t) & 1u) != 0u) ? bias_vl : bias;
+ for (uint i = tid; i < n; i += 512u) {
+ const float l = lg[i];
+ const float sp = l > 20.0f ? l : log(1.0f + exp(l));
+ sc[i] = sqrt(sp);
+ bs[i] = sc[i] + b[i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint lane = tid & 31u, wp = tid >> 5u;
+ for (uint r = 0; r < 8u; ++r) {
+ if (r >= kk) break;
+ float best = -3.402823466e+38f;
+ int bix = 0x7fffffff;
+ for (uint i = tid; i < n; i += 512u) {
+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) { best = bs[i]; bix = int(i); }
+ }
+ for (uint o = 16u; o > 0u; o >>= 1u) {
+ const float ov = simd_shuffle_xor(best, (ushort) o);
+ const int oi = simd_shuffle_xor(bix, (ushort) o);
+ if (ov > best || (ov == best && oi < bix)) { best = ov; bix = oi; }
+ }
+ if (lane == 0u && wp < 16u) { bv[wp] = best; bi[wp] = bix; }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ float bb = -3.402823466e+38f;
+ int ix = 0x7fffffff;
+ for (uint q = 0; q < 16u; ++q) {
+ if (bv[q] > bb || (bv[q] == bb && bi[q] < ix)) { bb = bv[q]; ix = bi[q]; }
+ }
+ pick[r] = ix;
+ bs[ix] = -3.402823466e+38f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) {
+ float sum = 0.0f;
+ for (uint r = 0; r < 8u; ++r) { if (r >= kk) break; sum += sc[pick[r]]; }
+ for (uint r = 0; r < 8u; ++r) {
+ if (r >= kk) break;
+ const uint p_ = uint(pick[r]);
+ const float wv = sc[p_] / (sum + 1e-20f) * route_scale;
+ idx[(size_t) t * 8u + r] = int(p_);
+ w[(size_t) t * 8u + r] = wv;
+ plan_ptr[(size_t) t * 8u + r] = vram[p_];
+ }
+ }
+ }
+
+
+ // ds4_kernels.cu swiglu_kernel - the expert MLP's activation: h = silu(g) * u, with the clamp when lim > 0
+ kernel void ds4_swiglu(device const float* g [[buffer(0)]], device const float* u [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant float& lim [[buffer(3)]],
+ device float* h [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ if (gid >= n) return;
+ float gv = g[gid], uv = u[gid];
+ if (lim > 0.0f) { // both inputs clamped, then the gate itself
+ gv = fmin(gv, lim);
+ uv = fmin(fmax(uv, -lim), lim);
+ }
+ h[gid] = (gv / (1.0f + exp(-gv))) * uv; // silu(g) * u
+ }
+
+ kernel void ds4_ring_to_ext(device const uchar* ring [[buffer(0)]], device ushort* ext [[buffer(1)]],
+ constant long& p0 [[buffer(2)]], constant uint& n [[buffer(3)]],
+ constant uint& d [[buffer(4)]], constant ulong& rb [[buffer(5)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const long p = p0 - long(n) + long(r);
+ const device uchar* row = ring + (size_t) (p >= 0 ? (p % long(n)) : 0) * rb;
+ for (uint e = tid; e < d; e += 256u) {
+ ext[(size_t) r * d + e] = p >= 0 ? ds4_f2bf16(ds4_e4m3(uint(row[e])) * ds4_ue8m0(uint(row[d + e / 32u]))) : ushort(0);
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_gguf.cu:577 mx_combine_kernel
+ //
+ // A token's expert outputs summed in plan order (an expert off the device adds 0), the CPU lane's part added when its
+ // flag is set, then the shared expert's - ds4_gguf.cu:169's combine(). Grid (n_embd / 128, T), 128 threads.
+ struct Ds4Comb {
+ device const ulong* plan_ptr; // [T][8]: nonzero when the device holds that expert
+ device const float* plan_w; // [T][8]
+ device const float* part; // [entries][n_embd]
+ device const float* cpu_part; // [T][cpu_stride], or null
+ device const int* cpu_flag; // [T], or null
+ device const float* sh_out; // [T][n_embd], or null
+ device float* out;
+ long cpu_stride, out_stride;
+ int k, n_embd;
+ };
+
+ kernel void ds4_mx_combine(constant Ds4Comb& a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.x * 128u + tid;
+ if (int(r) >= a.n_embd) return;
+ const int t = int(tgp.y);
+ float m = 0.0f;
+ for (int i = 0; i < a.k; ++i) { // kPicks <= 8
+ const int e = t * 8 + i;
+ const float pi = a.plan_ptr[e] != 0ul ? a.part[(size_t) e * (size_t) a.n_embd + r] : 0.0f;
+ m = fma(a.plan_w[t * 8 + i], pi, m);
+ }
+ if (a.cpu_flag != nullptr && a.cpu_flag[t] != 0) m += a.cpu_part[(size_t) t * (size_t) a.cpu_stride + r];
+ m += a.sh_out != nullptr ? a.sh_out[(size_t) t * (size_t) a.n_embd + r] : 0.0f;
+ a.out[(size_t) t * (size_t) a.out_stride + r] = m;
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:1286 engram_dequant_kernel / :1311 engram_gate_kernel
+ //
+ // The Engram tables' row decoder and gate (this model's tables are fmt 2: IQ4_NL, an 18-byte block of 32 values whose
+ // nibbles index ds4_kIq4nl above).
+ kernel void ds4_engram_dequant_rows(device const uchar* rows [[buffer(0)]], constant uint& hd [[buffer(1)]],
+ device float* out [[buffer(2)]], constant int& fmt [[buffer(3)]],
+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ if (fmt == 2) {
+ const device uchar* row = rows + (size_t) r * (size_t) hd;
+ for (uint e = tid; e < hd; e += 256u) {
+ const device uchar* b = row + (size_t) (e / 32u) * 18u;
+ const uint q = b[2u + (e & 15u)];
+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);
+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);
+ out[(size_t) r * (size_t) hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);
+ }
+ return;
+ }
+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: NaN rather than a silent zero (as ds4_engram_dequant does)
+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * (size_t) hd + e] = NAN;
+ }
+
+ // ds4_kernels.cu's block_sum for a 1024-thread threadgroup: a warp sum a warp, then every warp sums the 32 partials (the
+ // same result in all of them, so no warp diverges at a barrier)
+ static inline float ds4_block_sum32(float v, threadgroup float* red, ushort lane, ushort w) {
+ v = ds4_warp_sum(v);
+ if (lane == 0) red[w] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float t = lane < 32u ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return t;
+ }
+
+ kernel void ds4_engram_gate(device float* h [[buffer(0)]], device const float* kv [[buffer(1)]],
+ device const float* w [[buffer(2)]], constant uint& hc [[buffer(3)]],
+ constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const uint k = tgp.x, t = tgp.y;
+ device float* hk = h + ((size_t) t * hc + (size_t) k) * (size_t) dim;
+ const device float* kvt = kv + (size_t) t * (size_t) (hc + 1u) * (size_t) dim;
+ const device float* key = kvt + (size_t) k * (size_t) dim;
+ const device float* val = kvt + (size_t) hc * (size_t) dim;
+ const device float* wk = w + (size_t) k * (size_t) dim;
+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;
+ for (uint e = tid; e < dim; e += 1024u) {
+ const float hv = hk[e], kve = key[e];
+ sh += hv * hv;
+ sk += kve * kve;
+ sd += hv * wk[e] * kve;
+ }
+ sh = ds4_block_sum32(sh, red, lane, wg);
+ sk = ds4_block_sum32(sk, red, lane, wg);
+ sd = ds4_block_sum32(sd, red, lane, wg);
+ const float rstd = rsqrt(sh / float(dim) + eps) * rsqrt(sk / float(dim) + eps);
+ const float dot = sd * rstd * rsqrt(float(dim));
+ const float sq = copysign(sqrt(fmax(fabs(dot), 1e-6f)), dot);
+ const float gate = 1.0f / (1.0f + exp(-sq));
+ for (uint e = tid; e < dim; e += 1024u) hk[e] = hk[e] + gate * val[e];
+ }
+
+ // ds4_gguf.cu:72 bf16_mv_kernel<NC, true> - a BF16 matrix's matvec against F32 activations (the engram WKV here, and the
+ // BF16 dense matrices the jobs' f == 0 names). Same frame as ds4_mx_mv above: a warp a row, a lane strides the row; here a
+ // lane 8 BF16 weights (16 bytes = one uint4) and 8 activations, so the row is walked in 256-value steps.
+ // A pair's F32 is bit-exact - the low half zeroed - which is bf16_mv_kernel's __uint_as_float(w << 16).
+ kernel void ds4_bf16_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const device ushort* w = (const device ushort*) b.w + (size_t) row * b.ld;
+ thread float acc[16]; // (the CUDA template's largest NC; ncol cuts the rest)
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
+ for (uint i = uint(lane) * 8u; i < uint(b.k); i += 256u) {
+ const uint4 q = *(const device uint4*) (w + i);
+ const float wf[8] = {as_type<float>(q.x << 16), as_type<float>(q.x & 0xffff0000u),
+ as_type<float>(q.y << 16), as_type<float>(q.y & 0xffff0000u),
+ as_type<float>(q.z << 16), as_type<float>(q.z & 0xffff0000u),
+ as_type<float>(q.w << 16), as_type<float>(q.w & 0xffff0000u)};
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device float4* xp = (const device float4*) (b.x + c * b.xcs + (size_t) i);
+ const float4 x0 = xp[0], x1 = xp[1];
+ float a = acc[c];
+ a = fma(wf[0], x0.x, a);
+ a = fma(wf[1], x0.y, a);
+ a = fma(wf[2], x0.z, a);
+ a = fma(wf[3], x0.w, a);
+ a = fma(wf[4], x1.x, a);
+ a = fma(wf[5], x1.y, a);
+ a = fma(wf[6], x1.z, a);
+ a = fma(wf[7], x1.w, a);
+ acc[c] = a;
+ }
+ }
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:524 compress_kernel / :582 index_score_kernel
+ //
+ // The compressor (ratio > 1 layers): each token's kv / score row into its slot, and every ratio'th token pooled by a
+ // per-value softmax over the slots, scaled by a row RMS and the norm weights. Grid (1), 512 threads (d <= 1024 values).
+ static inline float ds4_block_sum_n(float v, threadgroup float* red, ushort lane, ushort w, uint nw) {
+ v = ds4_warp_sum(v);
+ if (lane == 0) red[w] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float t = lane < nw ? red[lane] : 0.0f;
+ t = ds4_warp_sum(t); // (every warp sums the same partials: no divergence at a barrier)
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return t;
+ }
+
+ kernel void ds4_compress(device const float* kv [[buffer(0)]], device const float* score [[buffer(1)]],
+ device float* kv_state [[buffer(2)]], device float* score_state [[buffer(3)]],
+ constant int& ratio [[buffer(4)]], constant int& p0 [[buffer(5)]], constant int& T [[buffer(6)]],
+ constant uint& d [[buffer(7)]], device const float* wn [[buffer(8)]],
+ constant float& eps [[buffer(9)]], device float* latent [[buffer(10)]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const float kfmax = -3.4028234663852886e+38f; // -FLT_MAX, as the CUDA writes it
+ int g = 0;
+ for (int t = 0; t < T; ++t) {
+ const int slot = (p0 + t) % ratio;
+ const bool pool = (p0 + t + 1) % ratio == 0;
+ for (uint e = tid; e < d; e += 512u) {
+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];
+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (!pool) continue;
+ float val[2] = {0.0f, 0.0f}; // (d <= 2 * 512)
+ float ss = 0.0f;
+ uint n = 0;
+ for (uint e = tid; e < d; e += 512u, ++n) {
+ float m = kfmax;
+ for (int s = 0; s < ratio; ++s) m = fmax(m, score_state[(size_t) s * d + e]);
+ float den = 0.0f;
+ for (int s = 0; s < ratio; ++s) den += exp(score_state[(size_t) s * d + e] - m);
+ float acc = 0.0f;
+ for (int s = 0; s < ratio; ++s)
+ acc += kv_state[(size_t) s * d + e] * (exp(score_state[(size_t) s * d + e] - m) / den);
+ val[n] = acc;
+ ss += acc * acc;
+ }
+ const float inv = rsqrt(ds4_block_sum_n(ss, red, lane, wg, 16u) / float(d) + eps);
+ n = 0;
+ for (uint e = tid; e < d; e += 512u, ++n) latent[(size_t) g * d + e] = val[n] * inv * wn[e];
+ ++g;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:582 index_score_kernel<128, KF>
+ //
+ // A thread a position: its key row (kfmt 3: 64 bytes of E2M1 codes (value 2i in the low nibble) then 4 UE8M0 scales; kfmt
+ // 0: BF16) against every head's query (a threadgroup tile), relu, then the heads' weighted sum. Grid (ceil(n / 256), T).
+ struct Ds4Rows { // ds4_kernels.cuh:115 Rows (kWin = 16)
+ int n[16];
+ int k[16];
+ int gen, R, kcap, ccap, blk;
+ long p0;
+ };
+
+ static inline void ds4_rows_at(constant Ds4Rows& r, int t, thread int& n, thread int& k) {
+ if (r.gen == 0) {
+ n = r.n[t];
+ k = r.k[t];
+ return;
+ }
+ const int nn = int((r.p0 + t + 1) / r.R);
+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);
+ switch (r.gen) {
+ case 1: n = nn; k = min(r.kcap, nn); break;
+ case 2: n = nn; k = cb; break;
+ case 3: n = cb * r.blk; k = min(r.kcap, nn); break;
+ default: n = (nn + r.blk - 1) / r.blk; k = r.kcap; break;
+ }
+ }
+
+ kernel void ds4_index_score(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],
+ device const uchar* kc [[buffer(2)]], device const int* cand [[buffer(3)]],
+ device float* out [[buffer(4)]], constant Ds4Rows& lim [[buffer(5)]],
+ constant int& heads [[buffer(6)]], constant int& n [[buffer(7)]],
+ constant int& cand_stride [[buffer(8)]], constant int& blk [[buffer(9)]],
+ constant long& out_stride [[buffer(10)]], constant int& kfmt [[buffer(11)]],
+ constant int& has_cand [[buffer(12)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ constexpr int HD = 128;
+ threadgroup float qs[32 * HD + 32];
+ const int t = int(tgp.y);
+ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qs[i] = q[(size_t) t * (size_t) heads * HD + i];
+ for (uint i = tid; i < uint(heads); i += 256u) qs[uint(heads) * HD + i] = wq[(size_t) t * 64 + i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int i = int(tgp.x) * 256 + int(tid);
+ if (i >= n) return;
+ device float* o = out + (long) t * out_stride;
+ int ln, lk;
+ ds4_rows_at(lim, t, ln, lk);
+ int p = i;
+ if (has_cand != 0) {
+ if (i >= lk * blk) {
+ o[i] = -INFINITY;
+ return;
+ }
+ p = cand[(size_t) t * (size_t) cand_stride + (size_t) (i / blk)] * blk + i % blk;
+ }
+ if (p >= ln) {
+ o[i] = -INFINITY;
+ return;
+ }
+ const size_t kb = kfmt == 3 ? (size_t) ((HD / 2 + HD / 32 + 15) / 16 * 16) : (size_t) HD * 2;
+ const device uchar* kr = kc + (size_t) p * kb;
+ float acc[32];
+ for (uint h = 0; h < 32u; ++h) acc[h] = 0.0f;
+ for (int c = 0; c < HD; c += 8) {
+ float kvv[8];
+ if (kfmt == 3) { // 8 codes in 4 bytes, the group's power-of-two scale (2^(b - 127))
+ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |
+ (uint(kr[c / 2 + 3]) << 24);
+ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);
+ for (uint x = 0; x < 8u; ++x) kvv[x] = ds4_e2m1_val14(cw >> (4 * x)) * 16384.0f * ks;
+ } else {
+ const device ushort* b = (const device ushort*) (kr + (size_t) c * 2);
+ for (uint x = 0; x < 8u; ++x) kvv[x] = as_type<float>(uint(b[x]) << 16);
+ }
+ for (uint h = 0; h < 32u; ++h) {
+ if (int(h) >= heads) break;
+ threadgroup const float* qh = qs + h * HD + c;
+ for (uint x = 0; x < 8u; ++x) acc[h] += qh[x] * kvv[x];
+ }
+ }
+ float s = 0.0f;
+ for (uint h = 0; h < 32u; ++h) {
+ if (int(h) >= heads) break;
+ s += fmax(acc[h], 0.0f) * qs[uint(heads) * HD + h];
+ }
+ o[i] = s;
+ }
+
++// The sparse-attention indexer with the GPU's matrix units. The scalar ds4_index_score spends 16.3 s of a 32K prompt
++// (30 % of it) doing 32 head-dots a (query, key) pair one FMA at a time; this stages 16 fp4 keys and one query's q as
++// halves and runs the 32 x 128 x 16 product through simdgroup_matrix, which measured 3.17-6.10x faster in the standalone
++// harness (tools/ds4/idx_harness/idx_mma) with every output matching the scalar path to half precision (<= 7e-4
++// relative). Same arithmetic, same order per (query, key) up to that rounding.
++//
++// The fp4 weights are staged as e2m1 * ks (<= 12, half-safe) with the 16384 applied once in the epilogue - the full
++// 16384 * ks reaches 2e5 and overflows half.
++kernel void ds4_index_score_mma(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],
++ device const uchar* kc [[buffer(2)]], device float* out [[buffer(4)]],
++ constant Ds4Rows& lim [[buffer(5)]], constant int& heads [[buffer(6)]],
++ constant int& n [[buffer(7)]], constant int& T [[buffer(8)]],
++ constant long& out_stride [[buffer(9)]],
++ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
++ uint sg [[simdgroup_index_in_threadgroup]]) {
++ constexpr int HD = 128;
++ constexpr int KB = 80; // the fp4 row: 64 code bytes + 4 scale bytes, padded to 80
++ constexpr int NK = 16; // keys a threadgroup scores (two key-blocks of 8)
++ threadgroup half qst[32 * HD]; // [head][dim]
++ threadgroup half kst[HD * NK]; // [dim][key], e2m1 * ks
++ threadgroup float part[4][NK]; // the four head-blocks' partial sums
++ threadgroup float accs[8][64]; // a simdgroup's 8x8 tile
++ const int k0 = int(tgp.x) * NK;
++ const int t = int(tgp.y);
++ if (t >= T) return;
++ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qst[i] = half(q[(size_t) t * (size_t) heads * HD + i]);
++ for (int key = int(tid) / 8; key < NK; key += 32) {
++ const int idx = k0 + key;
++ if (idx >= n) continue;
++ const device uchar* kr = kc + (size_t) idx * (size_t) KB;
++ for (int c = 0; c < HD; c += 8) {
++ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |
++ (uint(kr[c / 2 + 3]) << 24);
++ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);
++ for (uint x = 0; x < 8u; ++x) kst[(c + int(x)) * NK + key] = half(ds4_e2m1_val14(cw >> (4 * x)) * ks);
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ const int hb = int(sg) / 2, kb = int(sg) % 2; // eight heads x eight keys a simdgroup
++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
++ for (int c = 0; c < HD; c += 8) {
++ simdgroup_half8x8 a, b;
++ simdgroup_load(a, qst, HD, ulong2(ulong(c), ulong(hb) * 8ul));
++ simdgroup_load(b, kst, NK, ulong2(ulong(kb) * 8ul, ulong(c)));
++ simdgroup_multiply_accumulate(acc, a, b, acc);
++ }
++ simdgroup_store(acc, accs[sg], 8);
++ simdgroup_barrier(mem_flags::mem_threadgroup);
++ const uint lane = tid % 32u;
++ if (lane < 8u) {
++ const int key = kb * 8 + int(lane);
++ float ssum = 0.0f;
++ for (uint h = 0; h < 8u; ++h) {
++ const int hh = hb * 8 + int(h);
++ if (hh >= heads) break;
++ ssum += fmax(accs[sg][h * 8 + lane] * 16384.0f, 0.0f) * wq[(size_t) t * 64 + hh];
++ }
++ part[hb][key] = ssum;
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (tid < uint(NK)) {
++ const int key = int(tid);
++ const int idx = k0 + key;
++ if (idx >= n) return;
++ int ln, lk;
++ ds4_rows_at(lim, t, ln, lk);
++ device float* o = out + (long) t * out_stride;
++ o[idx] = idx >= ln ? -INFINITY : (part[0][key] + part[1][key] + part[2][key] + part[3][key]);
++ }
++}
++
+ // ---------------------------------------------------------------- ds4_kernels.cu:142 finish_kernel<true> (the KV pack)
+ //
+ // The same load, RMS and mrope as ds4_finish above, then the row written as the cache's packed codes (kv_row_bytes): the
+ // quantiser's scale byte a group (32 values, or 16 for the 4-bit MXFP4-style rows), the codes themselves from bit 0 up -
+ // for the 4-bit formats two values a byte, the second from the next lane. Written from the CUDA of the same name.
+
+
+
+ static inline uint ds4_e2m1_code(float q) { // (bit 3 the sign, -0 kept; the magnitudes as 0 .. 7)
+ const float a = fabs(q);
+ const uint m = a <= 1.5f ? uint(a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;
+ return m | ((as_type<uint>(q) >> 31) << 3);
+ }
+
+ kernel void ds4_finish_pack(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {
+ threadgroup float red[8];
+ device float* v = x + (long) b * stride;
+ const uint nj = (len + 255u) / 256u;
+ float r[32];
+ float ss = 0.0f;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ r[j] = e < len ? v[e] : 0.0f;
+ ss += r[j] * r[j];
+ }
+ if (f.w != nullptr) {
+ const float tot = ds4_block_sum(ss, red, lane, w);
+ const float inv = rsqrt(tot / float(len) + f.eps);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e < len) r[j] = r[j] * inv * f.w[e];
+ }
+ }
+ if (f.rd > 0) {
+ const uint r0 = len - uint(f.rd);
+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ const float other = simd_shuffle_xor(r[j], 1);
+ if (e >= r0 && e < len) {
+ const uint i = (e - r0) >> 1;
+ const float ang = float(pos) * f.freqs[i];
+ const float c = cos(ang);
+ const float sn = f.inverse ? -sin(ang) : sin(ang);
+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;
+ }
+ }
+ }
+ // the packed row: a group's scale and the group's codes as the format's (quant 1: d E4M3 codes at pr[e] and d/32
+ // scale bytes at pr[len + e/32]; 2: E2M1 pairs at pr[e/2] with E4M3 scales at pr[len/2 + e/16]; 3: E2M1 pairs with
+ // UE8M0 scales at pr[len/2 + e/32])
+ const uint g = f.quant == 2 ? 16u : 32u;
+ device uchar* pr = f.pdst + (long) b * f.pdst_stride;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ float a = e < len ? fabs(r[j]) : 0.0f;
+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));
+ float q = 0.0f, s = 1.0f;
+ if (e < len) {
+ if (f.quant == 1) {
+ s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));
+ q = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f));
+ } else if (f.quant == 2) {
+ s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);
+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
+ } else {
+ s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));
+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));
+ }
+ r[j] = q * s;
+ }
+ if (f.quant == 1) {
+ if (e < len) pr[e] = uchar(ds4_e4m3_code(q, 0));
+ } else {
+ const uint c = ds4_e2m1_code(q);
+ const uint hi = simd_shuffle_down(c, 1); // (value e + 1: the next lane's)
+ if (e < len && (e & 1u) == 0u) pr[e >> 1] = uchar(c | (hi << 4));
+ }
+ if (e < len && e % g == 0u)
+ pr[(f.quant == 1 ? len : len / 2u) + e / g] =
+ uchar(f.quant == 2 ? ds4_e4m3_code(s, 0) : (as_type<uint>(s) >> 23));
+ }
+ device ushort* d = f.dst != nullptr ? f.dst + (long) b * f.dst_stride : nullptr;
+ for (uint j = 0; j < 32u; ++j) {
+ if (j >= nj) break;
+ const uint e = tid + j * 256u;
+ if (e >= len) continue;
+ if (d != nullptr) {
+ const uint bits = as_type<uint>(r[j]);
+ d[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even
+ }
+ }
+ }
+
+ // ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations (the indexer's wproj here): a warp a row
+ // of the matrix, a lane 8 halves (16 bytes, the uint4 above) and 8 activations a step, kRowsT = 2 tokens a block. Grid
+ // (ceil(n / 8), ceil(T / 2)), threadgroup 256.
+ kernel void ds4_gemv_f16_rows(device const ushort* W [[buffer(0)]], device const float* x [[buffer(1)]],
+ constant ulong& ldx [[buffer(2)]], constant int& n [[buffer(3)]],
+ constant int& k [[buffer(4)]], constant int& T [[buffer(5)]],
+ constant float& alpha [[buffer(6)]], device float* y [[buffer(7)]],
+ constant ulong& ldy [[buffer(8)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ constexpr int kRowsT = 2;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ const int t0 = int(tgp.y) * kRowsT, nt = min(kRowsT, T - t0);
+ if (row >= n) return;
+ const device ushort* w = W + (size_t) row * (size_t) k;
+ float acc[kRowsT];
+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;
+ for (int c = int(lane) * 8; c < k; c += 32 * 8) {
+ const uint4 q = *(const device uint4*) (w + c);
+ const float2 f0 = float2(as_type<half2>(q.x)), f1 = float2(as_type<half2>(q.y));
+ const float2 f2 = float2(as_type<half2>(q.z)), f3 = float2(as_type<half2>(q.w));
+ const float2 fs[4] = {f0, f1, f2, f3};
+ float xv[kRowsT][8];
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ const device float4* xp = (const device float4*) (x + (size_t) (t0 + t) * ldx + (size_t) c);
+ const float4 a4 = xp[0], b4 = xp[1];
+ xv[t][0] = a4.x;
+ xv[t][1] = a4.y;
+ xv[t][2] = a4.z;
+ xv[t][3] = a4.w;
+ xv[t][4] = b4.x;
+ xv[t][5] = b4.y;
+ xv[t][6] = b4.z;
+ xv[t][7] = b4.w;
+ }
+ for (int i = 0; i < 4; ++i) {
+ const float2 f = fs[i];
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];
+ }
+ }
+ for (int t = 0; t < kRowsT; ++t)
+ if (t < nt) {
+ float a = acc[t];
+ a += simd_shuffle_xor(a, 16);
+ a += simd_shuffle_xor(a, 8);
+ a += simd_shuffle_xor(a, 4);
+ a += simd_shuffle_xor(a, 2);
+ a += simd_shuffle_xor(a, 1);
+ if (lane == 0u) y[(size_t) (t0 + t) * ldy + row] = alpha * a;
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:650 topk_kernel
+ //
+ // A token's k largest scores: four 8-bit radix passes over the order keys of the row (a threadgroup of 1024, shared
+ // histograms), then the picks written in index order by a scan a pass, so the attention sums them in the same order every
+ // run and the ties kept are the lowest positions.
+ static inline uint ds4_ord_key(float f) {
+ const uint b = as_type<uint>(f);
+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);
+ }
+
+ kernel void ds4_topk(device const float* vals [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant Ds4Rows& r [[buffer(2)]], device const int* cand [[buffer(3)]],
+ constant int& cand_stride [[buffer(4)]], constant int& blk [[buffer(5)]],
+ device int* out [[buffer(6)]], constant int& out_stride [[buffer(7)]],
+ constant int& has_cand [[buffer(8)]], uint tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wid [[simdgroup_index_in_threadgroup]]) {
+ threadgroup uint hist[256];
+ threadgroup uint prefix_s, mask_s;
+ threadgroup int remaining_s, cnt_gt, cnt_eq;
+ threadgroup int wgt[32], weq[32];
+ const int t = int(tgp);
+ int n, k;
+ ds4_rows_at(r, t, n, k);
+ device const float* v = vals + (long) t * stride;
+ device int* o = out + (size_t) t * (size_t) out_stride;
+ device const int* cd = has_cand != 0 ? cand + (size_t) t * (size_t) cand_stride : nullptr;
+ if (n <= 0) return;
+ if (k >= n) {
+ for (uint i = tid; i < uint(n); i += 1024u)
+ o[i] = cd != nullptr ? cd[i / uint(blk)] * blk + int(i % uint(blk)) : int(i);
+ return;
+ }
+ if (tid == 0u) {
+ prefix_s = 0u;
+ mask_s = 0u;
+ remaining_s = k;
+ cnt_gt = 0;
+ cnt_eq = 0;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int shift = 24; shift >= 0; shift -= 8) {
+ for (uint i = tid; i < 256u; i += 1024u) hist[i] = 0u;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint prefix = prefix_s, mask = mask_s;
+ for (int i = int(tid); i < n; i += 1024) {
+ const uint u = ds4_ord_key(v[i]);
+ if ((u & mask) == prefix) {
+ threadgroup atomic_uint* hp = (threadgroup atomic_uint*) &hist[(u >> shift) & 0xffu];
+ atomic_fetch_add_explicit(hp, 1u, memory_order_relaxed);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ const int rem = remaining_s;
+ int cum = 0;
+ for (int d = 255; d >= 0; --d) {
+ const int h = int(hist[d]);
+ if (cum + h >= rem) {
+ prefix_s = prefix | (uint(d) << shift);
+ mask_s = mask | (0xffu << shift);
+ remaining_s = rem - cum;
+ break;
+ }
+ cum += h;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const uint thr = prefix_s;
+ const int remaining = remaining_s;
+ const int n_gt = k - remaining;
+ for (int i0 = 0; i0 < n; i0 += 1024) {
+ const int i = i0 + int(tid);
+ const uint u = i < n ? ds4_ord_key(v[i]) : 0u;
+ const bool gt = i < n && u > thr, eq = i < n && u == thr;
+ const auto bgv = simd_ballot(gt);
+ const auto bev = simd_ballot(eq);
+ const uint bg = uint((simd_vote::vote_t)bgv), be = uint((simd_vote::vote_t)bev);
+ if (lane == 0u) {
+ wgt[wid] = int(popcount(bg));
+ weq[wid] = int(popcount(be));
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wid == 0u) {
+ const int g0 = wgt[lane], e0 = weq[lane];
+ int gs = g0, es = e0;
+ for (uint o2 = 1u; o2 < 32u; o2 <<= 1) {
+ const int a = simd_shuffle_up(gs, ushort(o2)), b = simd_shuffle_up(es, ushort(o2));
+ if (uint(lane) >= o2) {
+ gs += a;
+ es += b;
+ }
+ }
+ wgt[lane] = cnt_gt + gs - g0;
+ weq[lane] = cnt_eq + es - e0;
+ simdgroup_barrier(mem_flags::mem_none);
+ if (lane == 31u) {
+ cnt_gt += gs;
+ cnt_eq += es;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int slot = -1;
+ const uint below = (1u << lane) - 1u;
+ if (gt) slot = wgt[wid] + int(popcount(bg & below));
+ else if (eq) {
+ const int e = weq[wid] + int(popcount(be & below));
+ if (e < remaining) slot = n_gt + e;
+ }
+ if (slot >= 0) o[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+
+ // ds4_kernels.cu:751 block_max_kernel - a block of `blk` positions' largest score (the indexer's candidate blocks), the
+ // newest position's block pinned in at +inf. Grid (ceil(nb_max / 256), T), 256 threads.
+ kernel void ds4_block_max(device const float* score [[buffer(0)]], constant long& stride [[buffer(1)]],
+ constant Ds4Rows& r [[buffer(2)]], constant int& blk [[buffer(3)]],
+ device float* bm [[buffer(4)]], constant long& bm_stride [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const int b = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
+ int n, k_;
+ ds4_rows_at(r, t, n, k_);
+ const int nb = (n + blk - 1) / blk;
+ if (b >= nb) return;
+ float m = -INFINITY;
+ for (int j = 0; j < blk; ++j) {
+ const int p = b * blk + j;
+ if (p < n) m = fmax(m, score[(long) t * stride + p]);
+ }
+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in
+ bm[(long) t * bm_stride + b] = m;
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:1141 hc_mean_kernel / :1150 iota_rows_kernel
+ //
+ // The two small shapes the decode loop asks for: a token's mean over its hc streams, and a row's indices.
+ kernel void ds4_hc_mean(device const float* h [[buffer(0)]], constant int& hc [[buffer(1)]],
+ constant int& dim [[buffer(2)]], device float* out [[buffer(3)]],
+ constant long& out_stride [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const int e = int(tgp.x) * 256 + int(tid), t = int(tgp.y);
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * (size_t) hc + (size_t) s) * (size_t) dim + (size_t) e];
+ out[(long) t * out_stride + e] = a / float(hc);
+ }
+
+ kernel void ds4_iota_rows(device int* out [[buffer(0)]], constant int& n [[buffer(1)]], constant int& stride [[buffer(2)]],
+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint t = tg;
+ for (uint i = tid; i < uint(n); i += 64u) out[(size_t) t * (size_t) stride + i] = int(i);
+ }
+
+ // ---------------------------------------------------------------- vendor/sampler.cu:93 sampler_greedy_kernel
+ //
+ // The plain argmax over the vocabulary, ONE BLOCK (1024) a token: a thread's strided candidates with the repetition
+ // penalties applied (a threadgroup bitmap of the history window says which tokens can be penalised at all), a warp
+ // reduction, then warp 0's - a tie keeps the larger value and the SMALLER index, so this is the serial scan's answer.
+ kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constant int& n_vocab [[buffer(1)]],
+ device const int* history [[buffer(2)]], constant int& history_len [[buffer(3)]],
+ constant int& plen [[buffer(4)]], constant float& pen_repeat [[buffer(5)]],
+ constant float& pen_freq [[buffer(6)]], constant float& pen_present [[buffer(7)]],
+ device const float* logit_bias [[buffer(8)]], constant int& has_bias [[buffer(9)]],
+ device int* out [[buffer(10)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort wid [[simdgroup_index_in_threadgroup]]) {
+ threadgroup uint penal_bits[4096]; // (n_vocab + 31) / 32 words (the host checks it fits)
+ threadgroup float sv[32];
+ threadgroup int si[32];
+ const int t = int(tg);
+ device const float* l = logits + (size_t) t * (size_t) n_vocab;
+ device const int* hrow = history != nullptr ? history + (size_t) t * (size_t) history_len : nullptr;
+ int hlen = 0;
+ if (hrow != nullptr) {
+ hlen = min(plen, history_len);
+ if (hlen < 0) hlen = 0;
+ hrow += history_len - hlen; // the window is the tail
+ }
+ const int bits_words = (n_vocab + 31) / 32;
+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;
+ if (use_bits) {
+ for (int w = int(tid); w < bits_words; w += 1024) penal_bits[w] = 0u;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int i = int(tid); i < hlen; i += 1024)
+ if (hrow[i] >= 0 && hrow[i] < n_vocab) {
+ threadgroup atomic_uint* pb = (threadgroup atomic_uint*) &penal_bits[hrow[i] >> 5];
+ atomic_fetch_or_explicit(pb, 1u << (hrow[i] & 31), memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ float bv = -INFINITY;
+ int best = n_vocab; // (the "no candidate" index: it loses every comparison)
+ for (int v = int(tid); v < n_vocab; v += 1024) {
+ float s = has_bias != 0 ? l[v] + logit_bias[v] : l[v];
+ int cnt = 0;
+ if (use_bits && (penal_bits[v >> 5] & (1u << (v & 31))) != 0u)
+ for (int i = 0; i < hlen; ++i)
+ if (hrow[i] == v) ++cnt;
+ if (cnt > 0) {
+ if (s <= 0.0f) s *= pen_repeat;
+ else s /= pen_repeat;
+ s -= float(cnt) * pen_freq + 1.0f * pen_present;
+ }
+ if (s > bv) {
+ bv = s;
+ best = v;
+ }
+ }
+ for (uint o = 16u; o > 0u; o >>= 1) {
+ const float ov = simd_shuffle_down(bv, ushort(o));
+ const int oi = simd_shuffle_down(best, ushort(o));
+ if (ov > bv || (ov == bv && oi < best)) {
+ bv = ov;
+ best = oi;
+ }
+ }
+ if (lane == 0u) {
+ sv[wid] = bv;
+ si[wid] = best;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wid == 0u) {
+ float wv = sv[lane];
+ int wi = si[lane];
+ for (uint o = 16u; o > 0u; o >>= 1) {
+ const float ov = simd_shuffle_down(wv, ushort(o));
+ const int oi = simd_shuffle_down(wi, ushort(o));
+ if (ov > wv || (ov == wv && oi < wi)) {
+ wv = ov;
+ wi = oi;
+ }
+ }
+ if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)
+ }
+ }
+
+ // ---------------------------------------------------------------- sparse attention (the prompt path)
+ //
+ // ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false>. The packed kernel stages sixteen
+ // rows of a tile in threadgroup memory (the window rows as they are, the compressed rows decoded to BF16) once for all
+ // G heads of a threadgroup, then does each head's dots, softmax and weighted sum. The BF16 kernel is the same arithmetic
+ // for row sets already stored as BF16 (a DSpark block); one head and query a threadgroup.
+ struct Ds4AttnRows { // ds4_kernels.cuh:149 AttnRows (kWin = 16)
+ int win0[16], nwin[16], nsel[16];
+ int gen, t0, win, R, kcap;
+ long p0;
+ };
+
+ static inline void ds4_attn_counts(constant Ds4AttnRows& ar, int t, thread int& n_win, thread int& n_sel, thread int& w0) {
+ if (ar.gen != 0) {
+ const long ta = (long) ar.t0 + (long) t;
+ const long pa = ar.p0 + ta;
+ const long a = (long) ar.win - ar.p0, b = ta + 1;
+ const long first = a > b ? a : b; // max(win - p0, ta + 1)
+ w0 = int(first);
+ n_win = int((long) ar.win + ta - first + 1);
+ n_sel = ar.R > 0 ? int(((pa + 1) / (long) ar.R) < (long) ar.kcap ? (pa + 1) / (long) ar.R : (long) ar.kcap) : 0;
+ } else {
+ w0 = ar.win0[t];
+ n_win = ar.nwin[t];
+ n_sel = ar.nsel[t];
+ }
+ if (n_win < 0) n_win = 0;
+ if (n_sel < 0) n_sel = 0;
+ }
+
+ static inline float ds4_bf16_u32(uint v) { return as_type<float>(v << 16); }
+
+ // 32 values of a packed compressed row (kKvFp4, 512 values) as BF16 in four uint4s
+ static inline void ds4_attn_unpack32(const device uchar* rr, int part, threadgroup uint4* dst) {
+ const uint4 cw = *(const device uint4*) (rr + (size_t) part * 16u);
+ const float k0 = ds4_e4m3(uint(rr[256u + uint(part) * 2u])) * 16384.0f;
+ const float k1 = ds4_e4m3(uint(rr[256u + uint(part) * 2u + 1u])) * 16384.0f;
+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};
+ uint o[16];
+ for (uint k = 0; k < 16u; ++k) {
+ const uint b = wv[k >> 2] >> (8u * (k & 3u));
+ const float ks = k < 8u ? k0 : k1;
+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
+ }
+ dst[0] = uint4(o[0], o[1], o[2], o[3]);
+ dst[1] = uint4(o[4], o[5], o[6], o[7]);
+ dst[2] = uint4(o[8], o[9], o[10], o[11]);
+ dst[3] = uint4(o[12], o[13], o[14], o[15]);
+ }
+
+ // rows [s0, s0 + n) of the tile at t0 into tile4; a thread stages one row's 32 values
+ static inline void ds4_attn_stage(device const ushort* wbase, device const uchar* ckv, device const int* sl,
+ int n_win, int t0, int s0, int n, int sr, int part, threadgroup uint4* tile4) {
+ if (sr >= n) return;
+ const int j = s0 + sr;
+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) part * 4u;
+ const int jj = t0 + j;
+ if (jj < n_win) {
+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) part * 32u);
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
+ dst[3] = src[3];
+ return;
+ }
+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;
+ ds4_attn_unpack32(rr, part, dst);
+ }
+
+ static inline float ds4_attn_warp_max(float v) {
+ v = fmax(v, simd_shuffle_xor(v, 16));
+ v = fmax(v, simd_shuffle_xor(v, 8));
+ v = fmax(v, simd_shuffle_xor(v, 4));
+ v = fmax(v, simd_shuffle_xor(v, 2));
+ v = fmax(v, simd_shuffle_xor(v, 1));
+ return v;
+ }
+
+ template <int G>
+ static inline void ds4_sparse_packed_run(device const float* q, device const ushort* win, device const uchar* ckv,
+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
+ device const float* sink, uint heads, float scale, device float* out,
+ threadgroup uint4* tile4, threadgroup float* sc, threadgroup float* m_run_s,
+ threadgroup float* tscale_s, threadgroup float* den_s, uint2 tgp, uint tid,
+ ushort lane_in, ushort wp_in) {
+ constexpr int W = 8 / G; // warps a head
+ const int hb = int(tgp.x) * G, t = int(tgp.y);
+ int n_win, n_sel, w0;
+ ds4_attn_counts(ar, t, n_win, n_sel, w0);
+ const device ushort* wbase = win + (size_t) w0 * 512u;
+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;
+ const uint lane = uint(lane_in), wp = uint(wp_in);
+ const uint gw = wp / uint(W), wr = wp % uint(W);
+ float qr[16];
+ for (int x = 0; x < 16; ++x)
+ qr[x] = q[((size_t) t * heads + uint(hb) + gw) * 512u + lane * 16u + uint(x)];
+ if (tid < uint(G)) {
+ m_run_s[tid] = -1e30f;
+ den_s[tid] = 0.0f;
+ }
+ float o0[G], o1[G];
+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;
+ const int c = int(tid) * 2;
+ const int sr = int(tid) >> 4, part = int(tid) & 15;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += 64) {
+ const int nt = min(64, total - t0);
+ for (int s0 = 0; s0 < nt; s0 += 16) { // the dots
+ const int n = min(16, nt - s0);
+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int r = int(wr); r < n; r += W) {
+ const threadgroup uint4* rp = tile4 + (size_t) r * 64u + lane * 2u;
+ const uint4 a = rp[0], b = rp[1];
+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
+ float s = 0.0f;
+ for (int x = 0; x < 4; ++x) {
+ s += qr[2 * x] * ds4_bf16_u32(av[x] & 0xffffu);
+ s += qr[2 * x + 1] * ds4_bf16_u32(av[x] >> 16);
+ }
+ for (int x = 0; x < 4; ++x) {
+ s += qr[8 + 2 * x] * ds4_bf16_u32(bv[x] & 0xffffu);
+ s += qr[8 + 2 * x + 1] * ds4_bf16_u32(bv[x] >> 16);
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) sc[uint(gw) * 64u + uint(s0 + r)] = s * scale;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (wp < uint(G)) { // head wp's softmax over the tile
+ threadgroup float* scg = sc + wp * 64u;
+ float m = -INFINITY;
+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, scg[j]);
+ m = ds4_attn_warp_max(m);
+ const float m_old = m_run_s[wp];
+ const float m_new = fmax(m_old, m);
+ float s = 0.0f;
+ for (int j = int(lane); j < nt; j += 32) {
+ const float p = exp(scg[j] - m_new);
+ scg[j] = p;
+ s += p;
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) {
+ tscale_s[wp] = exp(m_old - m_new);
+ den_s[wp] = den_s[wp] * tscale_s[wp] + s;
+ m_run_s[wp] = m_new;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int g = 0; g < G; ++g) {
+ const float cs = tscale_s[g];
+ o0[g] *= cs;
+ o1[g] *= cs;
+ }
+ const threadgroup uint* tu = (const threadgroup uint*) tile4;
+ for (int s0 = 0; s0 < nt; s0 += 16) { // the weighted sums
+ const int n = min(16, nt - s0);
+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int r = 0; r < n; ++r) {
+ const uint pr = tu[(size_t) r * 256u + uint(tid)];
+ const float v0 = ds4_bf16_u32(pr & 0xffffu), v1 = ds4_bf16_u32(pr >> 16);
+ for (int g = 0; g < G; ++g) {
+ const float p = sc[uint(g) * 64u + uint(s0 + r)];
+ o0[g] = fma(p, v0, o0[g]);
+ o1[g] = fma(p, v1, o1[g]);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+ for (int g = 0; g < G; ++g) {
+ const float den = den_s[g] + exp(sink[hb + g] - m_run_s[g]);
+ device float* ov = out + ((size_t) t * heads + uint(hb + g)) * 512u;
+ ov[c] = o0[g] / den;
+ ov[c + 1] = o1[g] / den;
+ }
+ }
+
+ #define DS4_SPARSE_PACKED_SIG(NAME) \
+ kernel void NAME(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]], \
+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]], \
+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]], \
+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]], \
+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]], \
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], \
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]])
+
+ #define DS4_SPARSE_PACKED_BODY(G) \
+ { \
+ threadgroup uint4 tile4[16u * 64u]; \
+ threadgroup float sc[8u * 64u]; \
+ threadgroup float m_run_s[8u], tscale_s[8u], den_s[8u]; \
+ ds4_sparse_packed_run<G>(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, tile4, sc, m_run_s, \
+ tscale_s, den_s, tgp, tid, lane, wp); \
+ }
+
+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g8) DS4_SPARSE_PACKED_BODY(8)
+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g4) DS4_SPARSE_PACKED_BODY(4)
+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g2) DS4_SPARSE_PACKED_BODY(2)
+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)
+
+ #undef DS4_SPARSE_PACKED_SIG
+ #undef DS4_SPARSE_PACKED_BODY
+
+ #ifndef DS4_MX_PLO
-#define DS4_MX_PLO 1 // 1: the softmax weights as BF16 high + low (rms error vs the scalar kernel 2e-6 instead of 3e-4)
++#define DS4_MX_PLO 0 // 1: the softmax weights as BF16 high + low (rms error vs the scalar kernel 2e-6 instead of 3e-4)
+ #endif
+ struct Ds4AttnRaw { uint4 v[4]; int kind; }; // a thread's 32 values of one row, as read: kind 0 window (v[0..3] BF16), 1 compressed (v[0] codes, v[1].x the two scale bytes), 2 none
+
+ static inline void ds4_attn_ld(device const ushort* wbase, device const uchar* ckv, device const int* sl, int n_win, int t0, int n,
+ int sr, int pt, thread Ds4AttnRaw& r) {
+ r.kind = 2;
+ if (sr >= n) return;
+ const int jj = t0 + sr;
+ if (jj < n_win) {
+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) pt * 32u);
+ r.v[0] = src[0]; r.v[1] = src[1]; r.v[2] = src[2]; r.v[3] = src[3];
+ r.kind = 0;
+ } else {
+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;
+ r.v[0] = *(const device uint4*) (rr + (size_t) pt * 16u);
+ r.v[1] = uint4(uint(rr[256u + uint(pt) * 2u]), uint(rr[256u + uint(pt) * 2u + 1u]), 0u, 0u);
+ r.kind = 1;
+ }
+ }
+
+ static inline void ds4_attn_st(thread const Ds4AttnRaw& r, int sr, int pt, threadgroup uint4* tile4) {
+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) pt * 4u;
+ if (r.kind == 0) {
+ dst[0] = r.v[0]; dst[1] = r.v[1]; dst[2] = r.v[2]; dst[3] = r.v[3];
+ } else if (r.kind == 1) {
+ const float k0 = ds4_e4m3(r.v[1].x) * 16384.0f;
+ const float k1 = ds4_e4m3(r.v[1].y) * 16384.0f;
+ const uint wv[4] = {r.v[0].x, r.v[0].y, r.v[0].z, r.v[0].w};
+ uint o[16];
+ for (uint k = 0; k < 16u; ++k) {
+ const uint b = wv[k >> 2] >> (8u * (k & 3u));
+ const float ks = k < 8u ? k0 : k1;
+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;
+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);
+ }
+ dst[0] = uint4(o[0], o[1], o[2], o[3]);
+ dst[1] = uint4(o[4], o[5], o[6], o[7]);
+ dst[2] = uint4(o[8], o[9], o[10], o[11]);
+ dst[3] = uint4(o[12], o[13], o[14], o[15]);
+ } else {
+ dst[0] = uint4(0); dst[1] = uint4(0); dst[2] = uint4(0); dst[3] = uint4(0);
+ }
+ }
+
+ // ---------------------------------------------------------------- sparse attention on the matrix units (G = 8 heads a threadgroup)
+ //
+ // The same attention as ds4_sparse_attn_g8 (a token, eight heads, the window rows plus the selected compressed rows, softmax with the head's
+ // sink) but the dots and the weighted sum run on simdgroup 8x8 matrices: eight rows a pass are staged once as BF16 (exact), q and the
+ // softmax weights are split into a BF16 high and low part (so the products keep about 16 bits), accumulation is fp32, and the softmax is the
+ // online one (the running maximum rescales the accumulators through a diagonal matrix).
++#ifndef DS4_MX_QK_LO
++#define DS4_MX_QK_LO 0 // 0: the QK dot uses the BF16 high part of q only (faster, ~8 fewer bits)
++#endif
++// ATTN_PROBE (function constant 25): a timing probe. 1 skips the P.V accumulate, 2 skips the QK matmul - the numbers
++// it produces are wrong on purpose; it exists to say which half of the pass the 20 s of a 32K prompt is in.
++constant int ATTN_PROBE [[function_constant(25)]];
++constant int attn_probe = is_function_constant_defined(ATTN_PROBE) ? ATTN_PROBE : 0;
++
+ kernel void ds4_sparse_attn_mx(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {
+ threadgroup uint4 tile4[8u * 64u];
+ threadgroup float qbuf[4096]; // q hi / lo as BF16 (8 KB each), later the output rows as float
+ threadgroup float part[8u * 64u];
+ threadgroup float ssm[64];
+ threadgroup bfloat phi[64], plo[64];
+ threadgroup float dg[64];
+ threadgroup float m_s[8], den_s[8], sc_s[8];
+ threadgroup bfloat* qhi = (threadgroup bfloat*) qbuf;
+ threadgroup bfloat* qlo = qhi + 4096;
+ threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;
+
+ const int hb = int(tgp.x) * 8, t = int(tgp.y);
+ int n_win, n_sel, w0;
+ ds4_attn_counts(ar, t, n_win, n_sel, w0);
+ const device ushort* wbase = win + (size_t) w0 * 512u;
+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;
+ for (uint x = 0; x < 16u; ++x) {
+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
+ const float v = q[((size_t) t * heads + uint(hb) + h) * 512u + d];
+ const bfloat hi = bfloat(v);
+ qhi[idx] = hi;
- qlo[idx] = bfloat(v - float(hi));
++ if (DS4_MX_QK_LO) qlo[idx] = bfloat(v - float(hi)); // (unused when the QK dot takes the high part only)
+ }
+ if (tid < 8u) {
+ m_s[tid] = -1e30f;
+ den_s[tid] = 0.0f;
+ }
+ simdgroup_float8x8 O[8];
+ for (int c = 0; c < 8; ++c) O[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ const int sr = int(tid) >> 4, pt = int(tid) & 15;
+ const int total = n_win + n_sel;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ Ds4AttnRaw raw;
+ raw.kind = 2;
+ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, 0, min(8, total), sr, pt, raw);
+ for (int t0 = 0; t0 < total; t0 += 8) {
+ const int n = min(8, total - t0);
- if (sr < 8) ds4_attn_st(raw, sr, pt, tile4);
++ if (sr < 8 && attn_probe != 5) ds4_attn_st(raw, sr, pt, tile4);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, t0 + 8, min(8, total - t0 - 8), sr, pt, raw); // the next pass's rows, in flight during this one
- { // S = q . K^T over this simdgroup's eighth of the 512 values
++ if (attn_probe != 2 && attn_probe != 6) { // S = q . K^T over this simdgroup's eighth of the 512 values
+ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
+ simdgroup_bfloat8x8 a_hi, a_lo, b;
+ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
+ simdgroup_load(a_hi, qhi, 512, ulong2(ulong(kb) * 8ul, 0ul));
+ simdgroup_load(a_lo, qlo, 512, ulong2(ulong(kb) * 8ul, 0ul));
+ simdgroup_multiply_accumulate(acc, a_hi, b, acc);
- simdgroup_multiply_accumulate(acc, a_lo, b, acc);
++ if (DS4_MX_QK_LO) simdgroup_multiply_accumulate(acc, a_lo, b, acc); // (the low part of q: exactness vs speed)
+ }
+ simdgroup_store(acc, part + uint(sg) * 64u, 8);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- if (tid < 64u) {
++ if (tid < 64u && attn_probe != 4 && attn_probe != 6) {
+ float s = 0.0f;
+ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
+ ssm[tid] = s * scale;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- { // head sg's online softmax over the pass's rows
++ if (attn_probe != 3) { // head sg's online softmax over the pass's rows
+ const bool valid = int(lane) < n;
+ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
+ const float m = ds4_attn_warp_max(s);
+ const float m_old = m_s[sg];
+ const float m_new = fmax(m_old, m);
+ const float p = valid ? exp(s - m_new) : 0.0f;
+ const float ps = ds4_warp_sum(p);
+ if (lane < 8u) {
+ const bfloat hi = bfloat(p);
+ phi[uint(sg) * 8u + uint(lane)] = hi;
+ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
+ }
+ if (lane == 0u) {
+ const float cs = exp(m_old - m_new);
+ sc_s[sg] = cs;
+ den_s[sg] = den_s[sg] * cs + ps;
+ m_s[sg] = m_new;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[tid >> 3] : 0.0f;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- { // O = diag(rescale) O + P V on this simdgroup's 64 columns
++ if (attn_probe != 1 && attn_probe != 6) { // O = diag(rescale) O + P V on this simdgroup's 64 columns
+ simdgroup_float8x8 D;
+ simdgroup_bfloat8x8 ph, pl;
+ simdgroup_load(D, dg, 8);
+ simdgroup_load(ph, phi, 8);
+ simdgroup_load(pl, plo, 8);
+ for (int c = 0; c < 8; ++c) {
+ simdgroup_float8x8 tmp;
+ simdgroup_multiply(tmp, D, O[c]);
+ simdgroup_bfloat8x8 v;
+ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
+ simdgroup_multiply_accumulate(tmp, ph, v, tmp);
+ #if DS4_MX_PLO
+ simdgroup_multiply_accumulate(tmp, pl, v, tmp);
+ #endif
+ O[c] = tmp;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ for (int c = 0; c < 8; ++c) simdgroup_store(O[c], qbuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint x = 0; x < 16u; ++x) {
+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
+ const float den = den_s[h] + exp(sink[uint(hb) + h] - m_s[h]);
+ out[((size_t) t * heads + uint(hb) + h) * 512u + d] = qbuf[idx] / den;
+ }
+ }
+
++// TWO QUERIES A THREADGROUP, written out in full. The staged keys are shared; each query keeps its own accumulators,
++// its own online-softmax state and its own key count (the causal shape makes the second query's set a superset, so the
++// union staged is the second query's own set and the first masks its tail).
++kernel void ds4_sparse_attn_mx2(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
++ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
++ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
++ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
++ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
++ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
++ ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {
++ threadgroup uint4 tile4[8u * 64u];
++ threadgroup float qbuf[4096]; // two queries' q as BF16 (8 KB each), later a query's output rows
++ threadgroup float part[8u * 64u];
++ threadgroup float ssm[64];
++ threadgroup bfloat phi[64], plo[64];
++ threadgroup float dg[64];
++ threadgroup float m_s[16], den_s[16], sc_s[16];
++ threadgroup bfloat* qhiA = (threadgroup bfloat*) qbuf;
++ threadgroup bfloat* qhiB = qhiA + 4096; // a query's q is 4096 bf16
++ threadgroup float* obuf = qbuf; // 8 heads x 512 floats: qbuf again, once the key loop is done
++ threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;
++
++ const int hb = int(tgp.x) * 8;
++ const int tA = int(tgp.y) * 2;
++ const int rows_total = (ar.gen == 0 && ar.t0 > 0) ? ar.t0 : 0x7fffffff;
++ const int nq = (tA + 1 < rows_total) ? 2 : 1;
++ int nwA, nsA, w0A, nwB, nsB, w0B;
++ ds4_attn_counts(ar, tA, nwA, nsA, w0A);
++ nwB = nwA; nsB = nsA; w0B = w0A;
++ if (nq > 1) ds4_attn_counts(ar, tA + 1, nwB, nsB, w0B);
++ const int totalA = nwA + nsA;
++ const int totalB = (nq > 1) ? nwB + nsB : totalA;
++ const int total = totalA > totalB ? totalA : totalB;
++ const device ushort* wbase = win + (size_t) w0A * 512u;
++ const device int* sl = sel + (size_t) tA * (size_t) sel_stride;
++ for (uint x = 0; x < 16u; ++x) {
++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
++ qhiA[idx] = bfloat(q[((size_t) tA * heads + uint(hb) + h) * 512u + d]);
++ if (nq > 1) qhiB[idx] = bfloat(q[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d]);
++ }
++ if (tid < 16u) {
++ m_s[tid] = -1e30f;
++ den_s[tid] = 0.0f;
++ }
++ simdgroup_float8x8 OA[8], OB[8];
++ for (int c = 0; c < 8; ++c) {
++ OA[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
++ OB[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
++ }
++ const int sr = int(tid) >> 4, pt = int(tid) & 15;
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ Ds4AttnRaw raw;
++ raw.kind = 2;
++ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, 0, min(8, total), sr, pt, raw);
++ for (int t0 = 0; t0 < total; t0 += 8) {
++ const int np = min(8, total - t0);
++ (void) np;
++ if (sr < 8) ds4_attn_st(raw, sr, pt, tile4);
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, t0 + 8, min(8, total - t0 - 8), sr, pt, raw);
++ { // query A: S = q . K^T
++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
++ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
++ simdgroup_bfloat8x8 a_hi, b;
++ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
++ simdgroup_load(a_hi, qhiA, 512, ulong2(ulong(kb) * 8ul, 0ul));
++ simdgroup_multiply_accumulate(acc, a_hi, b, acc);
++ }
++ simdgroup_store(acc, part + uint(sg) * 64u, 8);
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (tid < 64u) {
++ float s = 0.0f;
++ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
++ ssm[tid] = s * scale;
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ { // query A: online softmax
++ const int nq_here = min(8, totalA - t0);
++ const bool valid = int(lane) < nq_here;
++ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
++ const float m = ds4_attn_warp_max(s);
++ const float m_old = m_s[0 + sg];
++ const float m_new = fmax(m_old, m);
++ const float p = valid ? exp(s - m_new) : 0.0f;
++ const float ps = ds4_warp_sum(p);
++ if (lane < 8u) {
++ const bfloat hi = bfloat(p);
++ phi[uint(sg) * 8u + uint(lane)] = hi;
++ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
++ }
++ if (lane == 0u) {
++ const float cs = exp(m_old - m_new);
++ sc_s[0 + sg] = cs;
++ den_s[0 + sg] = den_s[0 + sg] * cs + ps;
++ m_s[0 + sg] = m_new;
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[0 + (tid >> 3)] : 0.0f;
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ { // query A: O = diag(rescale) O + P V
++ simdgroup_float8x8 D;
++ simdgroup_bfloat8x8 ph, pl;
++ simdgroup_load(D, dg, 8);
++ simdgroup_load(ph, phi, 8);
++ simdgroup_load(pl, plo, 8);
++ for (int c = 0; c < 8; ++c) {
++ simdgroup_float8x8 tmp;
++ simdgroup_multiply(tmp, D, OA[c]);
++ simdgroup_bfloat8x8 v;
++ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
++ simdgroup_multiply_accumulate(tmp, ph, v, tmp);
++#if DS4_MX_PLO
++ simdgroup_multiply_accumulate(tmp, pl, v, tmp);
++#endif
++ OA[c] = tmp;
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (nq > 1) {
++ { // query B: S = q . K^T
++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
++ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {
++ simdgroup_bfloat8x8 a_hi, b;
++ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);
++ simdgroup_load(a_hi, qhiB, 512, ulong2(ulong(kb) * 8ul, 0ul));
++ simdgroup_multiply_accumulate(acc, a_hi, b, acc);
++ }
++ simdgroup_store(acc, part + uint(sg) * 64u, 8);
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (tid < 64u) {
++ float s = 0.0f;
++ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];
++ ssm[tid] = s * scale;
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ { // query B: online softmax
++ const int nq_here = min(8, totalB - t0);
++ const bool valid = int(lane) < nq_here;
++ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;
++ const float m = ds4_attn_warp_max(s);
++ const float m_old = m_s[8 + sg];
++ const float m_new = fmax(m_old, m);
++ const float p = valid ? exp(s - m_new) : 0.0f;
++ const float ps = ds4_warp_sum(p);
++ if (lane < 8u) {
++ const bfloat hi = bfloat(p);
++ phi[uint(sg) * 8u + uint(lane)] = hi;
++ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));
++ }
++ if (lane == 0u) {
++ const float cs = exp(m_old - m_new);
++ sc_s[8 + sg] = cs;
++ den_s[8 + sg] = den_s[8 + sg] * cs + ps;
++ m_s[8 + sg] = m_new;
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[8 + (tid >> 3)] : 0.0f;
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ { // query B: O = diag(rescale) O + P V
++ simdgroup_float8x8 D;
++ simdgroup_bfloat8x8 ph, pl;
++ simdgroup_load(D, dg, 8);
++ simdgroup_load(ph, phi, 8);
++ simdgroup_load(pl, plo, 8);
++ for (int c = 0; c < 8; ++c) {
++ simdgroup_float8x8 tmp;
++ simdgroup_multiply(tmp, D, OB[c]);
++ simdgroup_bfloat8x8 v;
++ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
++ simdgroup_multiply_accumulate(tmp, ph, v, tmp);
++#if DS4_MX_PLO
++ simdgroup_multiply_accumulate(tmp, pl, v, tmp);
++#endif
++ OB[c] = tmp;
++ }
++ }
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ }
++ }
++ for (int c = 0; c < 8; ++c) simdgroup_store(OA[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ for (uint x = 0; x < 16u; ++x) {
++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
++ out[((size_t) tA * heads + uint(hb) + h) * 512u + d] = obuf[idx] / (den_s[h] + exp(sink[uint(hb) + h] - m_s[h]));
++ }
++ if (nq > 1) {
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ for (int c = 0; c < 8; ++c) simdgroup_store(OB[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));
++ threadgroup_barrier(mem_flags::mem_threadgroup);
++ for (uint x = 0; x < 16u; ++x) {
++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;
++ out[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d] =
++ obuf[idx] / (den_s[8u + h] + exp(sink[uint(hb) + h] - m_s[8u + h]));
++ }
++ }
++}
++
+ // kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the
+ // softmax, every thread two output values.
+ static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,
+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,
+ device const float* sink, uint heads, float scale, device float* out,
+ threadgroup float* qs, threadgroup float* sc, threadgroup int* rowix,
+ threadgroup float* m_run_s, threadgroup float* tscale_s,
+ threadgroup float* den_s, uint2 tgp, uint tid, ushort lane_in,
+ ushort wp_in) {
+ const int h = int(tgp.x), t = int(tgp.y);
+ int n_win, n_sel, w0;
+ ds4_attn_counts(ar, t, n_win, n_sel, w0);
+ const device ushort* wbase = win + (size_t) w0 * 512u;
+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;
+ const uint lane = uint(lane_in), wp = uint(wp_in);
+ for (uint i = tid; i < 512u; i += 256u) qs[i] = q[((size_t) t * heads + uint(h)) * 512u + i];
+ if (tid == 0u) {
+ m_run_s[0] = -1e30f;
+ den_s[0] = 0.0f;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float o0 = 0.0f, o1 = 0.0f;
+ const int c = int(tid) * 2;
+ const int total = n_win + n_sel;
+ for (int t0 = 0; t0 < total; t0 += 64) {
+ const int nt = min(64, total - t0);
+ if (tid < uint(nt)) {
+ const int j = t0 + int(tid);
+ rowix[tid] = j < n_win ? -1 - j : sl[j - n_win];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int jj = int(wp); jj < nt; jj += 8) {
+ const int code = rowix[jj];
+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
+ : ckv + (size_t) code * 512u;
+ const device uint4* rp = (const device uint4*) (row + lane * 16u);
+ const uint4 a = rp[0], b = rp[1];
+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};
+ float s = 0.0f;
+ for (int x = 0; x < 4; ++x) {
+ s += qs[lane * 16u + uint(2 * x)] * ds4_bf16_u32(av[x] & 0xffffu);
+ s += qs[lane * 16u + uint(2 * x + 1)] * ds4_bf16_u32(av[x] >> 16);
+ }
+ for (int x = 0; x < 4; ++x) {
+ s += qs[lane * 16u + uint(8 + 2 * x)] * ds4_bf16_u32(bv[x] & 0xffffu);
+ s += qs[lane * 16u + uint(9 + 2 * x)] * ds4_bf16_u32(bv[x] >> 16);
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) sc[jj] = s * scale;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (wp == 0u) {
+ float m = -INFINITY;
+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, sc[j]);
+ m = ds4_attn_warp_max(m);
+ const float m_old = m_run_s[0];
+ const float m_new = fmax(m_old, m);
+ float s = 0.0f;
+ for (int j = int(lane); j < nt; j += 32) {
+ const float p = exp(sc[j] - m_new);
+ sc[j] = p;
+ s += p;
+ }
+ s = ds4_warp_sum(s);
+ if (lane == 0u) {
+ tscale_s[0] = exp(m_old - m_new);
+ den_s[0] = den_s[0] * tscale_s[0] + s;
+ m_run_s[0] = m_new;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const float cs = tscale_s[0];
+ o0 *= cs;
+ o1 *= cs;
+ for (int j = 0; j < nt; ++j) {
+ const int code = rowix[j];
+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u
+ : ckv + (size_t) code * 512u;
+ const uint pr = *(const device uint*) (row + c);
+ o0 = fma(sc[j], ds4_bf16_u32(pr & 0xffffu), o0);
+ o1 = fma(sc[j], ds4_bf16_u32(pr >> 16), o1);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ const float den = den_s[0] + exp(sink[h] - m_run_s[0]);
+ device float* ov = out + ((size_t) t * heads + uint(h)) * 512u;
+ ov[c] = o0 / den;
+ ov[c + 1] = o1 / den;
+ }
+
+ kernel void ds4_sparse_attn_bf16(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],
+ device const ushort* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],
+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],
+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],
+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float qs[512];
+ threadgroup float sc[64];
+ threadgroup int rowix[64];
+ threadgroup float m_run_s, tscale_s, den_s;
+ ds4_sparse_bf16_run(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, qs, sc, rowix, &m_run_s,
+ &tscale_s, &den_s, tgp, tid, lane, wp);
+ }
+
+ // ds4_kernels.cu:461 route_kernel - softplus scores, their top k with bias, weights from the unbiased scores. One
+ // threadgroup (512) a token, as the CUDA kernel; rows 1024 values apart, the picks kPicks = 8 apart.
+ kernel void ds4_route(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],
+ constant uint& n [[buffer(2)]], constant uint& k [[buffer(3)]], constant float& route_scale [[buffer(4)]],
+ device int* idx [[buffer(5)]], device float* w [[buffer(6)]], device const float* bias_vl [[buffer(7)]],
+ constant uint& has_vl [[buffer(8)]], device const uint* vlw [[buffer(9)]],
+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float sc[1024], bs[1024];
+ threadgroup float bv[32];
+ threadgroup int bi[32];
+ threadgroup int pick[16];
+ const uint t = tg;
+ device const float* l = logits + (size_t) t * 1024u;
+ device const float* bp = bias;
+ if (has_vl != 0u && ((vlw[t >> 5u] >> (t & 31u)) & 1u) != 0u) bp = bias_vl;
+ for (uint i = tid; i < n; i += 512u) {
+ const float v = l[i];
+ const float sp = v > 20.0f ? v : log(1.0f + exp(v));
+ sc[i] = sqrt(sp);
+ bs[i] = sc[i] + bp[i];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint r = 0; r < k; ++r) {
+ float best = -INFINITY;
+ int bix = 0x7fffffff;
+ for (uint i = tid; i < n; i += 512u)
+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) {
+ best = bs[i];
+ bix = int(i);
+ }
+ for (uint o = 16u; o > 0u; o >>= 1u) {
+ const float ov = simd_shuffle_xor(best, o);
+ const int oi = simd_shuffle_xor(bix, o);
+ if (ov > best || (ov == best && oi < bix)) {
+ best = ov;
+ bix = oi;
+ }
+ }
+ if (lane == 0u) {
+ bv[wp] = best;
+ bi[wp] = bix;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) {
+ float b = -INFINITY;
+ int ix = 0x7fffffff;
+ for (uint q = 0; q < 16u; ++q)
+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) {
+ b = bv[q];
+ ix = bi[q];
+ }
+ pick[r] = ix;
+ bs[ix] = -INFINITY;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) {
+ float s = 0.0f;
+ for (uint r = 0; r < k; ++r) s += sc[pick[r]];
+ for (uint r = 0; r < k; ++r) {
+ idx[(size_t) t * 8u + r] = pick[r];
+ w[(size_t) t * 8u + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;
+ }
+ }
+ }
+
+ // ds4_kernels.cu:1336 gather_kernel and :1342 combine_kernel - a token's gathered expert-output rows and the MoE's sum.
+ struct Ds4Gather {
+ int src[16 * 8];
+ };
+ struct Ds4Combine {
+ int n[16];
+ int row[16][8];
+ float w[16][8];
+ int cpu[16];
+ };
+
+ kernel void ds4_gather(device const float* x [[buffer(0)]], constant Ds4Gather& g [[buffer(1)]],
+ constant uint& cols [[buffer(2)]], device float* out [[buffer(3)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.y;
+ const uint c = tgp.x * 256u + tid;
+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];
+ }
+
+ kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float* cpu [[buffer(1)]],
+ device const float* sh [[buffer(2)]], constant Ds4Combine& cm [[buffer(3)]],
+ constant uint& dim [[buffer(4)]], constant uint& has_cpu [[buffer(5)]],
+ constant uint& has_sh [[buffer(6)]], device float* out [[buffer(7)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + tid, t = tgp.y;
+ if (e >= dim) return;
+ float a = 0.0f;
+ for (int j = 0; j < cm.n[t]; ++j) a = fma(cm.w[t][j], ed[(size_t) cm.row[t][j] * dim + e], a);
+ if (has_cpu != 0u && cm.cpu[t] != 0) a += cpu[(size_t) t * dim + e];
+ if (has_sh != 0u) a += sh[(size_t) t * dim + e];
+ out[(size_t) t * dim + e] = a;
+ }
+
+ // ---------------------------------------------------------------- ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel
+ //
+ // The drafter's head (DSpark's Markov chain and its confidence): markov_add is a warp a vocabulary row - the previous
+ // token's E row (BF16) staged as floats in the threadgroup, then E . H[v] added to logits[v] (H is F16, the pack's
+ // markov_head.head.weight); confidence is a threadgroup a token over x and the same E row against the head's weights,
+ // its block sum through the logistic.
+ struct Ds4MarkovJob {
+ device float* logits; // [V]: += the bias
+ device const ushort* E; // [vocab][R] BF16
+ device const ushort* H; // [vocab][R] F16
+ device const int* tok; // the previous token (one int)
+ int V, R;
+ };
+
+ struct Ds4ConfJob {
+ device const float* x; // [T][dim]
+ device const ushort* E; // [vocab][R] BF16
+ device const int* tok; // [T]
+ device const float* w; // [dim + R]
+ device float* out; // [T]
+ int dim, R;
+ };
+
+ // a BF16 bit pattern as its float (ds4_kernels.cu:28 bf16f: the top half, no rounding)
+ static inline float ds4_bf16f(ushort b) { return as_type<float>(uint(b) << 16u); }
+
+ // a warp a vocabulary row: logits[v] += E[*tok] . H[v]
+ kernel void ds4_markov_add(constant Ds4MarkovJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ threadgroup float er[1024]; // (R <= 1024: the host wrapper refuses more)
+ const device ushort* erow = j.E + (size_t) (*j.tok) * (size_t) j.R;
+ for (int i = int(tid); i < j.R; i += 256) er[i] = ds4_bf16f(erow[i]);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int v = int(tg) * 8 + int(tid >> 5);
+ if (v >= j.V) return;
+ const device ushort* hr = j.H + (size_t) v * (size_t) j.R;
+ float a = 0.0f;
+ for (int c = int(lane) * 8; c < j.R; c += 256) {
+ for (int i = 0; i < 8; ++i) a += float(as_type<half>(hr[c + i])) * er[c + i];
+ }
+ a = ds4_warp_sum(a);
+ if (lane == 0u) j.logits[v] += a;
+ }
+
+ // a threadgroup a token: the logistic of the head's sum over x and the token's E row
+ kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],
+ ushort w [[simdgroup_index_in_threadgroup]]) {
+ threadgroup float red[32];
+ const int t = int(tg);
+ const device float* xt = j.x + (size_t) t * (size_t) j.dim;
+ const device ushort* er = j.E + (size_t) j.tok[t] * (size_t) j.R;
+ float a = 0.0f;
+ for (int i = int(tid); i < j.dim + j.R; i += 1024) a += j.w[i] * (i < j.dim ? xt[i] : ds4_bf16f(er[i - j.dim]));
+ a = ds4_block_sum_n(a, red, lane, w, 32u);
+ if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));
+ }
+
+ // embed by raw device addresses (the runtime's MTLResidencySet makes gpuAddress pointers valid kernel arguments)
+ struct Ds4EmbedArgs {
+ device ushort* rows;
+ device float* h;
+ uint dim;
+ uint hc;
+ };
+ kernel void ds4_embed_j(device const Ds4EmbedArgs* a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ ushort lane [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + uint(lane);
+ if (e >= a->dim) return;
+ const uint t = tgp.y;
+ const uint bits = uint(a->rows[(size_t) t * a->dim + e]) << 16;
+ const float v = as_type<float>(bits);
+ for (uint s = 0; s < a->hc; ++s) a->h[((size_t) t * a->hc + s) * a->dim + e] = v;
+ }
+
+ // (diagnostic) write a constant through a device pointer passed as a value, and one passed in a buffer
+ kernel void ds4_test_write(device float* out [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
+ if (gid < n) out[gid] = 3.0f;
+ }
+ kernel void ds4_test_write_v(device const uint64_t* p [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
+ device float* out = (device float*) (uintptr_t) p[0];
+ if (gid < n) out[gid] = 7.0f;
+ }
+
+ // (diagnostic) report the arguments the kernel received
+ kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [[buffer(1)]],
+ constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {
+ if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }
+ }
+
+ // hc == 4 (every layer of this model): the warp-lane form of the CUDA kernel - lane j*4+k holds comb[j][k], the row and column
+ // sums are xor-shuffles (1, 2 within a row; 4, 8 within a column). One simdgroup a token instead of one lane.
+ kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],
+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],
+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],
+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],
+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],
+ uint lane [[thread_index_in_threadgroup]]) {
+ const device float* mx = mixes + (size_t) b * 32u;
+ if (lane < 4u) {
+ pre[(size_t) b * 8u + lane] = 1.0f / (1.0f + exp(-(mx[lane] * scale[0] + base[lane]))) + eps;
+ post[(size_t) b * 8u + lane] = 2.0f / (1.0f + exp(-(mx[4u + lane] * scale[1] + base[4u + lane])));
+ }
+ const bool on = lane < 16u;
+ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;
+ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));
+ m = fmax(m, simd_shuffle_xor(m, ushort(2)));
+ c = on ? exp(c - m) : 0.0f;
+ float rs = c + simd_shuffle_xor(c, ushort(1));
+ rs += simd_shuffle_xor(rs, ushort(2));
+ c = c / rs + eps;
+ c = on ? c : 0.0f;
+ float cs = c + simd_shuffle_xor(c, ushort(4));
+ cs += simd_shuffle_xor(cs, ushort(8));
+ c = c / (cs + eps);
+ for (uint it = 1; it < iters; ++it) {
+ float r = c + simd_shuffle_xor(c, ushort(1));
+ r += simd_shuffle_xor(r, ushort(2));
+ c = c / (r + eps);
+ float s = c + simd_shuffle_xor(c, ushort(4));
+ s += simd_shuffle_xor(s, ushort(8));
+ c = c / (s + eps);
+ }
+ if (on) comb[(size_t) b * 64u + lane] = c;
+ }
+
+ // Plain byte copies and fills on the context's queue: cudaMemcpyAsync / cudaMemsetAsync between runtime allocations (unified
+ // memory: a "device" and a "host" buffer are both just memory) ride in the same command buffer as the kernels around them,
+ // so their order is the program order and a layer needs no queue-to-queue handshake.
+ kernel void ds4_copy4(device const uint* src [[buffer(0)]], device uint* dst [[buffer(1)]], constant ulong& n4 [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n4) dst[gid] = src[gid];
+ }
+ kernel void ds4_copy1(device const uchar* src [[buffer(0)]], device uchar* dst [[buffer(1)]], constant ulong& n [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n) dst[gid] = src[gid];
+ }
+ kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffer(1)]], constant uint& v [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n) dst[gid] = uchar(v);
+ }
+
+ // ---------------------------------------------------------------------------------------------------------------------
+ // The prompt chunk's dense GEMM operands (see ds4_gemm_f16_nt on the host): float rows -> half, a quantized matrix -> half rows,
+ // strided half rows packed, and the float result scattered into the engine's strided output.
+
+ kernel void ds4_rows_f16(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],
+ device const int* idx [[buffer(2)]], constant uint& has_idx [[buffer(3)]],
+ constant uint& ncols [[buffer(4)]], device half* out [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint r = tgp.y, c = tgp.x * 256u + tid;
+ if (c >= ncols) return;
+ const ulong row = has_idx != 0u ? ulong(idx[r]) : ulong(r);
+ out[(ulong) r * ncols + c] = half(x[row * ldx + c]);
+ }
+
+ static inline void ds4_kscale2(const device uchar* q, int j, thread int& d, thread int& m) {
+ if (j < 4) {
+ d = int(q[j]) & 63;
+ m = int(q[j + 4]) & 63;
+ } else {
+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);
+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);
+ }
+ }
+
+ // a thread a 32-value sub-block of a Q5_K matrix (n rows of k values, ld bytes a row) -> out[n][k] half
+ kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const uint nb = k / 256u;
+ const ulong total = (ulong) n * nb * 8ul;
+ if (ulong(gid) >= total) return;
+ const uint ib = gid & 7u;
+ const uint blk = (gid >> 3) % nb;
+ const uint row = uint(ulong(gid) / (8ul * nb));
+ const device uchar* p = W + (ulong) row * ld + (ulong) blk * 176ul;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc, mm;
+ ds4_kscale2(p + 4, int(ib), sc, mm);
+ const device uchar* qh = p + 16;
+ const device uchar* qs = p + 48 + 32u * (ib >> 1);
+ const uint sh = 4u * (ib & 1u);
+ device packed_half4* o = (device packed_half4*) (out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u);
+ const float ds = d * float(sc), dm = dmin * float(mm);
+ for (uint l = 0; l < 32u; l += 4u) { // four values a step: 4-byte loads, 8-byte stores (the same arithmetic a value)
+ const uint4 a = uint4(*(const device packed_uchar4*) (qs + l));
+ const uint4 b = uint4(*(const device packed_uchar4*) (qh + l));
+ const uint4 q = ((a >> sh) & 15u) | (((b >> ib) & 1u) << 4);
+ o[l >> 2] = packed_half4(half4(float4(ds) * float4(q) - float4(dm)));
+ }
+ }
+
+ // Q8_0: 34-byte blocks (fp16 d, 32 int8)
+ kernel void ds4_dq_q8(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const uint ng = k / 32u;
+ const ulong total = (ulong) n * ng;
+ if (ulong(gid) >= total) return;
+ const uint g = gid % ng, row = gid / ng;
+ const device uchar* p = W + (ulong) row * ld + (ulong) g * 34ul;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ device half* o = out + (ulong) row * k + (ulong) g * 32ul;
+ for (uint l = 0; l < 32u; ++l) o[l] = half(d * float(char(p[2 + l])));
+ }
+
+ // BF16 values (ld values a row) -> half
+ kernel void ds4_dq_bf16(device const ushort* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {
+ const ulong total = (ulong) n * k;
+ if (ulong(gid) >= total) return;
+ const ulong r = ulong(gid) / k, c = ulong(gid) - r * k;
+ out[gid] = half(as_type<float>(uint(W[r * ld + c]) << 16));
+ }
+
+ // half rows with a row stride -> contiguous rows
+ kernel void ds4_pack_h16(device const ushort* x [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],
+ device ushort* out [[buffer(3)]], uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c < k) out[(ulong) tgp.y * k + c] = x[(ulong) tgp.y * ld + c];
+ }
+
+ // y[r][c] = alpha * t[r][c] + beta * y[r][c]
+ kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y [[buffer(1)]], constant ulong& ldy [[buffer(2)]],
+ constant uint& n [[buffer(3)]], constant float& alpha [[buffer(4)]], constant float& beta [[buffer(5)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c >= n) return;
+ const ulong yi = (ulong) tgp.y * ldy + c;
+ y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);
+ }
+
+ // ds4_q8_quant for several jobs in one dispatch (a matvec call's activations): grid (groups, max rows, jobs)
+ struct Ds4Q8Spec {
+ device const float* x;
+ ulong ldx;
+ uint rows;
+ uint k;
+ device char* q;
+ device float* d;
+ };
+ kernel void ds4_q8_quant_multi(device const Ds4Q8Spec* specs [[buffer(0)]], uint3 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const device Ds4Q8Spec& s = specs[tgp.z];
+ const uint warp = tid / 32u, lane = tid % 32u;
+ const uint b = tgp.x * 8u + warp;
+ const uint r = tgp.y;
+ if (r >= s.rows || (size_t) b * 32u >= (size_t) s.k) return;
+ const device float* xr = s.x + (size_t) r * s.ldx;
+ const float v = xr[(size_t) b * 32u + lane];
+ const float am = simd_max(fabs(v));
+ const float sc = am / 127.0f;
+ s.q[(size_t) r * s.k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
+ if (lane == 0u) s.d[(size_t) r * (s.k / 32u) + b] = sc;
+ }
diff --cc metal/src/engine/msl/ds4_iq.metal
index 000000000,15f60df47..ea17f7271
mode 000000,100644..100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@@ -1,0 -1,1060 +1,1092 @@@
+ #include <metal_stdlib>
+ using namespace metal;
+ // ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the
+ // 256-value blocks, sixteen accumulator columns for the token window), with the block decoder chosen per job by b.type:
+ // 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),
+ // so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a
+ // quarter of the traffic.
+ struct Ds4MvJob {
+ device const uchar* w;
+ ulong ld;
+ int type, k, n, ncol;
+ device const char* xq;
+ device const float* xd;
+ device float* y;
+ ulong ycs;
+ device const float* bias;
+ float alpha;
+ };
+
+ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nsb = uint(b.k) / 32u;
+ const uint nblk = uint(b.k) / 256u;
+ const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : (b.type == 18 ? 98u : 82u));
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ float acc[16];
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
+ for (uint blk = lane; blk < nblk; blk += 32u) {
+ const device uchar* p = w + (size_t) blk * bs;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ for (uint ib = 0; ib < 8u; ++ib) {
+ if (b.type == 18) { // IQ3_XXS: d, qs[64] (two grid bytes a 8 values), scales_and_signs[8] words (top nibble the scale)
+ const device uchar* qs = p + 2;
+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) | (uint(qs[8 * ib + 3]) << 24);
+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) | (uint(qs[8 * ib + 7]) << 24);
+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) | (uint(qs[67 + 4 * ib]) << 24);
+ const float db3 = d * (0.5f + float(aux >> 28)) * 0.5f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];
+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sum = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);
+ if ((sgn & (1u << j)) != 0u) v0 = -v0;
+ if ((sgn & (1u << (j + 4u))) != 0u) v1 = -v1;
+ sum += v0 * int(xq[j]) + v1 * int(xq[j + 4u]);
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db3, float(sum), acc[c]);
+ }
+ }
+ continue;
+ }
+ float db0, db1;
+ uint idx[4];
+ uint sg[4];
+ if (b.type == 16) {
+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |
+ (uint(p[5 + 8 * ib]) << 24); // (8 bytes a sub-block: grid bytes, then signs and scale)
+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |
+ (uint(p[9 + 8 * ib]) << 24);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ db0 = db1 = db;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = (aux0 >> (8u * l)) & 0xffu;
+ sg[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ }
+ } else if (b.type == 17) {
+ const device ushort* qs = (const device ushort*) (p + 2);
+ const device uchar* sc = p + 66;
+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[4u * ib + l]);
+ idx[l] = q & 511u;
+ sg[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]); // the sign byte of the 7 stored bits (bit j set: value j negated)
+ }
+ } else {
+ const device uchar* qs = p + 2;
+ const device uchar* sgb = qs + 32;
+ const device uchar* qh = p + 66;
+ const device uchar* sc = p + 74;
+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = uint(qs[4u * ib + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);
+ sg[l] = uint(sgb[4u * ib + l]);
+ }
+ }
+ for (uint l = 0; l < 4u; ++l) {
+ ulong g;
+ if (b.type == 16) g = iq2xxs_grid[idx[l]];
+ else if (b.type == 17) g = iq2xs_grid[idx[l]];
+ else g = iq2s_grid[idx[l] & 1023u];
+ const float db = (l < 2u) ? db0 : db1;
+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sum = 0;
+ for (uint j = 0; j < 8u; ++j) {
+ const int code = int((g >> (8u * j)) & 0xffu);
+ const bool neg = b.type == 16 ? ((sg[l] & uint(kmask_iq2xs[j])) != 0u)
+ : ((sg[l] & (1u << j)) != 0u);
+ sum += (neg ? -code : code) * int(xq[j]);
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db, float(sum), acc[c]);
+ }
+ }
+ }
+ }
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ // Q5_K / Q6_K (the dense matrices' K-quants; ggml blocks of 176 / 210 bytes): the same warp a row frame, a lane a
+ // 256-value block, each value's code times its activation's int8. Q5_K's weight is d*sc*q - dmin*m (both parts of the
+ // 32-value group's scales), Q6_K's d*sc*(q - 32).
+ static inline void ds4_k_scale(const device uchar* q, int j, thread int& d, thread int& m) {
+ if (j < 4) {
+ d = int(q[j]) & 63;
+ m = int(q[j + 4]) & 63;
+ } else {
+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);
+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);
+ }
+ }
+
+ kernel void ds4_iq_mv_k(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nsb = uint(b.k) / 32u;
+ const uint nblk = uint(b.k) / 256u;
+ const uint bs = b.type == 13 ? 176u : 210u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ float acc[16];
+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;
+ for (uint blk = lane; blk < nblk; blk += 32u) {
+ const device uchar* p = w + (size_t) blk * bs;
+ if (b.type == 13) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ const device uchar* scales = p + 4;
+ const device uchar* qh = p + 16;
+ const device uchar* qs = p + 48;
+ for (uint ib = 0; ib < 8u; ++ib) {
+ int sc, mm;
+ ds4_k_scale(scales, int(ib), sc, mm);
+ const device uchar* qsp = qs + 32u * (ib >> 1);
+ const uint sh = 4u * (ib & 1u);
+ for (uint t = 0; t < 8u; ++t) {
+ const uint q4 = (*(const device uint*) (qsp + 4u * t) >> sh) & 0x0f0f0f0fu;
+ const uint h4 = (*(const device uint*) (qh + 4u * t) >> ib) & 0x01010101u;
+ const uint q = q4 | (h4 << 4);
+ const uint off = blk * 256u + ib * 32u + t * 4u;
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sumq = 0, sumx = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ const int code = int((q >> (8u * j)) & 0xffu);
+ const int xv = int(xq[j]);
+ sumq += code * xv;
+ sumx += xv;
+ }
+ const float dot = d * float(sc) * float(sumq) - dmin * float(mm) * float(sumx);
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)], dot, acc[c]);
+ }
+ }
+ }
+ } else {
+ const float d = float(as_type<half>(*(const device ushort*) (p + 208)));
+ const device uchar* ql = p;
+ const device uchar* qh = p + 128;
+ const device uchar* scl = p + 192;
+ for (uint ib = 0; ib < 8u; ++ib) {
+ const uint h = ib >> 2, quarter = ib & 3u;
+ const device uchar* qlp = ql + 64u * h + ((quarter & 1u) ? 32u : 0u);
+ const device uchar* qhp = qh + 32u * h;
+ const uint sh = 4u * (quarter >> 1);
+ const int s0 = int(((const device char*) scl)[8u * h + 2u * quarter]);
+ const int s1 = int(((const device char*) scl)[8u * h + 2u * quarter + 1u]);
+ for (uint t = 0; t < 8u; ++t) {
+ const uint x = ((*(const device uint*) (qlp + 4u * t) >> sh) & 0x0f0f0f0fu) |
+ (((*(const device uint*) (qhp + 4u * t) >> (2u * quarter)) & 0x03030303u) << 4);
+ const uint off = blk * 256u + ib * 32u + t * 4u;
+ const float s = float(t < 4u ? s0 : s1);
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ const device char* xq = b.xq + (size_t) c * b.k + off;
+ int sumq = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ const int code = int((x >> (8u * j)) & 0xffu) - 32;
+ sumq += code * int(xq[j]);
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * d * s, float(sumq), acc[c]);
+ }
+ }
+ }
+ }
+ }
+ for (uint c = 0; c < 16u; ++c) {
+ if (int(c) >= b.ncol) break;
+ float v = acc[c];
+ v += simd_shuffle_xor(v, 16);
+ v += simd_shuffle_xor(v, 8);
+ v += simd_shuffle_xor(v, 4);
+ v += simd_shuffle_xor(v, 2);
+ v += simd_shuffle_xor(v, 1);
+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_iq.cu:536 gate_up_kernel / :588 down_kernel
+ //
+ // The prompt / window path's expert GEMM (ds4_iq.cu's launch_gu / launch_dn): a threadgroup of 256 = 8 warps, a warp
+ // RW rows of the gate and of the up (down: RW rows of the down), a lane 32 / RW sub-blocks of the row, and the group's
+ // entries NE at a pass so that a sub-block the weights decode once serves them all. Grid (rows / (8 RW), groups).
+ //
+ // The group's blob and its entries are the CUDA kernel's `group` (ds4_iq.cu:512): a plan group (grp_ptr null) is one
+ // expert whose entry is the group's own index and whose activation row is 0, a window group's entries come from
+ // grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.
+ struct Ds4IqJob {
+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)
+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob
+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)
+ device const int* grp_ent; // the entries (token * 8 + pick)
+ device const int* n_groups;
+ device const char* xq; // the int8 activations of a row (n_embd an entry)
+ device const float* xd; // their scales (n_embd / 32 a row)
+ device const char* hq; // down's int8 input (n_ff an entry)
+ device const float* hd;
+ device float* h; // [entries][n_ff]: the clamped swiglu
+ device float* part; // [entries][n_embd]: each entry's down output
+ long tr0, tr1, tr2; // the roles' offsets in a blob
+ int n_embd, n_ff;
+ int type; // 16 IQ2_XXS, 17 IQ2_XS, 22 IQ2_S (ds4_iq.cuh's kIQ2_*)
+ float limit; // swiglu_limit (0: none)
+ };
+
+ // a 32-value sub-block as 32 int8 codes (8 packed ints, values 4 w .. 4 w + 3 in word w) and the two halves' scales
+ // (ggml's d (2 s + 1) / 8, the .cu's Sub.d * Sub.s0 in one F32: the codes of the second half are scaled by s1)
+ struct Ds4IqSub {
+ int q[8];
+ float s0, s1;
+ };
+
+ // the block bytes of the types these kernels take: 16 IQ2_XXS, 17 IQ2_XS, 18 IQ3_XXS, 22 IQ2_S (ds4_iq.cu's Ty<T>::B)
+ static inline uint ds4_iq_bs(int type) {
+ return type == 16 ? 66u : type == 17 ? 74u : type == 18 ? 98u : 82u;
+ }
+
+ // ds4_iq.cu:139 decode<T>: the block's 8 sub-blocks, kb's is the (kb & 7)'th; the grids are ggml-common.h's
+ static inline Ds4IqSub ds4_iq_sub(device const uchar* row, uint bs, int kb, int type) {
+ Ds4IqSub s;
+ const int ib = kb & 7;
+ const device uchar* p = row + (size_t) (kb >> 3) * bs;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ uint idx[4], sm[4];
+ float db0, db1;
+ if (type == 18) { // IQ3_XXS: d, qs[64] grid bytes (4 a sub-block), scales[8] (the top nibble the group's s)
+ const device uchar* qs = p + 2;
+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) |
+ (uint(qs[8 * ib + 3]) << 24);
+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) |
+ (uint(qs[8 * ib + 7]) << 24);
+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) |
+ (uint(qs[67 + 4 * ib]) << 24);
+ for (uint l = 0; l < 4u; ++l) {
+ const uint w = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sg = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(w >> sh) & 255u], g1 = iq3xxs_grid[(w >> (sh + 8u)) & 255u];
+ int a0 = 0, a1 = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);
+ if ((sg & (1u << j)) != 0u) v0 = -v0;
+ if ((sg & (1u << (j + 4u))) != 0u) v1 = -v1;
+ a0 |= (v0 & 0xff) << int(8u * j);
+ a1 |= (v1 & 0xff) << int(8u * j);
+ }
+ s.q[2u * l] = a0;
+ s.q[2u * l + 1u] = a1;
+ }
+ const float ds = d * (0.5f + float(aux >> 28)) * 0.5f; // ggml's db: d (0.5 + s) 0.5
+ s.s0 = ds;
+ s.s1 = ds;
+ return s;
+ }
+ if (type == 16) { // d, qs[8] i16 (4 grid bytes, then 4 sign/scale bytes), scales in the top nibbles
+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |
+ (uint(p[5 + 8 * ib]) << 24);
+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |
+ (uint(p[9 + 8 * ib]) << 24);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ db0 = db1 = db;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = (aux0 >> (8u * l)) & 0xffu;
+ sm[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ }
+ } else if (type == 17) { // d, qs[16] i16 (9-bit grid, 7 sign bits), scales[8] (two 4-bit)
+ const device ushort* qs = (const device ushort*) (p + 2);
+ const device uchar* sc = p + 66;
+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[4u * uint(ib) + l]);
+ idx[l] = q & 511u;
+ sm[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ }
+ } else { // IQ2_S: d, qs[32] (8-bit grid), signs[32], qh[8] (2 bits more), scales[8]
+ const device uchar* qs = p + 2;
+ const device uchar* sgb = qs + 32;
+ const device uchar* qh = p + 66;
+ const device uchar* sc = p + 74;
+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;
+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ idx[l] = uint(qs[4u * uint(ib) + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);
+ sm[l] = uint(sgb[4u * uint(ib) + l]);
+ }
+ }
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = type == 16 ? iq2xxs_grid[idx[l]] : (type == 17 ? iq2xs_grid[idx[l]] : iq2s_grid[idx[l] & 1023u]);
+ int w0 = 0, w1 = 0;
+ for (uint j = 0; j < 4u; ++j) {
+ int v0 = int((g >> (8u * j)) & 0xffu), v1 = int((g >> (8u * (j + 4u))) & 0xffu);
+ if ((sm[l] & (1u << j)) != 0u) v0 = -v0;
+ if ((sm[l] & (1u << (j + 4u))) != 0u) v1 = -v1;
+ w0 |= (v0 & 0xff) << int(8u * j);
+ w1 |= (v1 & 0xff) << int(8u * j);
+ }
+ s.q[2u * l] = w0;
+ s.q[2u * l + 1u] = w1;
+ }
+ s.s0 = db0;
+ s.s1 = db1;
+ return s;
+ }
+
+ // 4 int8 codes against the 4 int8 activations packed in x (ds4_iq.cu:350's __dp4a: Metal has no such instruction)
+ static inline int ds4_iq_dot4(int q, int x) {
+ const char4 a = as_type<char4>(q), b = as_type<char4>(x);
+ return int(a.x) * int(b.x) + int(a.y) * int(b.y) + int(a.z) * int(b.z) + int(a.w) * int(b.w);
+ }
+
+ // ds4_iq.cu:349 sub_dot: the sub-block against its 32 int8 activations (8 packed ints), the scales in F32
+ static inline float ds4_iq_dot(Ds4IqSub s, device const int* xp) {
+ const int lo = ds4_iq_dot4(s.q[0], xp[0]) + ds4_iq_dot4(s.q[1], xp[1]) + ds4_iq_dot4(s.q[2], xp[2]) +
+ ds4_iq_dot4(s.q[3], xp[3]);
+ const int hi = ds4_iq_dot4(s.q[4], xp[4]) + ds4_iq_dot4(s.q[5], xp[5]) + ds4_iq_dot4(s.q[6], xp[6]) +
+ ds4_iq_dot4(s.q[7], xp[7]);
+ return s.s0 * float(lo) + s.s1 * float(hi);
+ }
+
+ // ds4_iq.cu:502 lanes_sum: the row's sum over the 32 / RW lanes that share it
+ template <int RW>
+ static inline float ds4_iq_lsum(float v) {
+ for (int o = 16 / RW; o > 0; o >>= 1) v += simd_shuffle_xor(v, ushort(o));
+ return v;
+ }
+
+ // ds4_iq.cu:497 swiglu: the limit's clamps, then g / (1 + e^-g) * u
+ static inline float ds4_iq_swiglu(float gv, float uv, float limit) {
+ if (limit > 0.0f) {
+ gv = min(gv, limit);
+ uv = min(max(uv, -limit), limit);
+ }
+ return (gv / (1.0f + exp(-gv))) * uv;
+ }
+
+ // the group's blob and its entry range (ds4_iq.cu:512 group); false: the threadgroup has nothing to do
+ static inline bool ds4_iq_group(constant Ds4IqJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {
+ if (j.grp_ptr == nullptr) {
+ const ulong p = j.plan_ptr[g];
+ if (p == 0ul) return false;
+ blob = p;
+ m0 = int(g);
+ m1 = m0 + 1;
+ return true;
+ }
+ if (int(g) >= *j.n_groups) return false;
+ blob = j.grp_ptr[g];
+ m0 = j.grp_start[g];
+ m1 = j.grp_start[g + 1];
+ return true;
+ }
+
+ // gate_up_kernel<T, NE> (kGuRW = 1: a warp a row, a lane a sub-block): h = the clamped swiglu of the gate and the up
+ kernel void ds4_iq_gu(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_ff) return;
+ const uint bs = ds4_iq_bs(j.type);
+ const uint nsb = uint(j.n_embd) / 32u;
+ const size_t rb = (size_t) (j.n_embd / 256) * (size_t) bs;
+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;
+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 4) {
+ const int ne = min(4, m1 - mm);
+ int es[4], xr[4];
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ es[i] = e;
+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);
+ }
+ }
+ float ag[4] = {0.0f, 0.0f, 0.0f, 0.0f}, au[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ for (uint kb = uint(lane); kb < nsb; kb += 32u) {
+ const Ds4IqSub sg = ds4_iq_sub(g, bs, int(kb), j.type);
+ const Ds4IqSub su = ds4_iq_sub(u, bs, int(kb), j.type);
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const device int* xp = (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd +
+ (size_t) kb * 32u);
+ const float dx = j.xd[(size_t) xr[i] * nsb + kb];
+ ag[i] += dx * ds4_iq_dot(sg, xp);
+ au[i] += dx * ds4_iq_dot(su, xp);
+ }
+ }
+ }
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);
+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);
+ }
+ }
+ }
+ }
+
+ // down_kernel<T, NE> (kDnRW = 2: a warp two rows, 16 lanes a row): part = the down against the int8 h
+ kernel void ds4_iq_dn(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = (int(tgp.x) * 8 + int(tid >> 5)) * 2 + int(lane) / 16;
+ if (row >= j.n_embd) return;
+ const uint kl = uint(lane) % 16u;
+ const uint bs = ds4_iq_bs(j.type);
+ const uint nsb = uint(j.n_ff) / 32u;
+ const size_t rb = (size_t) (j.n_ff / 256) * (size_t) bs;
+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 4) {
+ const int ne = min(4, m1 - mm);
+ int es[4];
+ for (int i = 0; i < 4; ++i)
+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ float ad[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ for (uint kb = kl; kb < nsb; kb += 16u) {
+ const Ds4IqSub sd = ds4_iq_sub(w, bs, int(kb), j.type);
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const device int* hp = (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff +
+ (size_t) kb * 32u);
+ ad[i] += j.hd[(size_t) es[i] * nsb + kb] * ds4_iq_dot(sd, hp);
+ }
+ }
+ }
+ for (int i = 0; i < 4; ++i) {
+ if (i < ne) {
+ const float v = ds4_iq_lsum<2>(ad[i]);
+ if (kl == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;
+ }
+ }
+ }
+ }
+
+ // ---------------------------------------------------------------- ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel
+ //
+ // The MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down / win_experts): the same frame as ds4_iq_gu / ds4_iq_dn
+ // above - a threadgroup of 256 is 8 warps, a warp a row of the gate and of the up (down: a row of the down), a lane
+ // strides the 32-value blocks of the row, and the group's entries go sixteen at a pass so that a block the weights decode
+ // once serves them all. A block is 17 bytes (kMxBytes): the E8M0 scale byte, then 16 bytes whose low nibble is value j and
+ // whose high nibble is value j + 16, each an index into the codebook {0, 1, 2, 3, 4, 6, 8, 12} and its negatives - the same
+ // decode as ds4_init.metal's ds4_mx_block / ds4_e8m0 (ds4_gguf.cu:143 / :154), kept as int8 codes here because the
+ // activations are the quantiser's int8, so the dot is integer (ds4_gguf.cu:225 mx_dot) and the two scales fold in at the
+ // end.
+ //
+ // The group's blob and its entries are read as ds4_iq_group above reads them (ds4_gguf.cu:429 mx_group): a plan group
+ // (grp_ptr null) is one expert whose entry is the group's own index and whose activation row is 0, a window group's
+ // entries come from grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.
+ struct Ds4MxJob {
+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)
+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob
+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)
+ device const int* grp_ent; // the entries (token * 8 + pick)
+ device const int* n_groups;
+ device const char* xq; // gate / up's int8 activations (n_embd an entry)
+ device const float* xd; // their scales (n_embd / 32 a row)
+ device const char* hq; // down's int8 input (n_ff an entry)
+ device const float* hd; // its scales
+ device float* h; // [entries][n_ff]: the clamped swiglu
+ device float* part; // [entries][n_embd]: each entry's down output
+ long tr0, tr1, tr2; // the roles' offsets in a blob
+ int n_embd, n_ff;
+ float limit; // swiglu_limit (0: none)
+ };
+
+ // the codebook of a nibble, in index order (ds4_init.metal's lut_init: {0,1,2,3,4,6,8,12}, then its negatives)
+ constant char ds4_mx_code[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
+
+ // 2^(e - 128) (ds4_init.metal:489 ds4_e8m0, ggml's GGML_E8M0_TO_FP32_HALF)
+ static inline float ds4_mx_e8m0(uint e) {
+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));
+ }
+
+ // a block's 16 code bytes as 8 int8 words: word j the low nibbles of bytes 4 j .. (values 4 j ..), word 4 + j their high
+ // nibbles (values 16 + 4 j ..) - the pairs ds4_gguf.cu:225 mx_dot reads xp[j] and xp[j + 4] with
+ static inline void ds4_mx_words(device const uchar* p, thread int* q) {
+ for (uint j = 0; j < 4u; ++j) {
+ uint lo = 0u, hi = 0u;
+ for (uint b = 0; b < 4u; ++b) {
+ const uint v = uint(p[1u + 4u * j + b]);
+ lo |= uint(uchar(ds4_mx_code[v & 15u])) << (8u * b);
+ hi |= uint(uchar(ds4_mx_code[v >> 4u])) << (8u * b);
+ }
+ q[j] = int(lo);
+ q[4u + j] = int(hi);
+ }
+ }
+
+ // a block's 32 int8 codes against its 32 int8 activations (8 packed words, 4 values a word)
+ static inline int ds4_mx_dot(thread const int* q, device const int* xp) {
+ int s = 0;
+ for (uint t = 0; t < 8u; ++t) s += ds4_iq_dot4(q[t], xp[t]);
+ return s;
+ }
+
+ // the group's blob and its entry range (ds4_iq_group's MX form, ds4_gguf.cu:429 mx_group); false: nothing to do
+ static inline bool ds4_mx_group(constant Ds4MxJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {
+ if (j.grp_ptr == nullptr) {
+ const ulong p = j.plan_ptr[g];
+ if (p == 0ul) return false;
+ blob = p;
+ m0 = int(g);
+ m1 = m0 + 1;
+ return true;
+ }
+ if (int(g) >= *j.n_groups) return false;
+ blob = j.grp_ptr[g];
+ m0 = j.grp_start[g];
+ m1 = j.grp_start[g + 1];
+ return true;
+ }
+
+ // mx_gate_up_kernel<NE> (a warp a row of the gate and of the up, a lane a block a step): h = the clamped swiglu
+ kernel void ds4_mx_gu(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_ff) return;
+ const uint nblk = uint(j.n_embd) / 32u;
+ const size_t rb = (size_t) nblk * 17u;
+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;
+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 16) {
+ const int ne = min(16, m1 - mm);
+ int es[16], xr[16];
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ es[i] = e;
+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);
+ }
+ }
+ float ag[16], au[16];
+ for (int i = 0; i < 16; ++i) {
+ ag[i] = 0.0f;
+ au[i] = 0.0f;
+ }
+ for (uint b = uint(lane); b < nblk; b += 32u) {
+ const device uchar* pg = g + (size_t) b * 17u;
+ const device uchar* pu = u + (size_t) b * 17u;
+ int wg[8], wu[8];
+ ds4_mx_words(pg, wg);
+ ds4_mx_words(pu, wu);
+ const float dg = ds4_mx_e8m0(uint(pg[0])), du = ds4_mx_e8m0(uint(pu[0]));
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const device int* xp =
+ (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd + (size_t) b * 32u);
+ const float dx = j.xd[(size_t) xr[i] * nblk + b];
+ ag[i] = fma(dg * dx, float(ds4_mx_dot(wg, xp)), ag[i]);
+ au[i] = fma(du * dx, float(ds4_mx_dot(wu, xp)), au[i]);
+ }
+ }
+ }
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);
+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);
+ }
+ }
+ }
+ }
+
+ // mx_down_kernel<NE> (a warp a row of the down, a lane a block a step): part = the down against the int8 h
+ kernel void ds4_mx_dn(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ ulong blob;
+ int m0, m1;
+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;
+ const int row = int(tgp.x) * 8 + int(tid >> 5);
+ if (row >= j.n_embd) return;
+ const uint nblk = uint(j.n_ff) / 32u;
+ const size_t rb = (size_t) nblk * 17u;
+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;
+ for (int mm = m0; mm < m1; mm += 16) {
+ const int ne = min(16, m1 - mm);
+ int es[16];
+ for (int i = 0; i < 16; ++i)
+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];
+ float ad[16];
+ for (int i = 0; i < 16; ++i) ad[i] = 0.0f;
+ for (uint b = uint(lane); b < nblk; b += 32u) {
+ const device uchar* pw = w + (size_t) b * 17u;
+ int ww[8];
+ ds4_mx_words(pw, ww);
+ const float dd = ds4_mx_e8m0(uint(pw[0]));
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const device int* hp =
+ (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff + (size_t) b * 32u);
+ ad[i] = fma(dd * j.hd[(size_t) es[i] * nblk + b], float(ds4_mx_dot(ww, hp)), ad[i]);
+ }
+ }
+ }
+ for (int i = 0; i < 16; ++i) {
+ if (i < ne) {
+ const float v = ds4_iq_lsum<1>(ad[i]);
+ if (lane == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;
+ }
+ }
+ }
+ }
+
+ // ---------------------------------------------------------------------------------------------------------------------
+ // One-activation matvecs (the decode step): every lane of the row's simdgroup works, on a 32-value sub-block (IQ2_XXS) or
+ // a 64-value pair (Q5_K), where the 16-column kernels above give a whole 256-value block to a lane (a k = 1280 row used 5
+ // lanes of 32) and read it a byte at a time. Same results up to the order of the float sums.
+
+ kernel void ds4_q5k_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint j = lane & 3u; // this lane's pair of sub-blocks: 2j and 2j+1 (one 32-byte run of qs, both nibbles)
+ const uint bsel = lane >> 2; // eight blocks a pass
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 8u) {
+ const device uchar* p = w + (size_t) blk * 176u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc0, mm0, sc1, mm1;
+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);
+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);
+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);
+ const device uint4* qh4 = (const device uint4*) (p + 16);
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 64u * j);
+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];
+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3]; // sub-block 2j: xa xb; 2j+1: xc xd4
+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};
+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};
+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};
+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ const uint q0 = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);
+ const uint q1 = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);
+ for (uint i = 0; i < 4u; ++i) {
+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));
+ sq0 += int((q0 >> (8u * i)) & 0xffu) * x0;
+ sq1 += int((q1 >> (8u * i)) & 0xffu) * x1;
+ sx0 += x0;
+ sx1 += x1;
+ }
+ }
+ acc = fma(b.xd[blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc);
+ acc = fma(b.xd[blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+
+ kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u; // this lane's 32-value sub-block
+ const uint bsel = lane >> 3; // four blocks a pass
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 66u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib); // (2-byte aligned: four 16-bit loads)
+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);
+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int sum = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];
+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ }
+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+
+ // The other low-bit expert types, one activation (see ds4_iq2xxs_mv1): a lane a 32-value sub-block, four blocks a pass.
+
+ kernel void ds4_iq2xs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 74u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);
+ const uint sc = uint(p[66u + ib]);
+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[l]);
+ const ulong g = iq2xs_grid[q & 511u];
+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ int sum = 0;
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ if (l < 2u) s0 += sum; else s1 += sum;
+ }
+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+
+ kernel void ds4_iq2s_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 82u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u + 4u * ib;
+ const device uchar* sgb = p + 34u + 4u * ib;
+ const uint qh = uint(p[66u + ib]);
+ const uint sc = uint(p[74u + ib]);
+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);
+ const ulong g = iq2s_grid[idx];
+ const uint sg = uint(sgb[l]);
+ int sum = 0;
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ if (l < 2u) s0 += sum; else s1 += sum;
+ }
+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+
+ kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 98u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u;
+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);
+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);
+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);
+ const float db = d * (0.5f + float(aux >> 28)) * 0.5f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int sum = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];
+ const uint xl = xw[2u * l], xh = xw[2u * l + 1u];
+ for (uint i = 0; i < 4u; ++i) {
+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);
+ if ((sgn & (1u << i)) != 0u) v0 = -v0;
+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;
+ sum += v0 * int(char(xl >> (8u * i))) + v1 * int(char(xh >> (8u * i)));
+ }
+ }
+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+
+ // ---------------------------------------------------------------------------------------------------------------------
+ // Matvecs for NC activations at once (1, 2, 4, 8): the prompt windows' and the drafter's verify windows' columns. A lane takes a
+ // 32-value sub-block (a 64-value pair for Q5_K) of the row, decodes its weights ONCE and dots them with each of the NC columns'
+ // int8 activations, so the weight bytes (the cost) are read once for the window. ncol <= NC (the rest of the accumulators idle).
+
+ template <uint NC>
+ static inline void ds4_mvn_store(const device Ds4MvJob& b, uint row, ushort lane, thread float (&acc)[NC]) {
+ for (uint c = 0; c < NC; ++c) {
+ const float v = simd_sum(acc[c]);
+ if (lane == 0u && int(c) < b.ncol) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+ }
+ }
+
+ template <uint NC>
+ kernel void ds4_q5k_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint j = lane & 3u;
+ const uint bsel = lane >> 2;
+ float acc[NC];
+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 8u) {
+ const device uchar* p = w + (size_t) blk * 176u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc0, mm0, sc1, mm1;
+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);
+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);
+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);
+ const device uint4* qh4 = (const device uint4*) (p + 16);
+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];
+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};
+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};
+ uint q0w[8], q1w[8];
+ for (uint t = 0; t < 8u; ++t) {
+ q0w[t] = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);
+ q1w[t] = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);
+ }
+ for (uint c = 0; c < NC; ++c) {
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 64u * j);
+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3];
+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};
+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ for (uint i = 0; i < 4u; ++i) {
+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));
+ sq0 += int((q0w[t] >> (8u * i)) & 0xffu) * x0;
+ sq1 += int((q1w[t] >> (8u * i)) & 0xffu) * x1;
+ sx0 += x0;
+ sx1 += x1;
+ }
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc[c]);
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc[c]);
+ }
+ }
+ ds4_mvn_store<NC>(b, row, lane, acc);
+ }
+
+ // the grid-coded types: decode the sub-block's 32 weights (signs applied) into w[32] ...
+ static inline void ds4_w32_iq2xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib);
+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);
+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);
+ db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];
+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+ }
+ static inline void ds4_w32_iq2xs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);
+ const uint sc = uint(p[66u + ib]);
+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[l]);
+ const ulong g = iq2xs_grid[q & 511u];
+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+ }
+ static inline void ds4_w32_iq2s(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u + 4u * ib;
+ const device uchar* sgb = p + 34u + 4u * ib;
+ const uint qh = uint(p[66u + ib]);
+ const uint sc = uint(p[74u + ib]);
+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;
+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);
+ const ulong g = iq2s_grid[idx];
+ const uint sg = uint(sgb[l]);
+ for (uint i = 0; i < 8u; ++i) {
+ const int code = int((g >> (8u * i)) & 0xffu);
+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;
+ }
+ }
+ }
+ static inline void ds4_w32_iq3xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u;
+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);
+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);
+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);
+ db = d * (0.5f + float(aux >> 28)) * 0.5f;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];
+ for (uint i = 0; i < 4u; ++i) {
+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);
+ if ((sgn & (1u << i)) != 0u) v0 = -v0;
+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;
+ w[l * 8u + i] = v0;
+ w[l * 8u + 4u + i] = v1;
+ }
+ }
+ }
+
++
++// IQ1_M (the 2.00 bpw pack's 1.75-bit type): the block has no d field - its F16 scale is the top nibbles of its four
++// u16 scales - and each 32-value sub-block splits into two 16-value halves with their own 3-bit scale (ggml's
++// dequantize_row_iq1_m: dl1 for the first two 8-value groups, dl2 for the last two, and a +-0.125 delta per group).
++// The weights are the grid's int8 codes plus that delta; the integer form here (8*code +- 1, scale / 8) is the same
++// number with no rounding difference, and keeps the dot in integers like the other types.
++static inline ushort ds4_iq1m_scale(const device uchar* p) {
++ const uint s0 = uint(p[48]) | (uint(p[49]) << 8), s1 = uint(p[50]) | (uint(p[51]) << 8);
++ const uint s2 = uint(p[52]) | (uint(p[53]) << 8), s3 = uint(p[54]) | (uint(p[55]) << 8);
++ return ushort((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u));
++}
++static inline void ds4_w32_iq1m(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {
++ const float d = float(as_type<half>(ds4_iq1m_scale(p)));
++ const ushort sc = ((const device ushort*) (p + 48))[ib >> 1];
++ db0 = d * float(2u * ((sc >> (6u * (ib & 1u))) & 7u) + 1u) * 0.125f;
++ db1 = d * float(2u * ((sc >> (6u * (ib & 1u) + 3u)) & 7u) + 1u) * 0.125f;
++ const uint qh0 = p[32u + 2u * ib], qh1 = p[33u + 2u * ib];
++ for (uint l = 0; l < 4u; ++l) {
++ const uint qhl = l < 2u ? qh0 : qh1;
++ const uint idx = uint(p[4u * ib + l]) | ((l & 1u) ? ((qhl << 4u) & 0x700u) : ((qhl << 8u) & 0x700u));
++ const bool neg = (l & 1u) ? ((qhl & 0x80u) != 0u) : ((qhl & 0x08u) != 0u);
++ const ulong g = iq1s_grid[idx];
++ for (uint j = 0; j < 8u; ++j) {
++ const uint code = uint((g >> (8u * j)) & 0xffu);
++ const int c8 = (code & 0x80u) != 0u ? int(code) - 256 : int(code); // -1 / 0 / +1
++ w[l * 8u + j] = 8 * c8 + (neg ? -1 : 1);
++ }
++ }
++}
++
+ // ... and dot them with each column (the sub-block's four 8-value groups: l < 2 use db0, l >= 2 db1)
+ template <uint NC, uint TYPE>
+ kernel void ds4_iqg_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;
- constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : 82u));
++ constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : (TYPE == 29 ? 56u : 82u)));
+ const device uchar* w0p = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc[NC];
+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w0p + (size_t) blk * bs;
+ int w[32];
+ float db0, db1;
+ if (TYPE == 16) { ds4_w32_iq2xxs(p, ib, w, db0); db1 = db0; }
+ else if (TYPE == 17) ds4_w32_iq2xs(p, ib, w, db0, db1);
+ else if (TYPE == 22) ds4_w32_iq2s(p, ib, w, db0, db1);
++ else if (TYPE == 29) ds4_w32_iq1m(p, ib, w, db0, db1);
+ else { ds4_w32_iq3xxs(p, ib, w, db0); db1 = db0; }
+ for (uint c = 0; c < NC; ++c) {
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ int part = 0;
+ for (uint i = 0; i < 4u; ++i) part += w[t * 4u + i] * int(char(xw[t] >> (8u * i)));
+ if (t < 4u) s0 += part; else s1 += part;
+ }
+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc[c]);
+ }
+ }
+ ds4_mvn_store<NC>(b, row, lane, acc);
+ }
+
+ #define DS4_MVN_INST(NC) \
+ template [[host_name("ds4_q5k_mvn" #NC)]] kernel void ds4_q5k_mvn<NC>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 16>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2xs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 17>(device const uchar*, uint2, uint, ushort); \
+ template [[host_name("ds4_iq2s_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 22>(device const uchar*, uint2, uint, ushort); \
- template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort);
++ template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort); \
++ template [[host_name("ds4_iq1m_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 29>(device const uchar*, uint2, uint, ushort);
+ DS4_MVN_INST(1)
+ DS4_MVN_INST(2)
+ DS4_MVN_INST(4)
+ DS4_MVN_INST(8)
diff --cc metal/src/engine/msl/ds4_mm.metal
index 000000000,61ebb303d..dfa7d11ec
mode 000000,100644..100644
--- a/metal/src/engine/msl/ds4_mm.metal
+++ b/metal/src/engine/msl/ds4_mm.metal
@@@ -1,0 -1,137 +1,138 @@@
+ // metal/src/engine/msl/ds4_mm.metal - the DeepSeek prompt chunk's expert GEMMs: moe_mm.metal's mm_gu3 / mm_down3 (matmul2d on dequantized
+ // weight tiles) over the engine's own expert slots and its UNPADDED sort order. Compiled after quant/units/mma/moe_mm.metal.
+ // tiles[i] = (slot, first row, valid rows): a tile is TM rows of one expert; rows past `valid` (the next expert's) are computed and
+ // dropped (the operand rows exist: the scratch carries TM rows of slack), the swiglu is DeepSeek's clamped one.
+
+ struct DsMmArgs {
+ ulong gate_off;
+ ulong up_off;
+ ulong down_off;
+ ulong stride; // bytes between slots
+ uint gu_row; // bytes per gate/up row (n_embd values)
+ uint d_row; // bytes per down row (n_ff values)
+ uint n_embd;
+ uint n_ff;
+ float limit;
+ uint pad0;
+ };
+
+ // xs[r][c] = half(x[idx[r]][c]) for r < rows
+ kernel void ds4_mm_gather(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]], device const int* idx [[buffer(2)]],
+ constant uint& n_embd [[buffer(3)]], device half* xs [[buffer(4)]],
+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
+ const uint c = tgp.x * 256u + tid;
+ if (c >= n_embd) return;
+ xs[(ulong) tgp.y * n_embd + c] = half(x[(ulong) idx[tgp.y] * ldx + c]);
+ }
+
+ template <int F, int TM, int TN, int TK, int SG>
+ kernel void ds4_mm_gu(device const uchar* W [[buffer(0)]], device const half* XS [[buffer(1)]], device half* HS [[buffer(2)]],
+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {
+ if (tg.y >= n_tiles) return;
+ threadgroup float ot[SG][TM * TN];
+ threadgroup uint tab[fmt<F>::TAB];
+ load_tab<F>(tab, tid, SG * 32);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int4 tl = tiles[tg.y];
+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;
+ device const uchar* E = W + ulong(e) * a.stride;
+ device const uchar* g0 = E + a.gate_off + ulong(n0) * a.gu_row;
+ device const uchar* u0 = E + a.up_off + ulong(n0) * a.gu_row;
+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);
+ matmul2d<desc, execution_simdgroup> op;
+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tX((device half*)XS, dextents<int32_t, 2>(a.n_embd, 1 << 30));
+ auto rg = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto ru = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto tA = tX.template slice<TK, TM>(0, row0);
+ auto accg = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();
+ auto accu = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();
+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)
+ if (accg.is_valid_element(i)) { accg[i] = 0.0f; accu[i] = 0.0f; }
+ for (int k0 = 0; k0 < int(a.n_embd); k0 += TK) {
+ const uint16_t cap = rg.get_capacity(), half_cap = cap / 2;
+ #pragma clang loop unroll(full)
+ for (uint16_t i = 0; i < half_cap; i += 4) {
+ const auto ix = rg.get_multidimensional_index(i);
+ const int k = k0 + ix[0], n = ix[1];
+ device const uchar* rgw = g0 + ulong(n) * a.gu_row;
+ device const uchar* ruw = u0 + ulong(n) * a.gu_row;
+ half4 wg, wu, wg2, wu2;
+ W8<F>::run(rgw, k, tab, wg, wg2);
+ W8<F>::run(ruw, k, tab, wu, wu2);
+ rg[i] = wg.x; rg[i + 1] = wg.y; rg[i + 2] = wg.z; rg[i + 3] = wg.w;
+ ru[i] = wu.x; ru[i + 1] = wu.y; ru[i + 2] = wu.z; ru[i + 3] = wu.w;
+ const uint16_t j = i + half_cap;
+ rg[j] = wg2.x; rg[j + 1] = wg2.y; rg[j + 2] = wg2.z; rg[j + 3] = wg2.w;
+ ru[j] = wu2.x; ru[j + 1] = wu2.y; ru[j + 2] = wu2.z; ru[j + 3] = wu2.w;
+ }
+ tA = tX.template slice<TK, TM>(k0, row0);
+ op.run(tA, rg, accg);
+ op.run(tA, ru, accu);
+ }
+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)
+ if (accg.is_valid_element(i)) {
+ const float g = fmin(accg[i], a.limit), u = fmin(fmax(accu[i], -a.limit), a.limit);
+ accg[i] = g / (1.0f + exp(-g)) * u;
+ }
+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));
+ accg.store(tO);
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ for (int it = lane; it < TM * TN; it += 32) {
+ const int m = it / TN, f = it - m * TN;
+ if (m < valid) HS[ulong(row0 + m) * a.n_ff + n0 + f] = half(ot[sg][it]);
+ }
+ }
+
+ template <int F, int TM, int TN, int TK, int SG>
+ kernel void ds4_mm_dn(device const uchar* W [[buffer(0)]], device half* YEH [[buffer(1)]], device const half* HS [[buffer(2)]],
+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],
+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {
+ if (tg.y >= n_tiles || a.pad0 == 3u) return;
+ threadgroup float ot[SG][TM * TN];
+ threadgroup uint tab[fmt<F>::TAB];
+ load_tab<F>(tab, tid, SG * 32);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const int4 tl = tiles[tg.y];
+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;
+ device const uchar* w0 = W + ulong(e) * a.stride + a.down_off + ulong(n0) * a.d_row;
+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);
+ matmul2d<desc, execution_simdgroup> op;
+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tH((device half*)HS, dextents<int32_t, 2>(a.n_ff, 1 << 30));
+ auto rw = op.template get_right_input_cooperative_tensor<half, half, float>();
+ auto tA = tH.template slice<TK, TM>(0, row0);
+ auto acc = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rw)>, float>();
+ for (uint16_t i = 0; i < acc.get_capacity(); ++i)
+ if (acc.is_valid_element(i)) acc[i] = 0.0f;
+ for (int k0 = 0; k0 < int(a.n_ff); k0 += TK) {
+ const uint16_t half_cap = rw.get_capacity() / 2;
+ #pragma clang loop unroll(full)
+ for (uint16_t i = 0; i < half_cap; i += 4) {
+ const auto ix = rw.get_multidimensional_index(i);
+ half4 w, w2;
+ if (a.pad0 == 1u) { w = half4(0); w2 = half4(0); } else if (a.pad0 == 2u) { w = half4(1); w2 = half4(1); W8<F>::run(w0, 0, tab, w, w2); w = half4(0); } else
+ W8<F>::run(w0 + ulong(ix[1]) * a.d_row, k0 + ix[0], tab, w, w2);
+ rw[i] = w.x; rw[i + 1] = w.y; rw[i + 2] = w.z; rw[i + 3] = w.w;
+ const uint16_t j = i + half_cap;
+ rw[j] = w2.x; rw[j + 1] = w2.y; rw[j + 2] = w2.z; rw[j + 3] = w2.w;
+ }
+ tA = tH.template slice<TK, TM>(k0, row0);
+ op.run(tA, rw, acc);
+ }
+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));
+ acc.store(tO);
+ simdgroup_barrier(mem_flags::mem_threadgroup);
+ for (int it = lane; it < TM * TN; it += 32) {
+ const int m = it / TN, f = it - m * TN;
+ if (m < valid) ((device float*) YEH)[ulong(row0 + m) * a.n_embd + n0 + f] = ot[sg][it];
+ }
+ }
+
-#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 16, 32, 32, 4>) ds4_mm_gu<F, 16, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 16, 32, 32, 4>) ds4_mm_dn<F, 16, 32, 32, 4>;
++#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 32, 32, 32, 4>) ds4_mm_gu<F, 32, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 32, 32, 32, 4>) ds4_mm_dn<F, 32, 32, 32, 4>;
++DS4_MM_INST(T_IQ1_M, 29)
+ DS4_MM_INST(T_IQ2_XXS, 16)
+ DS4_MM_INST(T_IQ2_XS, 17)
+ DS4_MM_INST(T_IQ2_S, 22)
+ DS4_MM_INST(T_IQ3_XXS, 18)
diff --cc metal/src/mtl_rt.mm
index 0fb436222,4950477bd..9a29db4f1
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@@ -47,8 -67,222 +67,228 @@@ static inline id<MTLBuffer> BO(const Bu
static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }
static std::string g_device_name = "unknown GPU";
+
+ // ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,
+ // its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.
+ static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded
+ namespace {
+ struct Timeline {
+ bool on = false;
+ id<MTLCounterSet> tset = nil;
+ std::vector<id<MTLCounterSampleBuffer>> bufs; // a sample buffer holds only ~1500 encoders: they rotate
+ size_t per = 0, cap = 0, n = 0; // encoders per buffer / in all (the buffers' limit) / recorded
+ long after = 0; // DS4_TL_AFTER: record from the Nth dispatch on
+ std::vector<std::string> names; // per encoder
+ uint64_t cpu0 = 0, gpu0 = 0;
+ bool reg = false;
+ bool gate = true; // ds4_timeline_gate: record only while open
+ bool att = false; // the encoder just made carries a sample slot
+ } g_tl;
+
+ id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBuffer> cb, MTLDispatchType dt) {
+ g_tl.att = false;
+ if (!g_tl.on) return [cb computeCommandEncoderWithDispatchType:dt];
+ if (g_tl.tset == nil) {
+ for (id<MTLCounterSet> cs in dev.counterSets)
+ if ([cs.name isEqualToString:MTLCommonCounterSetTimestamp]) g_tl.tset = cs;
+ if (g_tl.tset == nil) {
+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no timestamp counter set\n");
+ g_tl.on = false;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ for (size_t c = 4096; c >= 64 && g_tl.per == 0; c /= 2) {
+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
+ d.counterSet = g_tl.tset;
+ d.storageMode = MTLStorageModeShared;
+ d.sampleCount = c * 2;
+ NSError* err = nil;
+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
+ if (b != nil) {
+ g_tl.per = c;
+ g_tl.bufs.push_back(b);
+ }
+ }
+ if (g_tl.per == 0) {
+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no counter sample buffer\n");
+ g_tl.on = false;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ g_tl.cap = g_tl.per * 120;
+ MTLTimestamp c0 = 0, g0 = 0;
+ [dev sampleTimestamps:&c0 gpuTimestamp:&g0];
+ g_tl.cpu0 = c0;
+ g_tl.gpu0 = g0;
+ g_tl.names.assign(g_tl.cap, std::string());
+ g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;
+ if (std::getenv("DS4_TL_DRAFT")) g_tl.gate = false;
+ }
+ if (!g_tl.gate || g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];
+ const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;
+ if (bi >= g_tl.bufs.size()) {
+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
+ d.counterSet = g_tl.tset;
+ d.storageMode = MTLStorageModeShared;
+ d.sampleCount = g_tl.per * 2;
+ NSError* err = nil;
+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
+ if (b == nil) {
+ g_tl.cap = g_tl.n;
+ return [cb computeCommandEncoderWithDispatchType:dt];
+ }
+ g_tl.bufs.push_back(b);
+ }
+ MTLComputePassDescriptor* pd = [MTLComputePassDescriptor computePassDescriptor];
+ pd.dispatchType = dt;
+ pd.sampleBufferAttachments[0].sampleBuffer = g_tl.bufs[bi];
+ pd.sampleBufferAttachments[0].startOfEncoderSampleIndex = 2 * si;
+ pd.sampleBufferAttachments[0].endOfEncoderSampleIndex = 2 * si + 1;
+ g_tl.att = true;
+ return [cb computeCommandEncoderWithDescriptor:pd];
+ }
+
+ void tl_report(id<MTLDevice> dev) {
+ if (!g_tl.on && g_tl.bufs.empty()) return;
+ const size_t n = std::min(g_tl.n, g_tl.cap);
+ if (n < 2) return;
+ std::vector<MTLCounterResultTimestamp> tsv(2 * n);
+ for (size_t b = 0; b * g_tl.per < n; ++b) {
+ const size_t cnt = std::min(g_tl.per, n - b * g_tl.per);
+ NSData* data = [g_tl.bufs[b] resolveCounterRange:NSMakeRange(0, 2 * cnt)];
+ if (data == nil) { std::fprintf(stderr, "mtl: DS4_TIMELINE: resolve failed\n"); return; }
+ std::memcpy(&tsv[2 * b * g_tl.per], [data bytes], 2 * cnt * sizeof(MTLCounterResultTimestamp));
+ }
+ const MTLCounterResultTimestamp* ts = tsv.data();
+ MTLTimestamp c1 = 0, g1 = 0;
+ [dev sampleTimestamps:&c1 gpuTimestamp:&g1];
+ mach_timebase_info_data_t tb;
+ mach_timebase_info(&tb);
+ const double cpu_ns_per_tick = (double) tb.numer / (double) tb.denom;
+ (void) cpu_ns_per_tick; // (sampleTimestamps' CPU value is in ns already)
+ const double ns_per_gpu = (g1 > g_tl.gpu0) ? (double) (c1 - g_tl.cpu0) / (double) (g1 - g_tl.gpu0) : 1.0;
+ std::fprintf(stderr, "timeline: cap %zu n %zu scale %.6f ns/tick first ts %llu %llu last %llu\n", g_tl.cap, g_tl.n, ns_per_gpu, (unsigned long long) ts[0].timestamp, (unsigned long long) ts[1].timestamp, (unsigned long long) ts[2 * n - 1].timestamp);
+ size_t from = std::getenv("DS4_TL_FROM") ? (size_t) std::atoll(std::getenv("DS4_TL_FROM")) : 0;
+ struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };
+ std::map<std::string, Acc> acc;
+ double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;
+ double prev_end = -1, ovl_us = 0;
+ size_t used = 0, n_ovl = 0;
+ for (size_t k = from; k < n; ++k) {
+ if (g_tl.names[k].empty()) continue;
+ const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3; // us
+ if (!(en >= st)) continue;
+ if (prev_end < 0) t_first = st;
+ Acc& a = acc[g_tl.names[k]];
+ a.busy += en - st;
+ ++a.n;
+ total_busy += en - st;
+ if (prev_end >= 0 && st < prev_end) { ++n_ovl; ovl_us += prev_end - st; }
+ if (prev_end >= 0 && st >= prev_end) {
+ const double gp = st - prev_end;
+ a.gap += gp;
+ total_gap += gp;
+ if (gp > 300.0) { a.gap_big += gp; ++a.n_big; total_big += gp; }
+ }
+ prev_end = en;
+ t_last = en;
+ ++used;
+ }
+ std::vector<std::pair<double, std::string>> v;
+ for (auto& kv : acc) v.emplace_back(kv.second.busy + kv.second.gap, kv.first);
+ std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });
+ std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,
+ (t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);
+ std::fprintf(stderr, "timeline: %zu dispatches started before the previous one ended (%.1f ms overlapped)\n", n_ovl, ovl_us / 1000.0);
+ std::fprintf(stderr, "timeline: kernel n busy ms us/disp gap ms gap/disp big-gap ms (n)\n");
+ for (size_t i = 0; i < v.size() && i < 40; ++i) {
+ const Acc& a = acc[v[i].second];
+ std::fprintf(stderr, "timeline: %-28s %6zu %9.1f %8.1f %8.1f %8.1f %10.1f (%zu)\n", v[i].second.c_str(), a.n, a.busy / 1000.0, a.busy / (double) a.n,
+ a.gap / 1000.0, a.gap / (double) a.n, a.gap_big / 1000.0, a.n_big);
+ }
+ }
+ } // namespace
+
+ extern "C" void ds4_timeline_gate(int open) { g_tl.gate = open != 0; }
+
+
+ // ---- ds4_busy_begin / ds4_busy_end: a one-threadgroup kernel on a second queue that spins on a shared flag while the host works
+ // (the CPU lane), so the GPU is not idle between two bursts (idle -> the next kernels run 2-3x slower); the host sets the flag the
+ // moment it is done and the kernel leaves within microseconds. DS4_BUSY=0 turns it off.
+ namespace {
+ struct Busy {
+ bool init = false, ok = false;
+ id<MTLCommandQueue> q = nil;
+ id<MTLComputePipelineState> ps = nil;
+ id<MTLBuffer> flag = nil;
+ id<MTLCommandBuffer> last = nil;
+ bool running = false;
+ };
+ Busy g_busy;
+ } // namespace
+
+ extern "C" void ds4_busy_begin() {
+ static const bool off = std::getenv("DS4_BUSY") && std::string(std::getenv("DS4_BUSY")) == "0";
+ if (off) return;
+ @autoreleasepool {
+ if (!g_busy.init) {
+ g_busy.init = true;
+ id<MTLDevice> dev = MTLCreateSystemDefaultDevice();
+ static const char* kSrc = R"MSL(
+ #include <metal_stdlib>
+ using namespace metal;
+ kernel void busy_spin(device volatile atomic_uint* flag [[buffer(0)]], constant uint& cap [[buffer(1)]], uint t [[thread_position_in_grid]]) {
+ uint x = t;
++ float f = (float) t * 1e-9f;
+ for (uint i = 0; i < cap; ++i) {
+ for (uint j = 0; j < 64; ++j) x = x * 1664525u + 1013904223u;
++ for (uint j = 0; j < 16; ++j) f = fma(f, 1.0000001f, 1e-7f); // (a real FP chain: keeps the ALU clock up)
++ x ^= as_type<uint>(f);
+ if (atomic_load_explicit(flag, memory_order_relaxed) != 0u) break;
+ }
+ if (x == 0xdeadbeefu) atomic_store_explicit(flag, 2u, memory_order_relaxed);
+ }
+ )MSL";
+ NSError* err = nil;
+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSrc] options:nil error:&err];
+ if (lib) g_busy.ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"busy_spin"] error:&err];
+ if (g_busy.ps) {
+ g_busy.q = [dev newCommandQueue];
+ g_busy.flag = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];
+ g_busy.ok = true;
+ }
+ }
+ if (!g_busy.ok || g_busy.running) return;
+ g_busy.running = true;
+ if (g_busy.last != nil) [g_busy.last waitUntilCompleted]; // (the previous spin has left)
+ *(volatile uint32_t*) [g_busy.flag contents] = 0;
+ uint32_t cap = std::getenv("DS4_BUSY_CAP") ? (uint32_t) std::atoi(std::getenv("DS4_BUSY_CAP")) : 20000u; // ~ tens of ms at most
++ // the spin's width: one 256-thread threadgroup leaves most of the GPU idle and the clock drops anyway; DS4_BUSY_W
++ // threadgroups (256 threads each) keep more of the card's ALUs fed, which is what holds the performance state.
++ static const uint32_t width = std::getenv("DS4_BUSY_W") ? (uint32_t) std::max(1, std::atoi(std::getenv("DS4_BUSY_W"))) : 1u;
+ id<MTLCommandBuffer> cb = [g_busy.q commandBuffer];
+ id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
+ [enc setComputePipelineState:g_busy.ps];
+ [enc setBuffer:g_busy.flag offset:0 atIndex:0];
+ [enc setBytes:&cap length:4 atIndex:1];
- [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
++ [enc dispatchThreads:MTLSizeMake(256 * width, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
+ [enc endEncoding];
+ [cb commit];
+ g_busy.last = cb;
+ }
+ }
+ extern "C" void ds4_busy_end() {
+ if (!g_busy.ok || g_busy.last == nil || !g_busy.running) return;
+ *(volatile uint32_t*) [g_busy.flag contents] = 1;
+ g_busy.running = false;
+ }
+
Context::Context() : impl_(new Impl) {
impl_->dev = MTLCreateSystemDefaultDevice();
+ if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {
+ g_tl.on = true;
+ g_tl.reg = true;
+ static id<MTLDevice> tl_dev = impl_->dev;
+ std::atexit([] { tl_report(tl_dev); });
+ }
if (!impl_->dev) {
std::fprintf(stderr, "mtl: no Metal device\n");
std::exit(2);
@@@ -249,9 -616,12 +622,17 @@@ void Context::free(Buffer& b)
b = Buffer{};
}
+ static std::mutex g_hist_mu;
+ static std::unordered_map<void*, std::string> g_pipe_names;
+ static std::unordered_map<std::string, uint64_t> g_kernel_counts;
++// DS4_ENC_TIMES: where a dispatch's encoding time goes - the pipeline state, the arguments, the dispatch call
++static std::atomic<uint64_t> g_t_pipe{0}, g_t_args{0}, g_t_disp{0};
static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
require_fit(d, "dispatch");
++ static const bool enc_times = std::getenv("DS4_ENC_TIMES") != nullptr;
++ const auto t_a = std::chrono::steady_clock::now();
[enc setComputePipelineState:P(d.p)];
++ const auto t_b = std::chrono::steady_clock::now();
for (size_t i = 0; i < d.args.size(); ++i) {
const Arg& a = d.args[i];
if (a.buf) {
@@@ -262,6 -634,6 +645,7 @@@
else
[enc setBytes:a.bytes.data() length:a.bytes.size() atIndex:i];
}
++ const auto t_c = std::chrono::steady_clock::now();
if (d.tg_mem) [enc setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];
if (d.indirect)
[enc dispatchThreadgroupsWithIndirectBuffer:B(*d.indirect)
@@@ -270,10 -642,187 +654,212 @@@
else
[enc dispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])
threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
++ if (enc_times) {
++ const auto t_d = std::chrono::steady_clock::now();
++ g_t_pipe.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_b - t_a).count(), std::memory_order_relaxed);
++ g_t_args.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_c - t_b).count(), std::memory_order_relaxed);
++ g_t_disp.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_d - t_c).count(), std::memory_order_relaxed);
++ }
+ }
+
+ static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)
+ static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }
+ static double g_gpu_ms = 0.0;
+ static std::atomic<uint64_t> g_gpu_busy_us{0}; // DS4_GPU_TOTAL: the GPU time of every committed command buffer
+ static void add_busy_handler(id<MTLCommandBuffer> cb) {
+ static const bool on = std::getenv("DS4_GPU_TOTAL") != nullptr;
+ if (on) [cb addCompletedHandler:^(id<MTLCommandBuffer> c) { const double d = c.GPUEndTime - c.GPUStartTime; if (d > 0) g_gpu_busy_us += (uint64_t) (d * 1e6); }];
+ } // GPU time of every committed command buffer, summed
+ static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatches
+ static std::mutex g_prof_mu;
+ static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)
+ static std::string g_prof_last;
+ static std::atomic<uint64_t> g_cb_count{0};
+ static bool g_count_reg = false;
+ static void ds4_report_counts() {
+ if (g_gpu_busy_us.load()) std::fprintf(stderr, "mtl: GPU busy (every command buffer) %.1f ms\n", g_gpu_busy_us.load() / 1000.0);
+ std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,
+ (unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());
+ std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);
++ if (g_t_pipe.load() || g_t_args.load() || g_t_disp.load())
++ std::fprintf(stderr, "mtl: of the encoding: pipeline %.1f ms, arguments %.1f ms, dispatch %.1f ms\n",
++ g_t_pipe.load() / 1e6, g_t_args.load() / 1e6, g_t_disp.load() / 1e6);
+ if (!g_prof.empty()) {
+ std::vector<std::pair<double, std::string>> pv;
+ double tot = 0;
+ for (auto& kv : g_prof) { pv.emplace_back(kv.second.first, kv.first); tot += kv.second.first; }
+ std::sort(pv.begin(), pv.end(), [](auto& a, auto& b) { return a.first > b.first; });
+ std::fprintf(stderr, "prof: kernel GPU ms n us/dispatch (total %.1f ms)\n", tot / 1000.0);
+ for (size_t i = 0; i < pv.size() && i < 30; ++i) {
+ auto& e = g_prof[pv[i].second];
+ std::fprintf(stderr, "prof: %-30s %8.1f %6llu %8.1f\n", pv[i].second.c_str(), e.first / 1000.0, (unsigned long long) e.second, e.first / (double) e.second);
+ }
+ }
+ std::vector<std::pair<uint64_t, std::string>> v;
+ { std::lock_guard<std::mutex> lk(g_hist_mu); for (auto& kv : g_kernel_counts) v.emplace_back(kv.second, kv.first); }
+ std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a.first > b.first; });
+ for (size_t i = 0; i < v.size() && i < 14; ++i)
+ std::fprintf(stderr, " %6llu %s\n", (unsigned long long) v[i].first, v[i].second.c_str());
+ }
+
+ extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }
+
+ extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));
+ void Context::split_runs() {
+ // (the streams' pending copies come first, as in a full flush: they are queued on another MTLCommandQueue and would
+ // otherwise not have run when the commands committed here read what they write)
+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
+ // Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each
+ // one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same
+ // buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+ if (impl_->cb_def == nil) return;
+ ++g_cb_count;
+ [impl_->enc_def endEncoding];
+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
+ add_busy_handler(impl_->cb_def);
+ [impl_->cb_def commit];
+ impl_->inflight = impl_->cb_def;
+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
+ impl_->cb_def = nil;
+ impl_->enc_def = nil;
+ impl_->def_n = 0;
+ }
+
+ // Waiting for a committed command buffer: spin on its status for a while (the thread wake of waitUntilCompleted costs
+ // ~1 ms a wait; the decode token waits ~100 times), then block. DS4_NO_SPIN=1: always block.
+ static void wait_spin(id<MTLCommandBuffer> cb) {
+ static const bool no_spin = std::getenv("DS4_NO_SPIN") != nullptr;
+ static const double budget = std::getenv("DS4_SPIN_US") ? std::atof(std::getenv("DS4_SPIN_US")) * 1e-6 : 20e-3;
+ if (!no_spin) {
+ const auto t0 = std::chrono::steady_clock::now();
+ while (cb.status < MTLCommandBufferStatusCompleted) {
+ if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() > budget) break;
+ }
+ }
+ if (cb.status < MTLCommandBufferStatusCompleted) [cb waitUntilCompleted];
+ }
+
+ void* Context::mark() {
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+ if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)
+ if (impl_->inflight == nil) return nullptr;
+ return (__bridge_retained void*) impl_->inflight;
+ }
+
+ void Context::wait_mark(void* token) {
+ id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;
+ const auto tw0 = std::chrono::steady_clock::now();
+ wait_spin(cb);
+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
+ ++g_wait_n;
+ if (cb.status != MTLCommandBufferStatusCompleted)
+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
+ if (cb.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: recent: %s\n", recent_str().c_str()), std::fprintf(stderr, "mtl: info: %s\n", [[[cb.error userInfo] description] UTF8String]);
+ }
+
+ void Context::flush_runs() {
+ if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);
++ static const bool cb_log = std::getenv("DS4_CB_LOG") != nullptr; // debugging: one line a command buffer (what it held)
++ if (cb_log && impl_->cb_def != nil) {
++ std::string first, last;
++ if (!impl_->def_names.empty()) {
++ size_t sp = impl_->def_names.find(' ');
++ first = impl_->def_names.substr(0, sp);
++ size_t e = impl_->def_names.find_last_not_of(' ');
++ size_t b = impl_->def_names.find_last_of(' ', e);
++ last = impl_->def_names.substr(b == std::string::npos ? 0 : b + 1, e - (b == std::string::npos ? 0 : b));
++ }
++ std::fprintf(stderr, "CB %d disp first=%-24s last=%s\n", impl_->def_n, first.c_str(), last.c_str());
++ }
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+ if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }
+ if (impl_->cb_def != nil) {
+ ++g_cb_count;
+ [impl_->enc_def endEncoding];
+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
+ add_busy_handler(impl_->cb_def);
+ [impl_->cb_def commit];
+ impl_->inflight = impl_->cb_def;
+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
+ impl_->cb_def = nil;
+ impl_->enc_def = nil;
+ impl_->def_n = 0;
+ }
+ if (impl_->inflight != nil) {
+ if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }
+ const auto tw0 = std::chrono::steady_clock::now();
+ wait_spin(impl_->inflight);
+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
+ ++g_wait_n;
+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;
+ if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }
+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);
+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: failed cb held: %s\n", impl_->inflight_names.c_str());
+ impl_->inflight = nil;
+ }
+ for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
+ impl_->graveyard.clear();
}
double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
@autoreleasepool {
+ if (ds.size() == 1 && reps == 1 && !barrier_between) {
+ { static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string(); // debugging: drop these kernels (timing only)
+ if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }
+ if (impl_->cb_def == nil || impl_->def_n >= 2000000) {
+ if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)
+ if (std::getenv("DS4_ERRINFO")) {
+ MTLCommandBufferDescriptor* cd = [MTLCommandBufferDescriptor new];
+ cd.retainedReferences = NO;
+ cd.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;
+ impl_->cb_def = [impl_->q commandBufferWithDescriptor:cd];
+ } else
+ impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];
- if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
++ // (the wait on our own event is only for a *different* queue: this queue already runs the previous buffer
++ // before this one, and the wait costs a full GPU round trip at every buffer - 107 buffers a decode token,
++ // 0.28 ms each. DS4_QEV_WAIT=1 restores it.)
++ static const bool qev_wait = std::getenv("DS4_NO_QEV_WAIT") == nullptr; // (measured: off is slower - keep it)
++ if (qev_wait && impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial);
+ }
+ const auto te0 = std::chrono::steady_clock::now();
+ encode(impl_->enc_def, ds[0]);
+ if (g_tl.on && g_tl.att && g_tl.n < g_tl.cap) { g_tl.names[g_tl.n++] = ds[0].p->name; g_tl.att = false; }
+ g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);
+ g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();
+ if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";
+ g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();
+ static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;
+ if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }
+ // Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one
+ // dispatch reads what the previous wrote: without an explicit barrier the reader saw the stale value (measured: the
+ // second matvec's activation quantiser read the previous layer's x).
+ static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;
+ if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];
+ static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr || g_tl.on; // a new encoder per dispatch (debugging, the timeline)
+ if (enc_each) {
+ [impl_->enc_def endEncoding];
+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, (std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial));
+ }
+ ++impl_->def_n;
+ ++g_disp_count;
+ static const bool prof = std::getenv("DS4_PROF") != nullptr; // every dispatch alone, its GPU time kept per kernel
+ if (prof) { g_prof_last = ds[0].p->name; if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak(); flush_runs(); return 0.0; }
+ static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order
+ static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"
+ const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;
+ if (each || listed) {
+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
+ flush_runs();
+ }
+ return 0.0;
+ }
+ flush_runs();
id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
id<MTLComputeCommandEncoder> enc =
[cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
diff --cc packaging/mac/build_dsv41_engine.sh
index 000000000,000000000..1415a2078
new file mode 100644
--- /dev/null
+++ b/packaging/mac/build_dsv41_engine.sh
@@@ -1,0 -1,0 +1,58 @@@
++#!/bin/bash
++# Build the DeepSeek-V4.1-Flash engine component for Apple Silicon (strata_ds4, backend metal).
++#
++# bash packaging/mac/build_dsv41_engine.sh [--version 0.2.7] [--build DIR] [--out DIR]
++#
++# Run it in the source tree on the Mac, after `ninja -C <build> strata_ds4` (the script builds it too when the
++# binary is missing). It stages the binary and the kernel sources the engine compiles at start-up, packages them with
++# packaging/package_engine.py and verifies every hash. It does NOT sign: the feed key stays offline, and a package
++# made here is unsigned and loads (see packaging/ENGINE-COMPONENT.md, Signature).
++#
++# Layout (strata_ds4 finds these beside its own executable, metal/src/resource_path.h):
++# <exe>/metal/kernels/*.metal the shared Qwen kernels the DeepSeek host also compiles
++# <exe>/metal/src/engine/msl/*.metal ds4_init, ds4_iq, ds4_mm, ds4_quant, mtp, mtp_chain, ple
++# <exe>/metal/ggml-common.h
++set -u
++ROOT=$(cd "$(dirname "$0")/../.." && pwd)
++VERSION=0.2.7
++BUILD="$ROOT/build-metal"
++OUT="$ROOT/pkg/engine"
++while [ $# -gt 0 ]; do
++ case "$1" in
++ --version) VERSION="$2"; shift 2;;
++ --build) BUILD="$2"; shift 2;;
++ --out) OUT="$2"; shift 2;;
++ *) echo "usage: $0 [--version V] [--build DIR] [--out DIR]"; exit 2;;
++ esac
++done
++NINJA=${NINJA:-$HOME/strata-mac/tools/bin/ninja}
++EXE="$BUILD/strata_ds4"
++[ -x "$EXE" ] || "$NINJA" -C "$BUILD" strata_ds4 || exit 1
++
++STAGE="$ROOT/pkg/dsv41-stage"
++rm -rf "$STAGE"
++mkdir -p "$STAGE/metal/kernels" "$STAGE/metal/src/engine/msl"
++cp "$EXE" "$STAGE/"; chmod +x "$STAGE/strata_ds4"
++cp "$ROOT"/metal/kernels/*.metal "$STAGE/metal/kernels/"
++for f in ds4_init ds4_iq ds4_mm ds4_quant mtp mtp_chain ple; do
++ cp "$ROOT/metal/src/engine/msl/$f.metal" "$STAGE/metal/src/engine/msl/" || exit 1
++done
++cp "$ROOT/third_party/ggml/ggml-common.h" "$STAGE/metal/"
++
++python3 "$ROOT/packaging/package_engine.py" --src "$STAGE" --backend metal --version "$VERSION" \
++ --platform macos-arm64 --arch apple-gpu --entrypoint strata_ds4 --include 'metal/**' --out "$OUT" || exit 1
++DIR="$OUT/$VERSION-macos-arm64-metal"
++python3 "$ROOT/packaging/package_engine.py" --verify "$DIR" || exit 1
++
++# self-containment: the package must run with the source tree's kernels hidden
++PACK=${PACK:-$HOME/strata-mac/pack-2.25-real}
++if [ ! -d "$PACK" ]; then echo "== no pack at $PACK: package built, smoke test skipped"; exit 0; fi
++echo "== self-containment test (source kernels hidden, run from /tmp)"
++mv "$ROOT/metal/kernels" "$ROOT/metal/kernels.hidden"
++mv "$ROOT/metal/src/engine/msl" "$ROOT/metal/src/engine/msl.hidden"
++( cd /tmp && "$DIR/strata_ds4" --pack "$PACK" --ids 0 --n 1 \
++ --expert-profile none --vision off --dspark off --cpu-threads 4 --threads off --tp 1 2>&1 | grep -a -E 'prompt:|decode:|cannot read|failed' | head -3 )
++rc=$?
++mv "$ROOT/metal/kernels.hidden" "$ROOT/metal/kernels"
++mv "$ROOT/metal/src/engine/msl.hidden" "$ROOT/metal/src/engine/msl"
++echo "== done: $DIR (rc=$rc)"
diff --cc src/dsv41/ds4_attn_split.cuh
index 959830b09,9def53b8c..7d9028dfd
--- a/src/dsv41/ds4_attn_split.cuh
+++ b/src/dsv41/ds4_attn_split.cuh
@@@ -11,6 -11,6 +11,7 @@@
// phase 2: block (head, query): merge the chunks and the sink.
// Plain C++ (no PTX), shuffles and shared memory only: the HIP port is a rename.
#pragma once
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_kernels.cuh"
#include "ds4_numerics.cuh"
#include <cuda_fp16.h>
diff --cc src/dsv41/ds4_gguf.cu
index 55231c8d3,ba4ed3e3d..fe29d2034
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@@ -1,9 -1,13 +1,14 @@@
--// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
++// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_gguf.cuh"
+ #if defined(STRATA_METAL)
+ #include <ds4_metal_host.h>
+ #endif
#include "ds4_iq.cuh"
+ #include "ds4_gguf_cpu.hpp"
#include "ds4_numerics.cuh"
#include <cuda_fp16.h>
@@@ -198,65 -202,12 +203,77 @@@ __global__ void __launch_bounds__(kThre
}
void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {
const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);
- q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);
+ #if defined(STRATA_METAL)
+ if (::strata::dsv41::metal_host::q8_quant_rows(x, ldx, rows, k, (int8_t*) q, d)) return;
+ #endif
+ strata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);
}
+// Several activations' rounding in one launch (a GEMV call's jobs: wo_a's eight groups, the shared expert's two, ...): item z is
+// q8_quant's (x, ldx, rows, k, q, d), the same per-element arithmetic, so the bytes are the single launches'. Identical items
+// (the same x, stride, rows and k: two matrices reading one input) are rounded once and share the result.
+constexpr int kQuantItems = x3::kMaxMvJobs;
+struct QuantItems {
+ const float* x[kQuantItems];
+ size_t ldx[kQuantItems];
+ int k[kQuantItems], rows[kQuantItems];
+ int8_t* q[kQuantItems];
+ float* d[kQuantItems];
+ int n;
+};
+__global__ void __launch_bounds__(kThreads) q8_quant_multi_kernel(QuantItems it) {
+ const int z = blockIdx.z;
+ if (z >= it.n) return;
+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;
+ const int k = it.k[z];
+ if (r >= it.rows[z] || b * 32 >= k) return;
+ const float v = it.x[z][(size_t) r * it.ldx[z] + (size_t) b * 32 + lane];
+ float am = fabsf(v);
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));
+ const float sc = am / 127.0f;
+ it.q[z][(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);
+ if (lane == 0) it.d[z][(size_t) r * (k / 32) + b] = sc;
+}
+// adds an item unless an identical one is queued; returns the item whose result to read (its q and d)
+int quant_add(QuantItems& it, const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {
+ static const bool nodedup = std::getenv("DS4_QUANT_NODEDUP") != nullptr;
+ for (int i = 0; i < it.n && !nodedup; ++i)
+ if (it.x[i] == x && it.ldx[i] == ldx && it.rows[i] == rows && it.k[i] == k) return i;
+ const int i = it.n++;
+ it.x[i] = x;
+ it.ldx[i] = ldx;
+ it.rows[i] = rows;
+ it.k[i] = k;
+ it.q[i] = q;
+ it.d[i] = d;
+ return i;
+}
+void quant_flush(const QuantItems& it, cudaStream_t s) {
+ if (it.n <= 0) return;
+ static const bool single = std::getenv("DS4_QUANT_SINGLE") != nullptr;
++#if defined(STRATA_METAL)
++ if (!single && std::getenv("DS4_NO_Q8_BATCH") == nullptr) { // the call's roundings in one dispatch (a dispatch a job was 45% of the decode token's)
++ struct Spec { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } sp[kQuantItems];
++ for (int i = 0; i < it.n; ++i) sp[i] = Spec{it.x[i], (uint64_t) it.ldx[i], (uint32_t) it.rows[i], (uint32_t) it.k[i], it.q[i], it.d[i]};
++ if (::strata::dsv41::metal_host::q8_quant_multi(sp, it.n)) return;
++ }
++ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);
++ return;
++#endif
+ if (single) {
+ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);
+ return;
+ }
+ int kmax = 0, rmax = 0;
+ for (int i = 0; i < it.n; ++i) {
+ kmax = std::max(kmax, it.k[i]);
+ rmax = std::max(rmax, it.rows[i]);
+ }
+ const dim3 grid((unsigned) ((kmax / 32 + kWarps - 1) / kWarps), (unsigned) rmax, (unsigned) it.n);
- q8_quant_multi_kernel<<<grid, kThreads, 0, s>>>(it);
++ strata_ds4_launch(q8_quant_multi_kernel, grid, kThreads, 0, s, it);
+}
+
// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),
// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4
__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {
@@@ -1064,6 -1015,6 +1081,14 @@@ size_t type_row_bytes(int type, int64_
int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::known(type) ? iq::block_values(type) : 0; }
const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }
bool type_on_device(int type) { return type == kTypeMXFP4 || iq::supported(type); }
++bool type_on_chunk(int type) {
++#if defined(STRATA_METAL)
++ // ds4_mm.metal instantiates the tile GEMM for IQ2_XXS / IQ2_XS / IQ2_S only; IQ1_M (the 2.00 bpw pack's type) has
++ // the matvec kernel but no tile one, so a chunked prompt would stop in "kernel launch with no Metal dispatch".
++ if (type == iq::kIQ1_S || type == iq::kIQ4_NL) return false; // (IQ1_M: the tile kernels since DS4_MM_INST(29) and iq_bs)
++#endif
++ return type_on_device(type);
++}
namespace {
size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row
@@@ -1128,6 -1093,15 +1167,23 @@@ x3::Mat view(const x3::Mat& m, int c0,
bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
if (n <= 0 || n > x3::kMaxMvJobs) return false;
+ #if defined(STRATA_METAL)
- // the jobs' int8 activations are quantised in one dispatch before the matvecs (a dispatch a job was 45% of the decode token's)
- struct Q8Batch { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } q8_batch[x3::kMaxMvJobs];
- int nq8 = 0;
- const bool q8_deferred = std::getenv("DS4_NO_Q8_BATCH") == nullptr && std::getenv("DS4_DBG_Q8") == nullptr;
-#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) do { if (q8_deferred) q8_batch[nq8++] = Q8Batch{XF, (uint64_t) (LDX), (uint32_t) (ROWS), (uint32_t) (KK), Q, D}; else q8_quant(XF, LDX, ROWS, KK, Q, D, S); } while (0)
-#else
-#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) q8_quant(XF, LDX, ROWS, KK, Q, D, S)
++ if (std::getenv("DS4_MV_LOG")) { // how the engine batches its matvec jobs (one call = one quantiser dispatch)
++ static std::atomic<unsigned long long> calls{0}, jobsum{0};
++ static std::atomic<unsigned long long> hist[9];
++ static std::atomic<int> seen{0};
++ calls.fetch_add(1); jobsum.fetch_add((unsigned long long) n);
++ hist[n < 8 ? n : 8].fetch_add(1);
++ if (seen.fetch_add(1) % 256 == 255)
++ std::fprintf(stderr, "mvlog calls=%llu jobs=%llu avg=%.2f hist1..4=%llu,%llu,%llu,%llu 5+=%llu,%llu,%llu,%llu\n",
++ (unsigned long long) calls.load(), (unsigned long long) jobsum.load(),
++ (double) jobsum.load() / (double) calls.load(),
++ (unsigned long long) hist[1].load(), (unsigned long long) hist[2].load(),
++ (unsigned long long) hist[3].load(), (unsigned long long) hist[4].load(),
++ (unsigned long long) hist[5].load(), (unsigned long long) hist[6].load(),
++ (unsigned long long) hist[7].load(), (unsigned long long) hist[8].load());
++ }
+ #endif
BfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch each
int nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};
bool vec[3] = {true, true, true};
@@@ -1202,8 -1206,37 +1261,35 @@@
vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));
}
}
-#if defined(STRATA_METAL)
- if (nq8 > 0) ::strata::dsv41::metal_host::q8_quant_multi(q8_batch, nq8);
-#endif
+ quant_flush(QI, s);
for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);
+ #if defined(STRATA_METAL)
+ if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute
+ ds4_metal_flush();
+ const BfJob& b = J[2].j[0];
+ const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);
+ const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);
+ const float* xd = (const float*) strata::gpu::host_view(b.xd);
+ const float* y = (const float*) strata::gpu::host_view(b.y);
+ const float* xf = (const float*) strata::gpu::host_view(b.x);
+ for (int c = 0; c < std::min(b.ncol, 2); ++c)
+ for (int r = 0; r < 2; ++r) {
+ double acc = 0.0, accf = 0.0;
+ for (int kb = 0; kb < b.k / 32; ++kb) {
+ const uint8_t* p = w + (size_t) r * b.ld + (size_t) kb * 34;
+ uint16_t dh;
+ std::memcpy(&dh, p, 2);
+ float dw = __half2float(*(const __half*) &dh);
+ int sum = 0;
+ for (int j = 0; j < 32; ++j) sum += (int) (int8_t) p[2 + j] * (int) xq[(size_t) c * b.k + kb * 32 + j];
+ acc += (double) dw * xd[(size_t) c * (b.k / 32) + kb] * sum;
+ for (int j = 0; j < 32; ++j) accf += (double) dw * (int) (int8_t) p[2 + j] * xf[(size_t) c * b.xcs + kb * 32 + j];
+ }
+ std::fprintf(stderr, "DBG q8 job0 k=%d n=%d ncol=%d ld=%zu xcs=%zu c=%d r=%d recompute=%.5g exactf=%.5g kernel_y=%.5g xd0=%g xq0=%d\n", b.k, b.n, b.ncol,
+ b.ld, b.xcs, c, r, acc, accf, y[(size_t) c * b.ycs + r], xd[(size_t) c * (b.k / 32)], (int) xq[(size_t) c * b.k]);
+ }
+ }
+ #endif
if (nij > 0 && !iq::mv(IJ, nij, s)) {
fail("mv: a low-bit job not covered");
return false;
@@@ -1402,7 -1504,8 +1557,9 @@@ iq::SetArgs iq_set(const x3::SetArgs& s
q.n_embd = sa.n_embd;
q.n_ff = sa.n_ff;
q.limit = sa.limit;
+ q.xf16 = sa.x16;
+ q.hbounds = sa.hbounds;
+ q.total_rows = sa.total_rows;
return q;
}
} // namespace
diff --cc src/dsv41/ds4_gguf.cuh
index 05bb2696e,05bb2696e..4000093de
--- a/src/dsv41/ds4_gguf.cuh
+++ b/src/dsv41/ds4_gguf.cuh
@@@ -52,6 -52,6 +52,10 @@@ const char* type_name(int type)
/// Whether the device kernels take this type (MXFP4 is its own path; the i-quants and K-quants ds4_iq.cu instantiates).
/// A false here with a layout known to type_row_bytes() means the CPU lane reads it and it must not enter the VRAM tier.
bool type_on_device(int type);
++/// Whether the *chunk* path's expert tile GEMM (ds4_mm.metal on Metal) takes this type. IQ1_M has a matvec kernel on
++/// Metal but no tile kernel there, so a prompt in chunks would stream it to the GPU and stop; false here means the
++/// prompts must run in windows instead (ds4_model.cu switches, and says so).
++bool type_on_chunk(int type);
/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).
bool prepare(cudaStream_t s);
diff --cc src/dsv41/ds4_iq.cu
index da5e8e5d7,9d4965c44..186aedb9d
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@@ -2,9 -2,11 +2,13 @@@
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are
// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_iq.cuh"
+ #if defined(STRATA_METAL)
+ #include <ds4_metal_host.h>
+ #endif
#include "ds4_numerics.cuh"
+#include "ds4_experts_rule.hpp"
#include <cuda_fp16.h>
#if !defined(STRATA_USE_HIP)
@@@ -1116,74 -954,29 +1120,74 @@@ bool tc_ok() { // as ds4_gguf.cu's: s
default: return false; \
}
+// CUDA: a persistent grid (the resident blocks of the device, split over the groups), each block looping over row tiles with
+// its codebook loaded once; HIP keeps the full grid (one tile a block)
+template <auto Kernel>
+int resident_ctas() { // (a static a kernel: the occupancy differs by the codebook's shared memory)
- #if defined(STRATA_USE_HIP)
++#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)
+ return 1 << 30;
+#else
+ static std::atomic<int> cache[16];
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 16) return 1 << 30;
+ int v = cache[dev].load(std::memory_order_relaxed);
+ if (v == 0) {
+ int sms = 0, per = 0;
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
+ cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per, Kernel, kThreads, 0);
+ if (const char* e = std::getenv("DS4_IQ_MAX_SMS")) sms = std::min(sms, std::max(1, std::atoi(e))); // (benchmarking: a smaller card)
+ if (const char* e = std::getenv("DS4_IQ_CTA_PER_SM")) per = std::min(per, std::max(1, std::atoi(e)));
+ v = std::max(1, sms * std::max(per, 1));
+ cache[dev].store(v, std::memory_order_relaxed);
+ }
+ return v;
+#endif
+}
+// The grid: CUDA takes the device's resident blocks (at most one a work item) and every block loops over the items (group, row
+// tile) round the grid (group by group, so that the blocks work on one expert's rows together): whichever groups are active,
+// they share all the blocks, and a block loads its codebook once. HIP: a block an item (the full grid).
+template <int T>
+inline int items_grid(int items, int resident) { return std::max(1, std::min(items, resident)); }
template <int T, int NE>
-void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(gate_up_kernel<T, NE>, grid, kThreads, 0, s, a); }
+void launch_gu(const Args& a, int groups, cudaStream_t s) {
+ const int tiles = a.n_ff / (kWarps * kGuRW);
+ if constexpr (kPersist<T>)
- gate_up_kernel<T, NE><<<items_grid<T>(groups * tiles, resident_ctas<gate_up_kernel<T, NE, kGuRW>>()), kThreads, 0, s>>>(a, groups);
++ strata_ds4_launch(gate_up_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<gate_up_kernel<T, NE, kGuRW>>()), kThreads, 0, s, a, groups);
+ else
- gate_up_kernel<T, NE><<<dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s>>>(a, groups);
++ strata_ds4_launch(gate_up_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);
+}
template <int T, int NE>
-void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(down_kernel<T, NE>, grid, kThreads, 0, s, a); }
+void launch_dn(const Args& a, int groups, cudaStream_t s) {
+ const int tiles = a.n_embd / (kWarps * kDnRW);
+ if constexpr (kPersist<T>)
- down_kernel<T, NE><<<items_grid<T>(groups * tiles, resident_ctas<down_kernel<T, NE, kDnRW>>()), kThreads, 0, s>>>(a, groups);
++ strata_ds4_launch(down_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<down_kernel<T, NE, kDnRW>>()), kThreads, 0, s, a, groups);
+ else
- down_kernel<T, NE><<<dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s>>>(a, groups);
++ strata_ds4_launch(down_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);
+}
template <int T>
-void gu_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {
- if (ne <= 1) launch_gu<T, 1>(a, grid, s);
- else if (ne <= 4) launch_gu<T, 4>(a, grid, s);
- else if (ne <= 8) launch_gu<T, 8>(a, grid, s);
- else launch_gu<T, 16>(a, grid, s);
+void gu_ne(const Args& a, int groups, int ne, cudaStream_t s) {
+ if (ne <= 1) launch_gu<T, 1>(a, groups, s);
+ else if (ne <= 4) launch_gu<T, 4>(a, groups, s);
+ else if (ne <= 8) launch_gu<T, 8>(a, groups, s);
+ else launch_gu<T, 16>(a, groups, s);
}
template <int T>
-void dn_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {
- if (ne <= 1) launch_dn<T, 1>(a, grid, s);
- else if (ne <= 4) launch_dn<T, 4>(a, grid, s);
- else if (ne <= 8) launch_dn<T, 8>(a, grid, s);
- else launch_dn<T, 16>(a, grid, s);
+void dn_ne(const Args& a, int groups, int ne, cudaStream_t s) {
+ if (ne <= 1) launch_dn<T, 1>(a, groups, s);
+ else if (ne <= 4) launch_dn<T, 4>(a, groups, s);
+ else if (ne <= 8) launch_dn<T, 8>(a, groups, s);
+ else launch_dn<T, 16>(a, groups, s);
}
+#include "ds4_iq_hip.inc"
template <int T, bool GU>
void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {
+#if defined(STRATA_USE_HIP)
+ if (hipk::launch<T, GU>(a, s)) return; // (gfx12, STRATA_DS4_HIPK=1: the matrix-core tiles)
+#endif
#if !defined(STRATA_USE_HIP)
if (tc_ok()) {
- set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);
+ strata_ds4_launch(set_tc_kernel<T, GU>, grid, kThreads, 0, s, a);
return;
}
#endif
@@@ -1229,17 -1020,10 +1233,17 @@@ bool light_on() { // STRATA_DS4_NO_LI
}
template <int T, bool GU>
void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {
- const dim3 grid((unsigned) (((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps), (unsigned) a.n_exp);
- if (a.max_rows <= 4) strata_ds4_launch(light_kernel<T, 4, GU>, grid, kThreads, 0, s, a, b.q, b.d);
- else if (a.max_rows <= 8) strata_ds4_launch(light_kernel<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);
- else strata_ds4_launch(light_kernel<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);
+ const int tiles = ((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps;
+ if (a.max_rows <= 4) { // few rows: work items round a persistent grid (the codebook per block is then the cost)
+ if constexpr (kPersist<T>)
- light_kernel<T, 4, GU><<<items_grid<T>(a.n_exp * tiles, resident_ctas<light_kernel<T, 4, GU>>()), kThreads, 0, s>>>(a, b.q, b.d);
++ strata_ds4_launch(light_kernel<T, 4, GU>, items_grid<T>(a.n_exp * tiles, resident_ctas<light_kernel<T, 4, GU>>()), kThreads, 0, s, a, b.q, b.d);
+ else // (a small codebook: the release's kernel, which is the faster there)
- light_kernel_rows<T, 4, GU><<<dim3((unsigned) tiles, (unsigned) a.n_exp), kThreads, 0, s>>>(a, b.q, b.d);
++ strata_ds4_launch(light_kernel_rows<T, 4, GU>, dim3((unsigned) tiles, (unsigned) a.n_exp), kThreads, 0, s, a, b.q, b.d);
+ } else { // more rows: compute bound, the release's kernel and grid
+ const dim3 grid((unsigned) tiles, (unsigned) a.n_exp);
- if (a.max_rows <= 8) light_kernel_rows<T, 8, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);
- else light_kernel_rows<T, kLightRows, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);
++ if (a.max_rows <= 8) strata_ds4_launch(light_kernel_rows<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);
++ else strata_ds4_launch(light_kernel_rows<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);
+ }
}
bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {
const int K = gu ? a.n_embd : a.n_ff;
@@@ -1419,13 -1094,7 +1423,13 @@@ static bool dq_use_old_kernel()
bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {
if (raw == nullptr || k % 256 != 0 || n <= 0) return false;
const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);
-#define DS4_DQ(T) strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16)
+ const bool old = dq_use_old_kernel();
+#define DS4_DQ(T) do { \
+ if (old) \
- dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
++ strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16); \
+ else \
- dq_rows_kernel2<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \
++ strata_ds4_launch(dq_rows_kernel2<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16); \
+ } while (0)
switch (type) {
case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;
case kIQ2_XS: DS4_DQ(kIQ2_XS); break;
@@@ -1447,6 -1116,6 +1451,12 @@@
}
bool supported(int t) {
++#if defined(STRATA_METAL)
++ // The MSL kernels (metal/src/engine/msl/ds4_iq.metal) implement the 2.0-bit types plus IQ1_M; IQ1_S and IQ4_NL
++ // have CUDA kernels here but no Metal ones. Saying so makes those layers "cpu only" (ds4_model.cu, expert_layout),
++ // so their experts stay on the CPU lane instead of stopping in "kernel launch with no Metal dispatch".
++ if (t == kIQ1_S || t == kIQ4_NL) return false;
++#endif
// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)
switch (t) {
case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:
@@@ -1513,18 -1182,32 +1523,30 @@@ const char* name(int t)
}
bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ #if defined(STRATA_METAL)
+ // gate_up_kernel on the Metal dispatch (the prompt / window path's expert GEMM); the launch below is the fallback
+ if (::strata::dsv41::metal_host::iq_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
+ a.n_ff, a.limit, a.xq, a.xd, a.h, a.tg, groups))
+ return true;
+ #endif
if (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);
if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;
- const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);
-#define GU_(T) gu_ne<T>(a, grid, ne_max, s)
+#define GU_(T) gu_ne<T>(a, groups, ne_max, s)
DS4_IQ_SWITCH(a.tg, GU_)
#undef GU_
return true;
}
bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {
+ #if defined(STRATA_METAL)
+ // down_kernel on the Metal dispatch; the launch below is the fallback
+ if (::strata::dsv41::metal_host::iq_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,
+ a.n_ff, a.hq, a.hd, a.part, a.td, groups))
+ return true;
+ #endif
if (is_tq(a.td)) return tq::down(a, groups, ne_max, s);
if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;
- const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);
-#define DN_(T) dn_ne<T>(a, grid, ne_max, s)
+#define DN_(T) dn_ne<T>(a, groups, ne_max, s)
DS4_IQ_SWITCH(a.td, DN_)
#undef DN_
return true;
@@@ -1541,24 -1224,11 +1563,28 @@@ bool light_down(const SetArgs& a, cudaS
return light_set(a, false, s);
}
+bool hipk_force(int mode) {
+#if defined(STRATA_USE_HIP)
+ hipk::force_mode().store(mode, std::memory_order_relaxed);
+ return mode < 0 || hipk::ok_device() || mode == 0;
+#else
+ (void) mode;
+ return false;
+#endif
+}
+long long hipk_launches() {
+#if defined(STRATA_USE_HIP)
+ return hipk::launches().load(std::memory_order_relaxed);
+#else
+ return -1;
+#endif
+}
+
bool set_gate_up(const SetArgs& a, cudaStream_t s) {
+ #if defined(STRATA_METAL)
+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_gu(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xf, a.ldx, a.idx, a.h, a.ldo, a.tg, a.td))
+ return true;
+ #endif
if (is_tq(a.tg)) return tq::set_gate_up(a, s);
if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;
SetArgs t = a;
diff --cc src/dsv41/ds4_kernels.cu
index baf63dccd,f7edf5745..f6fdaccc8
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@@ -2,7 -2,10 +2,11 @@@
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary
// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_kernels.cuh"
+ #if defined(STRATA_METAL)
+ #include <ds4_metal_host.h>
+ #endif
#include "ds4_numerics.cuh"
#include "ds4_q6k.cuh"
#include "ds4_attn_split.cuh"
@@@ -2149,21 -1929,24 +2197,43 @@@ void finish(float* x, int n_vec, int le
}
void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {
- hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);
+ #if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::hc_mix(h, fn, hcdim, n_mix, T, eps, mixes)) return;
+ #endif
+ strata_ds4_launch(hc_mix_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes);
}
+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,
+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
+ float* comb, cudaStream_t s) {
- hc_mix_split_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,
++ strata_ds4_launch(hc_mix_split_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,
+ post, comb);
+}
void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb, cudaStream_t s) {
- hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);
+ #if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::hc_split(mixes, scale, base, hc, iters, hc_eps, T, pre, post, comb)) return;
+ #endif
+ strata_ds4_launch(hc_split_kernel, T, 32, 0, s, mixes, scale, base, hc, iters, hc_eps, pre, post, comb);
+ }
++void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,
++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
++ float* comb, cudaStream_t s) {
++#if defined(STRATA_METAL)
++ // one dispatch: the mix and the sinkhorn split of its result (bit-identical to the two, see ds4_init.metal)
++ if (strata::dsv41::metal_host::hc_mix_split(h, fn, hcdim, n_mix, T, eps, mixes, scale, base, hc, iters, hc_eps,
++ pre, post, comb))
++ return;
++#endif
++ hc_mix(h, fn, hcdim, n_mix, T, eps, mixes, s);
++ hc_split(mixes, scale, base, hc, iters, hc_eps, T, pre, post, comb, s);
+}
++
void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
cudaStream_t s) {
- hc_pre_norm_kernel<<<T, 1024, 0, s>>>(h, mix, w, dim, hc, eps, x);
+ #if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::hc_pre_norm(h, mix, w, dim, hc, T, eps, x)) return;
+ #endif
+ strata_ds4_launch(hc_pre_norm_kernel, T, 1024, 0, s, h, mix, w, dim, hc, eps, x);
}
void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,
float* out, cudaStream_t s) {
@@@ -2318,17 -2164,9 +2451,17 @@@ void sparse_attn(const float* q, const
heads, d, scale, out);
return;
}
+ // DS4_FAST_ATTN=1: the chunked prompt's matrix-core path (16 heads a block, a 16-key tile). The decode and the
+ // verify windows keep the scalar kernels: one query's heads are not the matrix's rows there.
+#if defined(STRATA_USE_HIP)
+ if (fast_attn_enabled() && a.gen != 0 && T >= kFaKeys && heads % kFaHeads == 0) {
- sparse_attn_wmma_kernel<<<dim3(heads / kFaHeads, T), kThreads, 0, s>>>(q, win, (const uint8_t*) ckv, sel, sel_stride,
++ strata_ds4_launch(sparse_attn_wmma_kernel, dim3(heads / kFaHeads, T), kThreads, 0, s, q, win, (const uint8_t*) ckv, sel, sel_stride,
+ a, sink, heads, scale, out);
+ return;
+ }
+#endif
// packed rows: as many heads a block as still leaves two blocks a multiprocessor (a decoded row serves them all)
- int G = heads_per_block;
- if (G != 1 && G != 2 && G != 4 && G != 8) {
+ int G = heads_per_block; if (G != 1 && G != 2 && G != 4 && G != 8) {
G = 8;
while (G > 1 && (heads % G != 0 || (heads / G) * T < 2 * sm_count())) G /= 2;
}
@@@ -2361,20 -2221,34 +2516,37 @@@ void ring_restore_batch(const RingBatch
kv_row_bytes(kKvFp8, d));
}
void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {
- hc_mean_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(h, hc, dim, out, out_stride);
+ #if defined(STRATA_METAL)
+ if (std::getenv("DS4_SKIP_HCMEAN")) return;
+ #endif
+ #if defined(STRATA_METAL)
+ // hc_mean_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::hc_mean(h, T, hc, dim, out, out_stride)) return;
+ #endif
+ strata_ds4_launch(hc_mean_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, h, hc, dim, out, out_stride);
+ }
+ void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) {
+ #if defined(STRATA_METAL)
+ // iota_rows_kernel on the Metal dispatch; the launch below is the fallback
+ if (strata::dsv41::metal_host::iota_rows(out, T, n, stride)) return;
+ #endif
+ strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride);
}
- void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kernel<<<T, 64, 0, s>>>(out, n, stride); }
void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {
- markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);
+ #if defined(STRATA_METAL)
+ // markov_kernel on the Metal dispatch (the drafter's Markov bias); the launch below is the fallback
+ if (strata::dsv41::metal_host::markov_add(logits, E, H, tok, V, R)) return;
+ #endif
+ strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);
}
+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s) {
- markov_kernel_q8<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, Hq, tok, V, R);
++ strata_ds4_launch(markov_kernel_q8, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, Hq, tok, V, R);
+}
void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {
- markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);
+ strata_ds4_launch(markov_share_kernel, cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, H, erow, vn, R);
}
void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s) {
- argmax_share_kernel<<<1, 1024, 0, s>>>(row, vn, v0, slot, seq);
+ strata_ds4_launch(argmax_share_kernel, 1, 1024, 0, s, row, vn, v0, slot, seq);
}
void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,
cudaStream_t s) {
@@@ -2406,38 -2300,18 +2598,38 @@@ void xfer_put(const Xfer& x, unsigned*
size_t total = 0;
for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));
- xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);
+ strata_ds4_launch(xfer_put_kernel, blocks, kThreads, 0, s, x, flag, seq, cnt);
+ // (another GPU, or the host, waits for this flag: on Windows the put must not sit in the queue - #303: two RTX 3080s
+ // hung at the first hand-off, the receiver spinning on a flag whose writer had never been submitted)
+ win_submit(s);
+}
+void win_submit(cudaStream_t s) {
+#if defined(_WIN32)
+ static const bool on = [] {
+ const char* v = std::getenv("DS4_WIN_SUBMIT");
+ return v == nullptr || std::atoi(v) != 0;
+ }();
+ if (on) {
+ (void) cudaStreamQuery(s);
+ (void) cudaGetLastError();
+ }
+#else
+ (void) s;
+#endif
}
void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {
size_t total = 0;
for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;
const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));
- xfer_get_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq);
+ strata_ds4_launch(xfer_get_kernel, blocks, kThreads, 0, s, x, flag, seq);
}
void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {
- if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);
+ if (n > 0) strata_ds4_launch(wait_flags_kernel, 1, 32, 0, s, flags, n, stride, seq);
}
-void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { strata_ds4_launch(set_flag_kernel, 1, 1, 0, s, flag, v); }
+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) {
- set_flag_kernel<<<1, 1, 0, s>>>(flag, v);
++ strata_ds4_launch(set_flag_kernel, 1, 1, 0, s, flag, v);
+ win_submit(s); // (the writer on another GPU waits for this acknowledgement before its next put - #384)
+}
void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {
const int slot = (int) (seq % (unsigned) a.slots);
if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)
diff --cc src/dsv41/ds4_kernels.cuh
index 08075fa00,72488e9b3..2727dc6df
--- a/src/dsv41/ds4_kernels.cuh
+++ b/src/dsv41/ds4_kernels.cuh
@@@ -61,13 -61,8 +61,17 @@@ void finish(float* x, int n_vec, int le
/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes
/// 32, pre / post 8, comb 64 floats a row.
void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);
+/// hc_mix then hc_split in one launch (cnt: T zeroed unsigned counters, left zero; n_mix <= 32, T <= kWin): the last block of a token
+/// runs the split - the same arithmetic as the two launches
+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,
+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,
+ float* comb, cudaStream_t s);
void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb, cudaStream_t s);
++/// the same two in one dispatch (the Metal kernel ds4_hc_mix_split); the CUDA path calls hc_mix then hc_split.
++void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,
++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre,
++ float* post, float* comb, cudaStream_t s);
/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).
void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,
cudaStream_t s);
diff --cc src/dsv41/ds4_launch.h
index 000000000,7d0bc9ad8..433f833ae
mode 000000,100644..100644
--- a/src/dsv41/ds4_launch.h
+++ b/src/dsv41/ds4_launch.h
@@@ -1,0 -1,28 +1,28 @@@
+ // strata_ds4_launch(kernel, grid, block, shared, stream, args...): the engine's one spelling of a kernel launch. On CUDA it is
+ // kernel<<<grid, block, shared, stream>>>(args...); on Metal metal/include/metal_compat/cuda_launch.h maps the same call onto
-// a dispatch. (Force-included for CUDA compiles by CMakeLists.txt.)
++// a dispatch. (Every file that launches kernels includes this header.)
+ #pragma once
-#if defined(__CUDACC__) && !defined(STRATA_METAL)
++#if (defined(__CUDACC__) || defined(__HIPCC__) || defined(__HIP__)) && !defined(STRATA_METAL)
+ #include <cuda_runtime.h>
+ #include <cstdio>
+ #include <cstdlib>
+ template <class... KA, class... CA>
+ inline void strata_ds4_launch_at(const char* file, int line, void (*kernel)(KA...), dim3 grid, dim3 block, size_t shared,
+ cudaStream_t stream, CA&&... args) {
+ kernel<<<grid, block, shared, stream>>>(static_cast<KA>(args)...);
+ static const bool check = std::getenv("DS4_LAUNCH_CHECK") != nullptr; // report the launch that failed (not a later call)
+ if (check) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess)
+ std::fprintf(stderr, "ds4 launch failed at %s:%d: %s (grid %u,%u,%u block %u,%u,%u shared %zu)\n", file, line,
+ cudaGetErrorString(e), grid.x, grid.y, grid.z, block.x, block.y, block.z, shared);
+ }
+ }
+ #define strata_ds4_launch(...) strata_ds4_launch_at(__FILE__, __LINE__, __VA_ARGS__)
+
+ // (the Metal debug dumps in ds4_model.cu name these; on CUDA there is no deferred queue and a device pointer is not a host view)
+ inline void ds4_metal_flush() {}
+ namespace strata::gpu {
+ inline void* host_view(const void* p) { return const_cast<void*>(p); }
+ }
+ #endif
diff --cc src/dsv41/ds4_main.cpp
index 3b259b2af,4916954c7..d576e348f
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@@ -11,18 -11,14 +11,24 @@@
// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that
// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to
// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures
-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.
+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With
+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the
+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but
+// does continue a kept conversation reads only its new tokens.
+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the
+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of
+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).
+ #if defined(__APPLE__)
+ #include <execinfo.h>
+ #include <csignal>
+ #include <unistd.h>
++#include <libproc.h> // proc_pid_rusage: the bytes this process had the storage read (io_read_bytes)
+ #endif
#include "ds4_model.hpp"
+#include "ds4_experts_rule.hpp"
+#include "ds4_os.hpp"
#include "ds4_packinfo.hpp"
+#include "ds4_slotmeta.hpp"
#include <algorithm>
#include <cmath>
@@@ -52,8 -42,8 +58,8 @@@ double now_ms()
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
--// the bytes this process had the storage read for it (Linux's /proc/self/io; -1 where there is none): the Monitor's
--// disk reads a token, the experts read in place from the model's files included
++// the bytes this process had the storage read for it (Linux's /proc/self/io, macOS's rusage; -1 where there is none): the
++// Monitor's disk reads a token, the experts read in place from the model's files included
long long io_read_bytes() {
#ifdef __linux__
std::FILE* f = std::fopen("/proc/self/io", "r");
@@@ -64,6 -54,6 +70,14 @@@
if (std::sscanf(line, "read_bytes: %lld", &v) == 1) break;
std::fclose(f);
return v;
++#elif defined(__APPLE__)
++ // (the same number `iotop`/Activity Monitor's "Bytes read" shows: every read the storage did for this process, the
++ // page cache's misses of the experts read in place included; a page-cache hit counts no storage read, which is what
++ // the 60 tps go/no-go needs)
++ struct rusage_info_v4 ri;
++ std::memset(&ri, 0, sizeof ri);
++ if (proc_pid_rusage(getpid(), RUSAGE_INFO_V4, (rusage_info_t*) &ri) != 0) return -1;
++ return (long long) ri.ri_diskio_bytesread;
#else
return -1;
#endif
@@@ -86,8 -76,8 +100,8 @@@ std::vector<int> parse_ids(const std::s
void usage() {
std::fprintf(stderr,
"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"
-- " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"
- " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"
++ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--profile-readonly] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"
+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F] [--experts N|auto]\n"
" [--vision on|off] [--mmap-experts on|off|auto]\n"
" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
" | --encode-image FILE.rgb --size WxH --dump-image OUT\n"
@@@ -166,9 -144,42 +181,42 @@@ float dspark_cum_default(const strata::
// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their
// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow
// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.
+ struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms, summed), printed at exit
+ double draft = 0, verify = 0, step = 0;
+ unsigned long long draft_d = 0, verify_d = 0, step_d = 0;
+ long rounds = 0, spec_rounds = 0, steps = 0;
+ ~RoundTimes() {
+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)
+ std::fprintf(stderr, "rounds: %ld (%ld speculative, %ld plain steps): draft %.1f ms (%.2f a round), verify %.1f ms (%.2f), plain step %.1f ms (%.2f)\n",
+ rounds, spec_rounds, steps, draft, spec_rounds ? draft / (double) spec_rounds : 0.0, verify, spec_rounds ? verify / (double) spec_rounds : 0.0,
+ step, steps ? step / (double) steps : 0.0);
+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)
+ std::fprintf(stderr, "dispatches a round: draft %.0f, verify %.0f, plain step %.0f\n", spec_rounds ? (double) draft_d / (double) spec_rounds : 0.0, spec_rounds ? (double) verify_d / (double) spec_rounds : 0.0, steps ? (double) step_d / (double) steps : 0.0);
+ }
+ } g_rt;
+ #if defined(STRATA_METAL)
+ extern "C" unsigned long long ds4_dispatch_seq();
+ #endif
+ #if defined(STRATA_METAL)
+ extern "C" void ds4_timeline_gate(int);
+ #endif
+ static void tl_gate(bool open) {
+ #if defined(STRATA_METAL)
+ static const bool only_draft = std::getenv("DS4_TL_DRAFT") != nullptr;
+ if (only_draft) ds4_timeline_gate(open ? 1 : 0);
+ #endif
+ }
+ static unsigned long long rt_seq() {
+ #if defined(STRATA_METAL)
+ return ds4_dispatch_seq();
+ #else
+ return 0;
+ #endif
+ }
+ static double rt_now() { return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,
float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,
- int& accepted, std::string& err) {
+ int& accepted, std::string& err, float cum_min = -1.0f) {
out.clear();
offered = accepted = 0;
if (spec && room > 1 && m.draft_block() > 0) {
@@@ -178,11 -198,7 +235,11 @@@
g_round_conf = conf;
g_round_nd = (int) drafts.size();
int keep = 0;
- while (keep < (int) drafts.size() && keep < room - 1 && keep < g_spec_depth && conf[(size_t) keep] >= conf_min) ++keep;
+ if (cum_min < 0) cum_min = dspark_cum_default(m);
+ float cum = 1.0f;
- while (keep < (int) drafts.size() && keep < room - 1 && conf[(size_t) keep] >= conf_min &&
++ while (keep < (int) drafts.size() && keep < room - 1 && keep < g_spec_depth && conf[(size_t) keep] >= conf_min &&
+ (cum *= conf[(size_t) keep]) >= cum_min)
+ ++keep;
if (keep > 0) {
drafts.resize((size_t) keep);
int acc = 0, next = -1;
@@@ -1182,15 -791,24 +1249,28 @@@ int spec_eval(strata::ds4::Model& m, co
}
} // namespace
+#ifndef DS4_BUILD_ID
+#define DS4_BUILD_ID "unknown"
+#endif
+
+ #if defined(STRATA_METAL)
+ #include "sampler_stat.inc"
+ #endif
int main(int argc, char** argv) {
+ strata::ds4::os::fine_timer();
+ #if defined(STRATA_METAL)
+ if (argc > 1 && std::string(argv[1]) == "--sampler-test") return sampler_stat::run(argc - 1, argv + 1); // (the sampler's statistics harness)
+ #endif
+ #if defined(__APPLE__)
+ if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)
+ signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });
+ signal(SIGBUS, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(138); });
+ signal(SIGABRT, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(134); });
+ }
+ #endif
-
strata::ds4::Options o;
std::string ids_s, dump, eos = "1";
- int n = 0, window = 16;
+ int n = 0, window = 0; // (0: the model's feed unit - a whole prompt on the chunked path; 16 for the scoring runs)
bool spec = true;
float temp = 0.0f; // the CLI's sampling (0: greedy)
unsigned long long cli_seed = 1;
@@@ -1213,9 -831,9 +1293,10 @@@
for (int g : parse_ids(val("--gpus"))) o.gpus.push_back(g);
} else if (a == "--max-context") o.max_context = std::atoll(val("--max-context").c_str());
else if (a == "--reserve-mib") o.reserve_mib = std::atoll(val("--reserve-mib").c_str());
- else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());
+ else if (a == "--vram-experts") { o.vram_experts = std::atof(val("--vram-experts").c_str()); o.vram_experts_auto = false; }
else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());
else if (a == "--expert-profile") o.profile = val("--expert-profile");
++ else if (a == "--profile-readonly") o.profile_readonly = true; // (read the placement, never write the counts back)
else if (a == "--ssd-cache") o.ssd_cache_gb = std::atof(val("--ssd-cache").c_str());
else if (a == "--ssd-policy") {
const std::string v = val("--ssd-policy");
@@@ -1227,13 -845,8 +1308,14 @@@
const std::string v = val("--dspark");
spec = v != "off";
o.dspark = v != "none"; // none: not even loaded
+ o.dspark_auto = v == "auto";
} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());
+ else if (a == "--experts") {
+ const std::string v = val("--experts");
+ o.experts = v == "auto" ? -1 : std::max(1, std::atoi(v.c_str()));
+ }
+ else if (a == "--dspark-block") o.dspark_block = std::max(0, std::atoi(val("--dspark-block").c_str()));
+ else if (a == "--dspark-depth") g_spec_depth = std::max(1, std::atoi(val("--dspark-depth").c_str()));
else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));
else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));
else if (a == "--threads") o.threads = val("--threads") != "off";
@@@ -1397,13 -992,16 +1482,21 @@@
std::vector<int> got;
for (;;) {
out.push_back(tok);
+ {
+ const double tn = now_ms();
+ tok_ms.push_back(tn - t_prev);
+ t_prev = tn;
+ }
std::printf("%d,", tok);
std::fflush(stdout);
+ if (std::getenv("DS4_PROGRESS") && out.size() % 200 == 0) { // (sustained-run time series)
+ static double tp = now_ms();
+ static size_t np = 0;
+ const double tn = now_ms();
+ std::fprintf(stderr, "progress %zu tokens: %.2f tok/s over the last %zu\n", out.size(), 1000.0 * (double) (out.size() - np) / (tn - tp), out.size() - np);
+ tp = tn;
+ np = out.size();
+ }
if ((tok == 1 && std::getenv("DS4_NO_EOS") == nullptr) || (int) out.size() >= n) break; // (DS4_NO_EOS: decode n tokens whatever)
if (got.empty()) {
int off = 0, acc = 0;
diff --cc src/dsv41/ds4_model.cu
index 91d29a266,4f605d077..c0ac558c8
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@@ -1,9744 -1,8826 +1,10417 @@@
-// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
-// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
-// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).
-//
-// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):
-// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables
-// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)
-// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus
-// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the
-// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b
-// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])
-// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert
-// h = post * moe + comb^T h; pre = f_pre
-// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
-#include "ds4_model.hpp"
-#include "ds4_engram_io.hpp"
-
-#include "ds4_gguf.cuh"
-#include "ds4_gguf_cpu.hpp"
-#include "ds4_kernels.cuh"
-#include "ds4_os.hpp"
-#include "ds4_pool.hpp"
-#include "ds4_store.hpp"
-#include "ds4_vision.cuh"
-#include "strata/dsv41/exl3_cpu.hpp"
-#include "strata/dsv41/exl3.hpp"
-#include "strata/kernels/f16_bits.hpp"
-#include "strata/dsv41/moe_win.hpp"
-#include "strata/dsv41/sampler.hpp"
-
-#include <cublas_v2.h>
-#include <cuda_runtime.h>
-
-#if defined(STRATA_METAL)
-extern "C" void ds4_busy_begin();
-extern "C" void ds4_busy_end();
-#include <sys/sysctl.h>
-#include <mach/mach.h>
-#endif
-#include <algorithm>
-#include <array>
-#include <condition_variable>
-#include <deque>
-#include <atomic>
-#include <chrono>
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <fstream>
-#include <functional>
-#include <future>
-#include <map>
-#include <mutex>
-#include <numeric>
-#include <sstream>
-#include <thread>
-#include <unordered_map>
-
-#if defined(_MSC_VER)
-#define strtok_r strtok_s
-#endif
-
-namespace strata::ds4 {
-namespace x3 = strata::dsv41k::exl3;
-namespace kc = strata::dsv41k::cpu;
-namespace gf = strata::dsv41k::glmf;
-namespace gg = strata::ds4::gguf;
-
-#define CK(call) \
- do { \
- cudaError_t e_ = (call); \
- if (e_ != cudaSuccess) { \
- err = std::string(#call) + ": " + cudaGetErrorString(e_); \
- return false; \
- } \
- } while (0)
-
-namespace {
-
-double now_ms() {
- return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
-}
-
-size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }
-
-// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:
-// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):
-// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before
-void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {
- const int b = whole / 128, q = b / np, r = b % np;
- auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };
- int at = 0;
- for (int i = 0; i < p; ++i) at += blocks(i);
- off = 128 * at;
- n = 128 * blocks(p);
-}
-
-std::string read_file(const std::string& p) {
- std::ifstream f(p);
- std::stringstream ss;
- ss << f.rdbuf();
- return ss.str();
-}
-std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
- std::vector<int> v;
- std::stringstream ss(s);
- std::string part;
- while (std::getline(ss, part, ',')) {
- if (part.empty() || part == "\n") continue;
- const size_t dash = part.find('-');
- const int a = std::atoi(part.c_str());
- const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);
- for (int c = a; c <= b; ++c) v.push_back(c);
- }
- return v;
-}
-// each NUMA node's physical cores (the first logical CPU of each core), in order
-std::vector<std::vector<int>> node_cores() {
-#if defined(_WIN32)
- static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)
- const char* v = std::getenv("DS4_CPU_SMT");
- return v != nullptr && std::atoi(v) != 0;
- }();
- std::vector<std::vector<int>> nodes = os::node_cores(smt);
-#else
- std::vector<std::vector<int>> nodes;
- for (int n = 0; n < 64; ++n) {
- const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");
- if (cl.empty()) break;
- std::vector<int> cores;
- for (int c : cpu_list(cl)) {
- const std::vector<int> sib = cpu_list(
- read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));
- if (sib.empty() || sib[0] == c) cores.push_back(c);
- }
- if (!cores.empty()) nodes.push_back(cores);
- }
-#endif
- if (nodes.empty()) {
- std::vector<int> all;
- for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);
- nodes.push_back(all);
- }
- return nodes;
-}
-// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages
-void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }
-void pin_self(int cpu) { os::pin_self(cpu); }
-// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root
-// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known
-uint64_t ram_avail() {
-#if defined(_WIN32)
- return os::ram_avail();
-#endif
- uint64_t avail = ~uint64_t{0};
- {
- std::istringstream mi(read_file("/proc/meminfo"));
- std::string k;
- uint64_t kb = 0;
- while (mi >> k >> kb) {
- if (k == "MemAvailable:") avail = kb << 10;
- mi.ignore(64, '\n');
- }
- }
- std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."
- const size_t at = cg.find("0::");
- if (at == std::string::npos) return avail;
- cg = cg.substr(at + 3);
- cg = cg.substr(0, cg.find('\n'));
- for (;;) {
- const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);
- const std::string mx = read_file(dir + "/memory.max");
- if (!mx.empty() && mx.compare(0, 3, "max") != 0) {
- const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);
- const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);
- uint64_t file = 0;
- std::istringstream st(read_file(dir + "/memory.stat"));
- std::string k;
- uint64_t v = 0;
- while (st >> k >> v)
- if (k == "file") file = v;
- const uint64_t used = cur > file ? cur - file : 0;
- avail = std::min(avail, lim > used ? lim - used : 0);
- }
- if (cg.empty() || cg == "/") break;
- const size_t sl = cg.rfind('/');
- cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);
- }
- return avail;
-}
-
-float bf16_to_f32(uint16_t b) {
- uint32_t u = (uint32_t) b << 16;
- float f;
- std::memcpy(&f, &u, 4);
- return f;
-}
-float f16_to_f32(uint16_t h) {
- const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
- uint32_t u;
- if (e == 0) {
- if (m == 0) u = s;
- else { // subnormal
- int ee = -1;
- uint32_t mm = m;
- do {
- ++ee;
- mm <<= 1;
- } while ((mm & 0x400) == 0);
- u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;
- }
- } else if (e == 31) u = s | 0x7f800000u | m << 13;
- else u = s | (e - 15 + 127) << 23 | m << 13;
- float f;
- std::memcpy(&f, &u, 4);
- return f;
-}
-
-// ------------------------------------------------------------------ the pack
-struct Tensor {
- std::string dtype;
- int file = -1;
- uint64_t off = 0, bytes = 0;
- std::vector<int64_t> shape;
-};
-
-struct Pack {
- std::string dir, model_dir;
- std::unordered_map<std::string, std::string> kv;
- std::vector<std::string> files;
- std::vector<os::fd_t> fds;
- std::unordered_map<std::string, Tensor> t;
- int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;
- std::vector<int32_t> eg_map;
- std::vector<int64_t> eg_mult, eg_primes, eg_off;
- // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the
- // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)
- bool gguf = false;
-
- std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)
- std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)
- std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)
-
- ~Pack() {
- for (auto& m : maps) os::unmap(m.first, m.second);
- for (os::fd_t fd : fds) os::close_fd(fd);
- for (os::fd_t fd : dfds) os::close_fd(fd);
- for (os::fd_t fd : pfds) os::close_fd(fd);
- }
- // file i mapped whole, read-only (once; not thread-safe: called while loading)
- const uint8_t* map(int file) {
- if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});
- auto& m = maps[(size_t) file];
- if (m.first == nullptr) {
- const uint64_t size = os::file_size(fds[(size_t) file]);
- void* p = os::map_ro(fds[(size_t) file], size);
- if (p == nullptr) return nullptr;
- m = {(uint8_t*) p, (size_t) size};
- }
- return m.first;
- }
-
- bool load(const std::string& d, std::string& err) {
- dir = d;
- {
- std::ifstream f(d + "/ds4.txt");
- if (!f) {
- err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";
- return false;
- }
- std::string line;
- while (std::getline(f, line)) {
- const size_t sp = line.find(' ');
- if (sp == std::string::npos) continue;
- kv[line.substr(0, sp)] = line.substr(sp + 1);
- }
- }
- model_dir = str("model_dir");
- std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");
- if (f == nullptr) {
- err = d + "/tensors.txt: not found";
- return false;
- }
- std::vector<char> buf(1 << 16);
- while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
- char* s = buf.data();
- if (s[0] == 'f') { // file <i> <name>
- char name[4096];
- int i = 0;
- if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
- if ((int) files.size() <= i) files.resize((size_t) i + 1);
- files[(size_t) i] = name;
- }
- continue;
- }
- if (s[0] != 't') continue;
- char* save = nullptr;
- ::strtok_r(s, " \n", &save);
- const char* name = ::strtok_r(nullptr, " \n", &save);
- const char* dt = ::strtok_r(nullptr, " \n", &save);
- const char* fi = ::strtok_r(nullptr, " \n", &save);
- const char* off = ::strtok_r(nullptr, " \n", &save);
- const char* nb = ::strtok_r(nullptr, " \n", &save);
- if (name == nullptr || nb == nullptr) continue;
- Tensor x;
- x.dtype = dt;
- x.file = std::atoi(fi);
- x.off = std::strtoull(off, nullptr, 10);
- x.bytes = std::strtoull(nb, nullptr, 10);
- while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));
- t.emplace(name, std::move(x));
- }
- std::fclose(f);
- for (const std::string& fn : files) {
- const std::string p = model_dir + "/" + fn;
- const os::fd_t fd = os::open_ro(p);
- if (fd == os::kBadFd) {
- err = p + ": cannot open";
- return false;
- }
- fds.push_back(fd);
- }
- gguf = str("format") == "gguf";
- if (gguf)
- for (size_t i = 0; i < fds.size(); ++i) {
- if (map((int) i) == nullptr) {
- err = model_dir + "/" + files[i] + ": cannot map";
- return false;
- }
- dfds.push_back(os::open_direct(model_dir + "/" + files[i]));
- pfds.push_back(os::open_par(model_dir + "/" + files[i]));
- }
- // the Engram tables
- std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");
- if (e == nullptr) {
- err = d + "/engram.bin: not found";
- return false;
- }
- char magic[4];
- int32_t hdr[6];
- bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&
- std::fread(hdr, 4, 6, e) == 6;
- if (ok) {
- eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];
- const int cols = (eg_ngram - 1) * eg_heads;
- eg_map.resize((size_t) eg_vocab);
- eg_mult.resize((size_t) eg_layers * eg_ngram);
- eg_primes.resize((size_t) eg_layers * cols);
- eg_off.resize((size_t) eg_layers * cols);
- ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&
- std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&
- std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&
- std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();
- }
- std::fclose(e);
- if (!ok) {
- err = d + "/engram.bin: unreadable";
- return false;
- }
- return true;
- }
-
- std::string str(const std::string& k, const std::string& def = "") const {
- auto it = kv.find(k);
- return it == kv.end() ? def : it->second;
- }
- double num(const std::string& k, double def) const {
- auto it = kv.find(k);
- return it == kv.end() ? def : std::atof(it->second.c_str());
- }
- std::vector<int> list(const std::string& k) const {
- std::vector<int> v;
- std::stringstream ss(str(k));
- std::string x;
- while (std::getline(ss, x, ','))
- if (!x.empty()) v.push_back(std::atoi(x.c_str()));
- return v;
- }
- const Tensor* get(const std::string& n) const {
- auto it = t.find(n);
- return it == t.end() ? nullptr : &it->second;
- }
- bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)
- if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)
- return os::pread_par(pfds[(size_t) file], dst, n, off);
- return os::pread_full(fds[(size_t) file], dst, n, off);
- }
- bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }
- const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)
-};
-
-// ------------------------------------------------------------------ geometry
-struct Geo {
- int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,
- k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,
- cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;
- float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;
- double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;
- std::vector<int> ratios, kv_src, ix_src, eg_layers;
- // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks
- int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;
- std::vector<int> d_targets;
-
- void from(const Pack& p) {
- vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);
- nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);
- hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);
- q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);
- o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);
- k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);
- win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);
- sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);
- ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);
- cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);
- cand_blk = (int) p.num("candidate_block_size", 0);
- orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);
- eos = (int) p.num("eos_token_id", 1);
- eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);
- route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);
- rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);
- factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);
- beta_slow = p.num("rope_scaling.beta_slow", 1);
- ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");
- ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");
- d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);
- d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);
- d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");
- // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)
- n_mtp = d_block > 0 && (int) ratios.size() > nl &&
- (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)
- ? (int) ratios.size() - nl : 0;
- }
-};
-
-// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them
-std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {
- std::vector<float> f((size_t) dim / 2);
- for (int i = 0; i < dim / 2; ++i) {
- const float e = (float) (2 * i) / (float) dim;
- f[(size_t) i] = 1.0f / std::pow((float) base, e);
- }
- if (original > 0) {
- auto corr = [&](double rot) {
- return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)
- };
- const double low = std::max(std::floor(corr(beta_fast)), 0.0);
- const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);
- const float den = (float) std::max(high - low, 1e-3);
- for (int i = 0; i < dim / 2; ++i) {
- float ramp = ((float) i - (float) low) / den;
- ramp = std::min(1.0f, std::max(0.0f, ramp));
- const float smooth = 1.0f - ramp;
- f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;
- }
- }
- return f;
-}
-
-// ------------------------------------------------------------------ devices, layers
-struct Dev {
- int id = 0;
- cudaStream_t st = nullptr;
- cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)
- int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)
- float* rw_a = nullptr;
- size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;
- std::vector<void*> allocs;
- // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)
- float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,
- *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,
- *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,
- *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,
- *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,
- *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,
- *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,
- *ed = nullptr;
- float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)
- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;
- unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)
- uint8_t* eg_rows = nullptr;
- uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows
- AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)
- // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the
- // driver's staging, ~1.7 ms a copy): h | pre | topk | cand
- uint8_t* bounce = nullptr;
- cudaEvent_t ev_out = nullptr;
- float* tap = nullptr; // a window's stream means before a DSpark target layer
- // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's
- // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window
- // MoE's grouping scratch
- int grp = 0, mi = 0;
- unsigned ar_seq = 0;
- unsigned* xcnt = nullptr;
- // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
- // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
- x3::Mat head;
- float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;
- unsigned head_seq = 0;
- // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the
- // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)
- uint16_t *mk_share = nullptr, *mk_erow = nullptr;
- int* mk_ints = nullptr;
- int mk_v0 = 0, mk_vn = 0;
- uint8_t* hand = nullptr;
- unsigned* hand_flag = nullptr;
- unsigned long long* grp_ptr = nullptr;
- int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
-};
-
-// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of
-// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS
-struct Chunk {
- int C = 0, A = 0, AI = 0;
- float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,
- *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,
- *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,
- *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,
- *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;
- uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,
- *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;
- size_t w16_elems = 0;
- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;
- uint8_t* eg_rows = nullptr;
- int* bounds_pin = nullptr; // pinned: the sort's bounds, read back
- // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,
- // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)
- static constexpr int kParts = 4;
- uint8_t* ring = nullptr;
- size_t ring_stride = 0;
- int ring_n = 0;
- cudaStream_t cs = nullptr;
- cudaEvent_t filled[kParts] = {}, freed[kParts] = {};
- // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of
- // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,
- // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s
- uint8_t* stage = nullptr;
- size_t stage_stride = 0;
- std::unique_ptr<Pool> readers;
- cublasHandle_t blas = nullptr;
- // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each
- // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)
- uint8_t* hand[2] = {nullptr, nullptr};
- unsigned* hand_flag[2] = {nullptr, nullptr};
- unsigned* hand_ack = nullptr;
- unsigned arb_seq = 0; // the group's chunk all-reduces
- // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)
- uint16_t* emb_host[2] = {nullptr, nullptr};
- uint8_t* eg_host[2] = {nullptr, nullptr};
- cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};
- double stream_bytes = 0; // the experts streamed in, in all
- // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a
- // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the
- // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once
- // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io
- // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits
- // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot
- // i % ring_n; the compute takes the items as they land, in runs of what has.
- struct Item {
- int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)
- int mi = 0; // the device's member index in the layer's group
- bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA
- const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read
- const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)
- uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there
- int64_t after = -1; // (layer-major) the items up to this one read first
- std::atomic<int> left{0}; // parts still being read
- std::atomic<bool> bad{false};
- };
- bool flow = false;
- bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring
- int dev = -1; // the device (index into devs) this chunk state belongs to
- std::unique_ptr<IoQueue> io;
- std::deque<Item> items; // items[0] is item `base`; the indices below are absolute
- size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;
- uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)
- int S = 0; // staging slots, stage_stride apart
- std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)
- std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use
- int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
- size_t spec_begin = 0;
- std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
- int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
- int *bc_pin = nullptr, *bc_dev = nullptr;
- uint64_t n_batch = 0;
- double wait_ms = 0; // the compute waiting for a read
- uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;
- uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started
- // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through
- // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in
- // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for
- // its later chunks, so each is read once a run
- bool lm = false, lm_pinned = false;
- uint8_t* lm_state = nullptr;
- size_t lm_state_bytes = 0;
- uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side
- size_t lc_bytes = 0;
- int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all
- size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read
- std::vector<int> lc_of; // expert -> its slot (-1: none)
- std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it
- uint64_t it_lc = 0; // items taken from the layer cache
-};
-
-struct Layer {
- int il = 0, dev = 0, ratio = 0;
- // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the
- // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert
- // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)
- int grp = 0, nm = 1, mi = 0;
- std::vector<Layer> mates;
- x3::ExpertLayout hlay;
- size_t hblob = 0;
- int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)
- // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more
- // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their
- // attention adds nothing to the sum
- int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;
- // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's
- // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member
- int* vs_pin = nullptr;
- int* vs_dev = nullptr;
- std::vector<int> host_list;
- std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."
- int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)
- bool mtp = false; // a DSpark block (index stage = il - n_layers)
- bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;
- int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)
- int ik_src = -1; // the kv source whose index keys its indexer reads
- int eg = -1; // engram layer index, or -1
- x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;
- float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,
- *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,
- *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,
- *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)
- uint16_t* i_wproj = nullptr;
- float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)
- uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)
- float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)
- // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row
- uint8_t* snap_win = nullptr;
- float *snap_kv = nullptr, *snap_sc = nullptr;
- // routed experts
- x3::ExpertLayout lay;
- size_t blob = 0; // bytes of one expert's blob
- int n_vram = 0;
- bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's
- uint8_t* vram_base = nullptr;
- std::vector<int> slot; // expert -> VRAM slot or -1
- std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
- std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)
- // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),
- // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)
- struct Half {
- size_t trellis[3], suh[3], svh[3], bytes;
- int k[3], n[3];
- } hl;
- std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes
- // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it
- // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)
- int n_slots = 0, spare_vram = -1;
- std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};
- std::vector<float> heat;
- std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked
- bool swap_pending = false;
- unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes
- // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and
- // the window's raw rows (kv, then score) to replay the accepted ones
- uint8_t* ring_bak = nullptr;
- float *c_snap = nullptr, *c_rows = nullptr;
- unsigned long long* vram_tab = nullptr; // device [n_exp]
- std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe
- unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it
- std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
- std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
- std::vector<uint64_t> region_bytes;
- std::vector<int> region_file;
- size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region
- size_t piece_bytes[3][3];
- size_t dense_bytes = 0;
- // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +
- // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane
- int mx_file[3] = {-1, -1, -1};
- uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};
-};
-
-struct Src {
- int layer = 0, ratio = 1;
- int64_t cap = 0;
- // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)
- std::vector<uint8_t*> ckv, ik;
- uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices
- cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;
- // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows
- // (ckv rows, then ik rows at kWin * hd) and its flag
- bool remote = false;
- uint8_t* x_stage = nullptr;
- unsigned* x_flag = nullptr;
- // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements
- // [slot][device] (32 apart); readers: the devices outside the source's group that read it
- uint8_t* x_stage1 = nullptr;
- unsigned* x_flag1 = nullptr;
- unsigned* x_ack = nullptr;
- std::vector<int> readers;
- uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }
- unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }
-};
-
-// a host thread that enqueues one device's work (threads mode)
-struct Worker {
- std::thread th;
- std::mutex mu;
- std::condition_variable cv;
- std::function<void()> job;
- bool busy = false, quit = false;
- void start() {
- th = std::thread([this] {
- for (;;) {
- std::function<void()> j;
- {
- std::unique_lock<std::mutex> lk(mu);
- cv.wait(lk, [&] { return quit || job != nullptr; });
- if (quit) return;
- j = std::move(job);
- job = nullptr;
- }
- j();
- {
- std::lock_guard<std::mutex> lk(mu);
- busy = false;
- }
- cv.notify_all();
- }
- });
- }
- void submit(std::function<void()> f) {
- {
- std::lock_guard<std::mutex> lk(mu);
- job = std::move(f);
- busy = true;
- }
- cv.notify_all();
- }
- void wait() {
- std::unique_lock<std::mutex> lk(mu);
- cv.wait(lk, [&] { return !busy; });
- }
- bool idle() {
- std::lock_guard<std::mutex> lk(mu);
- return !busy;
- }
- void stop() {
- {
- std::lock_guard<std::mutex> lk(mu);
- quit = true;
- }
- cv.notify_all();
- if (th.joinable()) th.join();
- }
-};
-
-struct Group {
- std::vector<int> mem; // device indices, member order (member 0 the lead)
- ArGroup ar, arb; // a window's all-reduces, a chunk's
- // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the
- // lead's acknowledgements (host-mapped)
- float* lg_stage = nullptr;
- unsigned *lg_flag = nullptr, *lg_ack = nullptr;
-};
-
-constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers
-
-// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the
-// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate
-// blocks as each layer sees them, the window's hand-off seq
-struct WinPlan {
- bool par = false, async = false;
- unsigned win = 0;
- int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2
- int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)
- int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)
- unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)
- unsigned seq[kMaxL] = {};
- int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};
-};
-
-} // namespace
-
-// ------------------------------------------------------------------ the model
-struct Model::Impl {
- Options o;
- Pack pk;
- Geo g;
- std::vector<Dev> devs;
- std::vector<Layer> L;
- std::vector<Src> srcs;
- std::vector<Group> groups;
- std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device
- bool par = false; // threads mode (the windows: decode)
- bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked
- unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)
- // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows
- int chunk = 0, chunk_min = 512;
- static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)
- // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk
- // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers
- // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token
- static constexpr int kMaxChunk = 4 * kVlRows;
- std::vector<std::unique_ptr<Chunk>> chunks; // per device
- size_t eg_cap = kWin; // tokens h_eg holds
- int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
- bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
- std::vector<float> chunk_logits;
- std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
- std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
- std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
- // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
- // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took
- struct PathModel {
- bool on = false;
- double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)
- // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and
- // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)
- double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};
- int64_t kn[2][4] = {};
- double win_s = 0; // windows: seconds a window took, so far
- int64_t win_n = 0;
- int last_cls = 0, last_nw = 0; // the last pick's size class, its windows
- double m = 1; // the picks' spread, against the profile's
- int64_t n_m = 0;
- int last = -1; // the last pick: 1 the flow, 0 windows
- double last_pred = 0; // its prediction (s, before its k)
- } pm;
- static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)
- static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)
- std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through
- WinPlan wplan;
- int head_dev = 0;
- x3::Mat head;
- float* norm_w = nullptr; // on the head device
- // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the
- // confidence head (on the head device: the drafts' logits come from the trunk's head)
- x3::Mat d_main_proj;
- float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
- uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
- // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
- // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
- // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
- bool mk_split = false;
- uint16_t* h_mk_embed = nullptr;
- unsigned* mk_slots = nullptr;
- unsigned mk_seq = 0;
- int d_dev = 0; // the drafter's first block's device
- bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)
- float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps
- cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it
- float* d_confv = nullptr; // on the head device
- int* d_ints = nullptr; // on the head device: drafts in, picks out
- float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;
- // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for
- // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()
- // checks it is its own)
- strata::dsv41k::SpecList* d_specq = nullptr;
- std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);
- bool drafts_q = false;
- float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;
- int dq_top_k = 0;
- uint64_t dq_seed = 0, dq_counter = 0;
- // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the
- // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5
- // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the
- // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs
- // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)
- bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;
- float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;
- int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;
- std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;
- float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)
- uint16_t* h_dump_ring = nullptr;
-
- uint16_t* embed_pinned = nullptr;
- std::vector<float> freqs_win, freqs_comp;
- std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores
- std::vector<std::vector<int>> cores; // each node's physical cores
- int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)
- // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps
- // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)
- bool file_experts = false;
- uint64_t warm_bytes = 0;
- static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)
- uint8_t* file_part(const Layer& l, int e) const {
- return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];
- }
- // host side of the CPU lane, a window's rows
- float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;
- int* h_idx = nullptr;
- int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks
- uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales
- // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one
- // an Engram layer: asked for when the window's tokens are known, collected right before that layer)
- EgReader eg_io;
- EgRowCache eg_cache;
- std::mutex eg_cache_mu;
- std::vector<std::unique_ptr<EgBatch>> eg_win;
- // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1
- // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8
- // codes with their E8M0 scales in a tensor of their own (see eg_issue)
- std::vector<uint8_t> eg_fmt;
- std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
- uint16_t* h_emb = nullptr; // the window's embedding rows
- std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
- // the state
- int64_t pos = 0;
- std::vector<int32_t> toks, ctoks;
- struct Check { // the state at a position: every window ring, the compressors' partial groups
- int64_t pos = -1;
- std::vector<std::vector<uint8_t>> win;
- std::vector<std::vector<float>> comp;
- };
- Check ck[2]; // the end of the last prompt, and the position before it
- std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
- Stats st;
- int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
- // (plan_window's: the windows' plans are made in order)
- int n_logits = 0; // the last forward's logit rows
- // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread
- // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum
- Req* reqs = nullptr;
- Resp* resps = nullptr;
- unsigned seq_issued = 0;
- std::atomic<int> layer_of[kRing];
- std::thread service;
- std::atomic<bool> quit{false};
-#if defined(STRATA_METAL)
- // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the
- // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)
- bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;
-#else
- bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
-#endif
- // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
- // before the wait for the CPU lane - the same bits, no faster)
- bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
- int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this
- // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same
- // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files
- // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next
- // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the
- // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's
- // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).
- int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;
- int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;
- // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)
- bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;
- std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;
- // 8 asked for (prefetch)
- std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)
- std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for
- std::thread pf_th;
- std::mutex pf_mu;
- std::condition_variable pf_cv;
- std::vector<std::pair<const void*, size_t>> pf_q;
- // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster
- // on one V100 - more for a PC to try: each its staging buffers and
- // copy streams), switched between windows
- struct Swap {
- int layer = 0, in = -1, out = -1;
- int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM
- };
- static constexpr int kMaxSwaps = 8;
- // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay
- // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)
- static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }
- const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),
- c_decay = env_f("DS4_CACHE_DECAY", 0.9f);
- const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));
- // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the
- // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the
- // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the
- // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in
- // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4
- // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.
- struct RamSwap {
- int layer = 0, in = -1, out = -1, slot = -1;
- };
- std::vector<uint8_t*> ram_slots, ram_chunks;
- std::vector<size_t> ram_chunk_bytes;
- bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)
- size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers
- std::vector<int> ram_free;
- std::deque<RamSwap> ram_queue;
- std::vector<RamSwap> ram_ready;
- std::condition_variable ram_cv;
- std::thread ram_th;
- int ram_inflight = 0, ram_held = 0;
- bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;
- // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a
- // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route
- // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every
- // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the
- // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left
- // out still counts as used (the expert cache, the profile).
- const float disk_skip_w = [] {
- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");
- return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;
- }();
- const int disk_skip_from = [] {
- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
- return v != nullptr ? std::max(0, std::atoi(v)) : 0;
- }();
- // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
- std::unique_ptr<ExpertStore> store;
- uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token
- size_t zc_stride = 0;
- bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves
- bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- for (int r = 0; r < 3; ++r)
- if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;
- return true;
- }
- return !file_experts || os::in_ram(file_part(l, e), l.blob);
- }
- // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads
- // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between
- // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of
- // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'
- // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes
- // the RAM place of the one it takes in)
- // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private
- // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever
- // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the
- // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds
- // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the
- // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold
- // them (the log's WARNING, a NOTE and INFO for the server)
- double commit_kept = 0, pagefile_hint = 0;
- double commit_margin() const {
- const os::Commit c = os::commit();
- return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);
- }
- // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)
- double commit_room() const {
- const os::Commit c = os::commit();
- double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;
- if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);
- return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());
- }
- // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)
- bool commit_charged(const Dev& d) const {
- if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;
-#if defined(_WIN32)
- int v = 0;
- if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {
- (void) cudaGetLastError();
- return false;
- }
- return v != 0;
-#else
- (void) d;
- return false;
-#endif
- }
- bool copy_mode = false;
- std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts
- std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes
- std::vector<int> copy_spare; // layer -> its spare slot (-1: none)
- std::mutex sw_mu;
- std::condition_variable sw_cv;
- std::deque<Swap> sw_queue;
- std::vector<Swap> sw_ready;
- struct SwapBufs {
- uint8_t *a = nullptr, *b = nullptr, *h = nullptr;
- std::vector<cudaStream_t> st; // a copy stream a device
- };
- std::vector<SwapBufs> sw_bufs;
- std::vector<std::thread> cache_ths;
- int sw_inflight = 0;
- int64_t decay_tokens = 0;
- bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;
- std::vector<float> host_logits;
- std::vector<void*> host_allocs;
- std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;
- // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or
- // chunk path (tests: where the two part)
- std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;
- // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records
- // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)
- std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;
- std::vector<int> dump_s_layers = [] {
- std::vector<int> v;
- const char* e = std::getenv("DS4_DUMP_S_LAYERS");
- std::string str = e ? e : "0,1,2,3";
- size_t a = 0;
- while (a < str.size()) {
- size_t b = str.find(',', a);
- if (b == std::string::npos) b = str.size();
- if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));
- a = b + 1;
- }
- return v;
- }();
- // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),
- // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the
- // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias
- // [n_exp] (F32)
- std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;
- // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,
- // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]
- // (F16) and y rows [rows dim] (F32)
- int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;
- std::FILE* dump_y = [] {
- const char* v = std::getenv("DS4_DUMP_Y");
- const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;
- return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;
- }();
- std::mutex dump_mu;
- // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its
- // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU
- // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it
- bool prof_on = std::getenv("DS4_PROF") != nullptr;
- std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;
- struct ProfMark {
- const Dev* d;
- cudaEvent_t ev;
- int stage;
- };
- std::mutex prof_mu;
- std::vector<ProfMark> prof_marks;
- std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on
- bool prof_window = false; // (decode windows only: T == 1)
- // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
- // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
- bool prof_chunk = false;
- double cprof_ms[4] = {};
- void prof_mark(const Dev& d, int stage) {
- if (!prof_window && !prof_chunk) return;
- std::lock_guard<std::mutex> lk(prof_mu);
- int cur = 0;
- cudaGetDevice(&cur);
- cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")
- cudaEvent_t e = nullptr;
- std::vector<cudaEvent_t>& pool = prof_pool[d.id];
- if (pool.empty()) cudaEventCreate(&e);
- else {
- e = pool.back();
- pool.pop_back();
- }
- cudaEventRecord(e, d.st);
- cudaSetDevice(cur);
- prof_marks.push_back({&d, e, stage});
- }
- void prof_collect() {
- if (!prof_window && !prof_chunk) return;
- std::lock_guard<std::mutex> lk(prof_mu);
- std::vector<std::pair<const Dev*, cudaEvent_t>> last;
- for (const ProfMark& m : prof_marks) {
- auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
- if (it != last.end()) {
- float ms = 0.0f;
- cudaSetDevice(m.d->id);
- if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
- (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
- it->second = m.ev;
- } else {
- last.push_back({m.d, m.ev});
- }
- }
- for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);
- prof_marks.clear();
- if (!prof_chunk) ++st.prof_tokens;
- }
- void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
- std::vector<float> v((size_t) T * g.hc * g.dim);
- cudaSetDevice(d.id);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
- cudaDeviceSynchronize();
-#endif
- cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[3] = {il, p0, T};
- std::fwrite(hdr, 8, 3, dump_l);
- std::fwrite(v.data(), 4, v.size(), dump_l);
- std::fflush(dump_l);
- }
-
- bool dump_s_on(int il) const {
- if (dump_s == nullptr) return false;
- for (int x : dump_s_layers)
- if (x == il) return true;
- return false;
- }
- bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)
- static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();
- return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;
- }
- void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {
- if (!dump_s_on(il) || !dump_s_name(name)) return;
- std::vector<uint8_t> v(nbytes);
- cudaSetDevice(d.id);
-#if defined(STRATA_METAL)
- ds4_metal_flush();
- cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)
-#endif
- cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- dump_sub_host(name, il, p0, T, v.data(), nbytes);
- }
- void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {
- if (!dump_s_on(il) || !dump_s_name(name)) return;
- std::lock_guard<std::mutex> lk(dump_mu);
- char nm[16] = {};
- std::strncpy(nm, name, 15);
- const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};
- std::fwrite(nm, 1, 16, dump_s);
- std::fwrite(hdr, 8, 4, dump_s);
- std::fwrite(host, 1, nbytes, dump_s);
- std::fflush(dump_s);
- }
-
- ~Impl() {
- eg_io.stop(); // (its threads write into h_eg)
- for (auto& w : workers) w->stop();
- for (size_t i = 0; i < chunks.size(); ++i)
- if (chunks[i] && chunks[i]->blas) {
- cudaSetDevice(devs[i].id);
- cublasDestroy(chunks[i]->blas);
- }
- quit.store(true);
- if (service.joinable()) service.join();
- {
- std::lock_guard<std::mutex> lk(pf_mu);
- }
- pf_cv.notify_all();
- if (pf_th.joinable()) pf_th.join();
- {
- std::lock_guard<std::mutex> lk(sw_mu);
- }
- sw_cv.notify_all();
- ram_cv.notify_all();
- for (std::thread& t : cache_ths)
- if (t.joinable()) t.join();
- if (ram_th.joinable()) ram_th.join();
- for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- for (void* p : d.allocs) cudaFree(p);
- if (d.st) cudaStreamDestroy(d.st);
- if (d.ev) cudaEventDestroy(d.ev);
- }
- for (void* p : host_allocs) cudaFreeHost(p);
- for (Layer& l : L)
- for (uint8_t* b : l.host)
- if (b != nullptr) {
- (void) b; // host blobs live in one mapping per layer, unmapped below
- }
- for (auto& m : host_maps) {
- if (chunk > 0) cudaHostUnregister(m.first);
- os::anon_free(m.first, m.second);
- }
- }
- std::vector<std::pair<void*, size_t>> host_maps;
-
- template <class T>
- bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {
- void* v = nullptr;
- cudaSetDevice(d.id);
- const size_t b = align256(n * sizeof(T));
- if (cudaMalloc(&v, b) != cudaSuccess) {
- err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st
- // does not order against - a zeroing still queued there could wipe the weights just copied
- if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {
- cudaFree(v);
- err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- d.allocs.push_back(v);
- if (acct) *acct += b;
- else d.scratch += b;
- p = (T*) v;
- return true;
- }
- static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
- static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
- // bytes of a VRAM slot of l's experts (a member's part of each, in a group)
- static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }
- // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64
- // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share
- int head_members(int nm) const {
- int na = nm;
- while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;
- return na;
- }
- // host-mapped memory (the same address on the host and every device), zeroed
- template <class T>
- bool hmap(T*& p, size_t n, std::string& err) {
- void* v = nullptr;
- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";
- return false;
- }
- std::memset(v, 0, n * sizeof(T) + 64);
- host_allocs.push_back(v);
- p = (T*) v;
- return true;
- }
- template <class T>
- bool halloc(T*& p, size_t n, std::string& err) {
- void* v = nullptr;
- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {
- err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";
- return false;
- }
- host_allocs.push_back(v);
- p = (T*) v;
- return true;
- }
-
- // ---- weights
- bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {
- std::vector<uint8_t> tmp(t.bytes);
- if (!pk.read(t, tmp.data())) {
- err = "read failed";
- return false;
- }
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
- return true;
- }
- // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs
- bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,
- int np, bool cols) {
- if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {
- err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";
- return false;
- }
- int n = (int) t.shape[0], k = (int) t.shape[1];
- const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);
- std::vector<uint8_t> w((size_t) n * rb);
- if (t.bytes != w.size() || !pk.read(t, w.data())) {
- err = pre + ": read failed";
- return false;
- }
- if (np > 1) {
- const int whole = cols ? n : k, wd = whole / np;
- if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {
- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";
- return false;
- }
- std::vector<uint8_t> p;
- if (cols) {
- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
- n = wd;
- } else {
- const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;
- p.resize((size_t) n * pb);
- for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);
- k = wd;
- }
- w.swap(p);
- }
- uint8_t* dp = nullptr;
- if (!dalloc(d, dp, w.size(), err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
- m = x3::Mat{};
- m.fmt = gg::kFmtGG;
- m.K = ggt;
- m.raw = dp;
- m.k = k;
- m.n = n;
- m.ld = (int) gg::type_row_bytes(ggt, k);
- return true;
- }
- // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
- // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
- // standalone [n][k] matrix
- bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
- int np, bool cols) {
- const bool q8 = t.dtype == "Q8_0";
- // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)
- const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;
- if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);
- if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
- err = pre + ": a " + t.dtype + " tensor that is not a matrix";
- return false;
- }
- int n = (int) t.shape[0], k = (int) t.shape[1];
- // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)
- auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };
- std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));
- if (t.bytes != w.size() || !pk.read(t, w.data())) {
- err = pre + ": read failed";
- return false;
- }
- if (np > 1) {
- const int whole = cols ? n : k, wd = whole / np;
- if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {
- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +
- std::to_string(np) + " parts";
- return false;
- }
- const size_t rb = bytes_of((size_t) k);
- std::vector<uint8_t> p;
- if (cols) {
- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
- n = wd;
- } else {
- const size_t pb = bytes_of((size_t) wd);
- p.resize((size_t) n * pb);
- for (size_t r = 0; r < (size_t) n; ++r)
- std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);
- k = wd;
- }
- w.swap(p);
- }
- uint8_t* dp = nullptr;
- if (!dalloc(d, dp, w.size(), err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
- m = x3::Mat{};
- m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;
- m.raw = dp;
- m.k = k;
- m.n = n;
- m.ld = q8 ? (int) bytes_of((size_t) k) : k;
- return true;
- }
- // a Q8_0 tensor's values as F32 (host)
- static std::vector<float> q8_values(const uint8_t* q, size_t n) {
- std::vector<float> v(n);
- for (size_t b = 0; b < n / 32; ++b) {
- const uint8_t* p = q + b * gg::kQ8Bytes;
- const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));
- for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];
- }
- return v;
- }
- // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone
- // matrix of its own (the rotations' 128-blocks stay whole)
- // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix
- // splits evenly, its groups only as wide as that goes)
- bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
- bool cols = true, int rot = 0) {
- if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))
- return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
- const Tensor* tr = pk.get(pre + ".trellis");
- const Tensor* su = pk.get(pre + ".suh");
- const Tensor* sv = pk.get(pre + ".svh");
- if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {
- err = pre + ": not an EXL3 matrix in the pack";
- return false;
- }
- if (pk.get(pre + ".mul1") == nullptr) {
- err = pre + ": only the mul1 codebook is covered";
- return false;
- }
- m.k = (int) tr->shape[0] * 16;
- m.n = (int) tr->shape[1] * 16;
- m.K = (int) tr->shape[2] / 16;
- m.cb = 2;
- m.ld = 0;
- if (np > 1) {
- const int whole = cols ? m.n : m.k;
- int at = 0, w = 0;
- if (whole % 128 != 0 || whole / 128 < np) {
- err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +
- std::to_string(np) + " parts of 128-blocks";
- return false;
- }
- split128(whole, np, part, at, w, rot);
- std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);
- if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {
- err = pre + ": read failed";
- return false;
- }
- const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));
- std::vector<uint8_t> t1, u1, v1;
- if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)
- const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;
- t1.resize((size_t) (m.k / 16) * nth * tile);
- for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)
- std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);
- u1 = u0;
- v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));
- m.n = w;
- } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)
- const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;
- t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));
- u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));
- v1 = v0;
- m.k = w;
- }
- uint32_t* dt = nullptr;
- uint16_t *ds = nullptr, *dv = nullptr;
- if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||
- !dalloc(d, dv, v1.size() / 2, err, acct))
- return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));
- CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));
- CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));
- m.trellis = dt;
- m.suh = ds;
- m.svh = dv;
- return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);
- }
- uint32_t* dt = nullptr;
- uint16_t *ds = nullptr, *dv = nullptr;
- if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||
- !dalloc(d, dv, sv->bytes / 2, err, acct))
- return false;
- if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;
- m.trellis = dt;
- m.suh = ds;
- m.svh = dv;
- if (!x3::supported(m)) {
- err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +
- " not covered by the EXL3 kernels";
- return false;
- }
- return true;
- }
- // a float tensor (F32 / F16 / BF16) as F32 on the device
- bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {
- const Tensor* t = pk.get(name);
- if (t == nullptr) {
- err = name + ": not in the pack";
- return false;
- }
- std::vector<uint8_t> raw(t->bytes);
- if (!pk.read(*t, raw.data())) {
- err = name + ": read failed";
- return false;
- }
- size_t n = 0;
- std::vector<float> v;
- if (t->dtype == "F32") {
- n = t->bytes / 4;
- v.resize(n);
- std::memcpy(v.data(), raw.data(), t->bytes);
- } else if (t->dtype == "F16" || t->dtype == "BF16") {
- n = t->bytes / 2;
- v.resize(n);
- const uint16_t* s = (const uint16_t*) raw.data();
- for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);
- } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)
- n = t->bytes / gg::kQ8Bytes * 32;
- v = q8_values(raw.data(), n);
- } else {
- err = name + ": dtype " + t->dtype + " is not a float";
- return false;
- }
- if (want != 0 && n != want) {
- err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);
- return false;
- }
- if (!dalloc(d, p, n, err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));
- return true;
- }
- bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {
- const Tensor* t = pk.get(name);
- if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)
- const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);
- std::vector<uint8_t> raw(t->bytes);
- if (!pk.read(*t, raw.data())) {
- err = name + ": read failed";
- return false;
- }
- const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();
- std::vector<uint16_t> h(n);
- for (size_t i = 0; i < n; ++i) {
- float v;
- if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);
- else if (t->dtype == "Q8_0") v = q[i];
- else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);
- h[i] = strata::kernels::f16_from_f32(v);
- }
- if (!dalloc(d, p, n, err, acct)) return false;
- cudaSetDevice(d.id);
- CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));
- return true;
- }
- if (t == nullptr || t->dtype != "F16") {
- err = name + ": not an F16 tensor in the pack";
- return false;
- }
- if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;
- return upload(d, *t, p, err);
- }
-
- size_t mat_bytes(const std::string& pre) const {
- if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
- size_t b = 0;
- for (const char* s : {".trellis", ".suh", ".svh"})
- if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
- return b;
- }
- size_t tensor_bytes(const std::string& name) const {
- const Tensor* t = pk.get(name);
- return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
- }
-
- // ---- expert regions and blob layout
- // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every
- // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
- // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
- static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})
- if (dtype == gg::type_name(t)) return t;
- return -1;
- }
- bool expert_layout_mx(Layer& l, std::string& err) {
- const std::string P = l.name + "ffn.experts.";
- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;
- const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");
- const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;
- const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;
- const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;
- if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {
- err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +
- "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";
- return false;
- }
- l.mx_file[r] = t0->file;
- l.mx_off[r] = t0->off;
- l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;
- for (int e = 0; e < l.n_exp; ++e) {
- const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");
- if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||
- t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {
- err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";
- return false;
- }
- }
- l.lay.trellis[r] = off;
- l.lay.suh[r] = l.lay.svh[r] = off;
- l.lay.K[r] = ty;
- off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)
- }
- l.lay.fmt = gg::kFmtMXFP4;
- if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
- gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||
- gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {
- err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +
- " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";
- return false;
- }
- l.blob = off;
- // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where
- // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM
- for (int r = 0; r < 3; ++r)
- if (!gg::type_on_device(l.lay.K[r])) {
- l.cpu_only = true;
- std::fprintf(stderr,
- "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "
- "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",
- gg::type_name(l.lay.K[r]));
- break;
- }
- return true;
- }
- const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
- if (store) {
- const int s = store->find(l.il, e);
- if (s >= 0) return store->mat(s, r);
- }
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
- return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
- : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
- return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);
- }
- size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)
- return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);
- }
- // (copy_mode) matrix r's rows, and the rows of one NUMA part of it
- int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }
- size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }
- // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them
- const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {
- if (copy_mode && l.ram[(size_t) e] >= 0)
- return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
- return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- }
- // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)
- void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);
- }
- }
- void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);
- }
- }
- // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down
- // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back
- bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {
- if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);
- else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);
- return true;
- }
- return read_mx_file(l, e, blob);
- }
- bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)
- for (int r = 0; r < 3; ++r)
- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),
- blob + l.lay.trellis[r]))
- return false;
- return true;
- }
- void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);
- for (size_t row = 0; row < (size_t) g.dim; ++row)
- std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);
- }
- void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);
- for (size_t row = 0; row < (size_t) g.dim; ++row)
- std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);
- }
-
- bool expert_layout(Layer& l, std::string& err) {
- if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)
- return expert_layout_mx(l, err);
- const std::string P = l.name + "ffn.experts.";
- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
- const char* pieces[3] = {".trellis", ".suh", ".svh"};
- l.region_off.resize((size_t) l.n_exp);
- l.region_bytes.resize((size_t) l.n_exp);
- l.region_file.resize((size_t) l.n_exp);
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");
- if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {
- err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";
- return false;
- }
- l.lay.K[r] = (int) tr->shape[2] / 16;
- l.lay.trellis[r] = off;
- off += align256(tr->bytes);
- l.lay.suh[r] = off;
- off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);
- l.lay.svh[r] = off;
- off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);
- }
- l.lay.cb = 2;
- l.blob = off;
- for (int e = 0; e < l.n_exp; ++e) {
- const std::string E = P + std::to_string(e) + ".";
- uint64_t lo = UINT64_MAX, hi = 0;
- int file = -1;
- for (int r = 0; r < 3; ++r)
- for (int q = 0; q < 3; ++q) {
- const Tensor* t = pk.get(E + parts[r] + pieces[q]);
- if (t == nullptr) {
- err = E + parts[r] + pieces[q] + ": missing";
- return false;
- }
- if (file < 0) file = t->file;
- if (t->file != file) {
- err = E + ": split across files";
- return false;
- }
- lo = std::min(lo, t->off);
- hi = std::max(hi, t->off + t->bytes);
- if (e == 0) l.piece_bytes[r][q] = t->bytes;
- else if (t->bytes != l.piece_bytes[r][q]) {
- err = E + parts[r] + ": not the shape of expert 0's";
- return false;
- }
- }
- l.region_off[(size_t) e] = lo;
- l.region_bytes[(size_t) e] = hi - lo;
- l.region_file[(size_t) e] = file;
- for (int r = 0; r < 3; ++r)
- for (int q = 0; q < 3; ++q) {
- const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;
- if (e == 0) l.piece_src[r][q] = src;
- else if (src != l.piece_src[r][q]) {
- err = E + ": the pieces lie differently from expert 0's";
- return false;
- }
- }
- }
- return true;
- }
- // the region of expert e (read from the file) -> its blob
- void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};
- for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);
- }
- }
-
- // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)
- void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- const uint8_t* tr = region + l.piece_src[r][0];
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- (size_t) nth * tile);
- std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);
- std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
- }
- }
-
- // ---- loading
- bool load(const Options& opt, std::string& err) {
- o = opt;
- const double t0 = now_ms();
- if (!pk.load(o.pack, err)) return false;
- g.from(pk);
- if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {
- err = o.pack + "/ds4.txt: incomplete geometry";
- return false;
- }
- if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {
- err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +
- std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";
- return false;
- }
- freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);
- freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);
-
- // devices
- int ndev = 0;
- if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
- err = "no CUDA device";
- return false;
- }
- std::vector<int> ids = o.gpus;
- if (ids.empty())
- for (int i = 0; i < ndev; ++i) ids.push_back(i);
- devs.resize(ids.size());
- for (size_t i = 0; i < ids.size(); ++i) {
- Dev& d = devs[i];
- d.id = ids[i];
- CK(cudaSetDevice(d.id));
- CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));
- CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));
- if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {
- err = "exl3 scratch on CUDA" + std::to_string(d.id);
- return false;
- }
- CK(cudaMemGetInfo(&d.free0, &d.total));
- }
- // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each
- // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)
- int tp = o.tp;
- if (pk.gguf && tp > 1) {
- int blk = 32;
- for (const auto& kv : pk.t) {
- const std::string& n = kv.first;
- if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||
- n.compare(n.size() - 9, 9, "w2.weight") != 0)
- continue;
- const int ty = gg_type_of(kv.second.dtype);
- if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
- }
- while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||
- g.o_groups % tp != 0))
- --tp;
- if (tp != o.tp)
- std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "
- "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);
- }
- // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone
- // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the
- // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the
- // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down
- // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
- par = o.threads || tp > 1;
- prompt_par = par || o.tp > 1;
-#if defined(STRATA_METAL)
- if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path
-#endif
- const bool tp_all = tp > 1 && tp == (int) devs.size();
- for (size_t i = 0; i < devs.size();) {
- Group G;
- G.mem.push_back((int) i);
- size_t j = i + 1;
- while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))
- G.mem.push_back((int) j++);
- if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own
- for (int m : G.mem) {
- Group one;
- one.mem.push_back(m);
- groups.push_back(one);
- }
- } else groups.push_back(G);
- i = j;
- }
- for (size_t gi = 0; gi < groups.size(); ++gi)
- for (size_t m = 0; m < groups[gi].mem.size(); ++m) {
- devs[(size_t) groups[gi].mem[m]].grp = (int) gi;
- devs[(size_t) groups[gi].mem[m]].mi = (int) m;
- }
- // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the
- // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM
- // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place
- // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times
- if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM
- // after VRAM's are in, by --mmap-experts)
- else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;
- else {
- double exp_b = 0, other_b = 0, room = 0;
- for (const auto& kt : pk.t) {
- const std::string& n = kt.first;
- if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
- (!o.dspark && n.rfind("mtp.", 0) == 0))
- continue;
- (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
- }
- for (const Dev& d : devs)
- room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -
- (prompt_par ? 1.5 : 0.4) * (1 << 30)) *
- o.vram_experts;
- file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
- }
- chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;
- const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)
- if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
- if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
- if (chunk > 0) {
- chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));
- eg_cap = (size_t) chunk;
- tap_cap = std::max(kWin, 128);
- } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)
- std::fprintf(stderr,
- "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"
- " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"
- " 461 / 1010 in chunks of %d) - %s\n",
- kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,
- o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"
- : "leave --tp 1 out, or pass --threads on, for chunks");
- // the vision tower on the first GPU, before the split: the layers get what it leaves
- if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,
- if (o.vision_only) return false; // and the server is told pictures are off - info_lines)
- std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "
- "text only\n", err.c_str());
- vision_err = err.empty() ? "unknown reason" : err;
- err.clear();
- vt.release();
- img_tok = -1;
- }
- if (o.vision_only) return true;
-
- // layers: roles and the sources they read
- if (!o.dspark) g.n_mtp = 0;
- L.resize((size_t) (g.nl + g.n_mtp));
- int last_src = -1;
- for (int il = 0; il < g.nl + g.n_mtp; ++il) {
- Layer& l = L[(size_t) il];
- l.il = il;
- l.mtp = il >= g.nl;
- l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
- l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
- l.k_exp = l.mtp ? g.d_k : g.k_exp;
- l.ratio = g.ratios[(size_t) il];
- if (l.mtp) { // window attention only: no compressor, indexer or Engram
- if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
- err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +
- std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";
- return false;
- }
- if (!expert_layout(l, err)) return false;
- continue;
- }
- l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();
- l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();
- l.owns_k = l.kv_src;
- l.cand_src = il == g.cand_src;
- l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;
- if (l.kv_src) {
- Src s;
- s.layer = il;
- s.ratio = l.ratio;
- s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;
- srcs.push_back(s);
- last_src = (int) srcs.size() - 1;
- }
- if (l.ratio > 0) {
- if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {
- err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +
- " without a KV source of that ratio before it";
- return false;
- }
- l.src = last_src;
- l.ik_src = last_src;
- }
- auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);
- if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());
- if (!expert_layout(l, err)) return false;
- }
-
- // the split: each layer's dense weights + its experts, the head on the last GPU
- for (Layer& l : L) {
- const std::string& P = l.name;
- size_t b = 0;
- if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");
- for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",
- "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",
- "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})
- b += mat_bytes(P + m);
- for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));
- b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");
- b += (size_t) g.win * win_rb() + (2u << 20);
- l.dense_bytes = b;
- }
- const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);
- const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;
- const size_t kv_per_src_dev = 0; // counted after the split
- (void) kv_per_src_dev;
- std::vector<double> avail(groups.size(), 0.0);
- double avail_sum = 0;
- for (size_t gi = 0; gi < groups.size(); ++gi) {
- for (int m : groups[gi].mem)
- avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));
- avail_sum += avail[gi];
- }
- double need_sum = (double) head_b;
- for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;
- {
- // contiguous layer ranges, each group's share of the total by its free memory
- size_t d = 0;
- double cum = 0, target = avail[0] / avail_sum * need_sum;
- std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers
- for (Layer& l : L)
- if (l.mtp) order.push_back(&l);
- for (Layer& l : L)
- if (!l.mtp) order.push_back(&l);
- for (Layer* lp : order) {
- Layer& l = *lp;
- const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;
- if (cum + lb / 2 > target && d + 1 < groups.size()) {
- ++d;
- target += avail[d] / avail_sum * need_sum;
- }
- l.grp = (int) d;
- cum += lb;
- }
- // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its
- // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage
- // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all
- if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {
- const int G = (int) groups.size(), NL = g.nl;
- std::vector<double> cap(groups.size());
- for (int gi = 0; gi < G; ++gi) {
- const int nm = (int) groups[(size_t) gi].mem.size();
- double mn = 1e300;
- for (int m : groups[(size_t) gi].mem)
- mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -
- 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)
- cap[(size_t) gi] = mn * nm;
- }
- for (const Layer& l : L)
- if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;
- cap[(size_t) G - 1] -= (double) head_b;
- std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense
- for (int il = 0; il < NL; ++il) {
- pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;
- pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;
- }
- auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi
- const int nm = (int) groups[(size_t) gi].mem.size();
- const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);
- return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;
- };
- // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)
- std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));
- std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));
- for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);
- for (int gi = 1; gi < G; ++gi)
- for (int b = gi + 1; b <= NL; ++b)
- for (int a = gi; a < b; ++a) {
- const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));
- if (v < best[(size_t) gi][(size_t) b]) {
- best[(size_t) gi][(size_t) b] = v;
- cut[(size_t) gi][(size_t) b] = a;
- }
- }
- int b = NL;
- for (int gi = G - 1; gi >= 0; --gi) {
- const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;
- for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;
- b = a;
- }
- }
- // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)
- if (const char* sp = std::getenv("DS4_SPLIT")) {
- std::vector<int> n;
- for (const char* q = sp; *q;) {
- n.push_back(std::atoi(q));
- while (*q && *q != ',') ++q;
- if (*q == ',') ++q;
- }
- int il = 0;
- for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)
- for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;
- for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;
- }
- for (Layer& l : L) {
- l.dev = groups[(size_t) l.grp].mem[0];
- l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)
- }
- head_dev = L[(size_t) g.nl - 1].dev;
- d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;
- }
- // the groups' members: copies of each layer's device state, a member's part of an expert
- for (Layer& l : L) {
- l.ff0 = 0;
- l.ffn = g.ff;
- l.ah0 = l.og0 = 0;
- l.ahn = g.heads;
- l.ogn = g.o_groups;
- if (l.nm == 1) continue;
- const int na = head_members(l.nm);
- if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {
- err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";
- return false;
- }
- for (int m = 1; m < l.nm; ++m) {
- Layer c = l;
- c.mates.clear();
- c.mi = m;
- c.dev = groups[(size_t) l.grp].mem[(size_t) m];
- l.mates.push_back(std::move(c));
- }
- // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even
- // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its
- // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)
- const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- const int ha = m - (l.nm - na); // (its place among the members with heads)
- x.ahn = ha < 0 ? 0 : g.heads / na;
- x.ah0 = ha < 0 ? 0 : ha * x.ahn;
- x.ogn = ha < 0 ? 0 : g.o_groups / na;
- x.og0 = ha < 0 ? 0 : ha * x.ogn;
- if (gguf_x) {
- x.ffn = g.ff / l.nm;
- x.ff0 = m * x.ffn;
- } else
- split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;
- x.hlay.K[r] = l.lay.K[r];
- x.hlay.trellis[r] = off;
- if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks
- const int ty = gg::expert_type(l.lay, r);
- if (k % gg::type_block(ty) != 0) {
- err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +
- " blocks do not split";
- return false;
- }
- x.hlay.suh[r] = x.hlay.svh[r] = off;
- off += align256((size_t) n * gg::type_row_bytes(ty, k));
- continue;
- }
- off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));
- x.hlay.suh[r] = off;
- off += align256((size_t) k * 2);
- x.hlay.svh[r] = off;
- off += align256((size_t) n * 2);
- }
- x.hlay.cb = l.lay.cb;
- x.hlay.fmt = l.lay.fmt;
- x.hblob = off;
- }
- }
- // which devices hold each source's compressed KV / index keys
- for (Src& s : srcs) {
- s.ckv.assign(devs.size(), nullptr);
- s.ik.assign(devs.size(), nullptr);
- }
- for (const Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m) {
- const Layer& l = member(l0, m);
- if (l.src >= 0) {
- srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;
- if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;
- }
- if (l.ix_src && l.ik_src >= 0) {
- srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;
- if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;
- }
- }
-
- // per device: the scratch, the caches, the dense weights
- for (size_t di = 0; di < devs.size(); ++di) {
- Dev& d = devs[di];
- const int D = g.dim, H = g.hc;
- const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;
- bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&
- dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&
- dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&
- dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&
- dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&
- dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&
- dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&
- dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&
- dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&
- dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&
- dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&
- dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&
- dalloc(d, d.rw, W * kPicks, err) &&
- dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&
- dalloc(d, d.moe_out, W * D, err) &&
- dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&
- dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&
- dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&
- dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&
- dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&
- dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
- dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
- dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
- if (ok && std::getenv("DS4_DBG_DEVH"))
- std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);
- if (!ok) return false;
- d.attn_scr.max_t = kAttnScratchT;
- d.attn_scr.cap = (int) topk_stride();
- if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;
- if (!g.eg_layers.empty()) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
- ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&
- dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);
- if (!ok) return false;
- }
- if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
- if (!halloc(d.bounce, bounce_bytes(), err)) return false;
- if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
- !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
- !dalloc(d, d.n_groups, 4, err))
- return false;
- if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;
- if (g.n_mtp > 0 && (int) di == head_dev &&
- (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))
- return false;
- CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));
- cudaSetDevice(d.id);
- CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));
- CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));
- for (Src& s : srcs) {
- if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)
- if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))
- return false;
- CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));
- CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));
- CK(cudaSetDevice(d.id));
- }
- if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))
- return false;
- if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))
- return false;
- }
- }
- for (Src& sr : srcs) {
- if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))
- return false;
- if (sr.remote && chunk > 0 &&
- (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||
- !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))
- return false;
- const int sg = L[(size_t) sr.layer].grp;
- for (size_t di = 0; di < devs.size(); ++di)
- if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);
- }
- for (Group& G : groups) {
- G.ar.nm = (int) G.mem.size();
- G.ar.slots = 4;
- G.ar.stride = (size_t) kWin * g.dim;
- if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||
- !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))
- return false;
- // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)
- if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||
- !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))
- return false;
- if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&
- (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||
- !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))
- return false;
- G.arb = G.ar; // (a chunk's: two slots of chunk rows)
- G.arb.slots = 2;
- G.arb.stride = (size_t) chunk * g.dim;
- if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||
- !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))
- return false;
- if (chunk > 0 && G.arb.nm > 2 &&
- (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||
- !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))
- return false;
- }
- if (g.n_mtp > 0) {
- if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;
- CK(cudaSetDevice(devs[(size_t) d_dev].id));
- CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));
- CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));
- }
- // the dense weights, each device's layers by a thread of its own
- {
- std::vector<std::string> errs(devs.size());
- std::vector<std::thread> th;
- for (size_t di = 0; di < devs.size(); ++di)
- th.emplace_back([this, di, &errs] {
- std::string e;
- if (!load_dense((int) di, e)) errs[di] = e;
- });
- for (auto& t : th) t.join();
- for (auto& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- }
- if (o.verbose)
- std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);
- // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E
- // host-mapped, the steps' slots
- // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)
- bool mk_rows = g.n_mtp > 0 && head_nm() > 1;
- for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;
- if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {
- const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");
- if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {
- err = "the markov embed: not a BF16 [vocab][rank] table";
- return false;
- }
- if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||
- !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {
- if (err.empty()) err = "the markov embed: read failed";
- return false;
- }
- devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)
- mk_split = true;
- }
-
- // the embedding table, pinned on the host
- {
- const Tensor* t = pk.get("embed.weight");
- const size_t nv = (size_t) g.vocab * g.dim;
- const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
- // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):
- // each row through ggml's to_float, to the same BF16 the host table holds
- const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;
- const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;
- const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;
- if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {
- err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";
- return false;
- }
- if (!halloc(embed_pinned, nv, err)) return false;
- if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)
- std::vector<uint8_t> row(erb);
- for (size_t r = 0; r < (size_t) g.vocab; ++r) {
- if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {
- err = "embed.weight: read failed";
- return false;
- }
- if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {
- err = "embed.weight: ggml has no decode for " + t->dtype;
- return false;
- }
- }
- } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
- std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
- for (size_t r = 0; r < (size_t) g.vocab; ++r) {
- if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
- err = "embed.weight: read failed";
- return false;
- }
- const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);
- for (size_t c = 0; c < (size_t) g.dim; ++c) {
- uint32_t u;
- std::memcpy(&u, &v[c], 4);
- u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even
- embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);
- }
- }
- } else if (!pk.read(*t, embed_pinned)) {
- err = "embed.weight: read failed";
- return false;
- }
- if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,
- "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",
- (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],
- (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],
- (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);
-
- }
- // the host side of the CPU lane and the Engram rows
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
- if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||
- !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||
- !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||
- !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))
- return false;
- if (pk.eg_layers > 0) {
- // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);
- // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
- const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
- const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
- int nth = (int) std::min(16u, std::max(4u, hw / 2));
- if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
- const uint64_t avail = ram_avail();
- uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
- if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);
- eg_cache.init((size_t) ((mb << 20) / row), row);
- std::vector<std::string> paths;
- for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
- eg_io.start(paths, nth);
- for (int li = 0; li < pk.eg_layers; ++li) {
- eg_win.emplace_back(new EgBatch());
- const std::string id = std::to_string(g.eg_layers[(size_t) li]);
- const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");
- // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read
- // as some other format's bytes - the old code then read the scale tensor of a pack that has none and
- // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.
- int fmt;
- if (tw == nullptr) {
- std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());
- std::exit(2);
- } else if (tw->dtype == "MXFP8") {
- fmt = 0;
- } else if (tw->dtype == "Q6_K") {
- fmt = 1;
- } else if (tw->dtype == "IQ4_NL") {
- fmt = 2;
- } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {
- fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)
- } else {
- std::fprintf(stderr,
- "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "
- "(MXFP8, Q6_K, IQ4_NL)\n",
- id.c_str(), tw->dtype.c_str());
- std::exit(2);
- }
- if (fmt == 1 && pk.eg_hd % 256 != 0) {
- err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +
- " values (not whole superblocks of 256)";
- return false;
- }
- if (fmt == 2 && pk.eg_hd % 32 != 0) {
- err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +
- " values (not whole blocks of 32)";
- return false;
- }
- eg_fmt.push_back((uint8_t) fmt);
- }
- if (o.verbose)
- std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
- "their layer; row cache %llu MB (%zu rows)\n",
- nth, (unsigned long long) mb, eg_cache.capacity());
- }
- prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);
- if (!o.profile.empty()) {
- std::FILE* f = std::fopen(o.profile.c_str(), "rb");
- if (f != nullptr) {
- std::vector<uint32_t> p(prof.size());
- const size_t n = std::fread(p.data(), 4, p.size(), f);
- if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());
- std::fclose(f);
- }
- }
-
- // the chunked prompt path's buffers (before the experts: they take what is left)
- if (chunk > 0 && !alloc_chunks(err)) return false;
- // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM
- // now leaves over
- if (file_experts) {
- double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- size_t fr = 0, tot = 0;
- CK(cudaMemGetInfo(&fr, &tot));
- room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;
- }
- for (const Layer& l : L) all += (double) l.blob * l.n_exp;
- const double need = std::max(0.0, all - room);
- const uint64_t avail = ram_avail();
- warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
- if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
- if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)
- std::fprintf(stderr,
- "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
- "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
- need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"
- : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",
- avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);
- }
- // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in
- // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)
- cores = node_cores();
- NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());
- if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;
- if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)
- {
- const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;
- c_xh.resize((size_t) NH * S * 2 * E);
- c_xsb.resize((size_t) NH * S * 2 * (E / 128));
- c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));
- c_xhd.resize(S * FF);
- c_xsd.resize(S * (FF / 128));
- c_accd.resize((size_t) NH * S * DQ * (E / NH));
- c_yd.resize(S * E);
- }
- // the experts: VRAM slots in what each device has left, the rest on the host
- if (!place_experts(err)) return false;
- if (!load_experts(err)) return false;
- if (pk.gguf) make_pools();
- if (chunk > 0) {
- // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
- const double tr = now_ms();
- for (auto& m : host_maps)
- if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {
- err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";
- return false;
- }
- if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);
- if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)
- size_t mr = 0;
- for (const Layer& l : L)
- if (!l.mtp) {
- for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);
- if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)
- mr = std::max(mr, l.blob);
- for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));
- }
- }
- const char* nr = std::getenv("DS4_READERS");
- const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);
- const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;
- // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the
- // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the
- // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)
- const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;
- for (size_t di = 0; di < chunks.size(); ++di) {
- auto& cp = chunks[di];
- if (cp == nullptr || cp->ring == nullptr) continue;
- cp->stage_stride = (mr + 4095) & ~(size_t) 4095;
- // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)
- bool fl = flow_on;
- int n_exp_max = 0;
- for (const Layer& l : L)
- if (!l.mtp)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di) {
- // (a group's members: each its share of the down's blocks in every row)
- fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&
- (l.nm == 1 || (flow_tp && (g.ff / l.nm) %
- std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));
- n_exp_max = std::max(n_exp_max, l.n_exp);
- }
- if (fl) // (each part read unbuffered from its offset rounded down: a page more each)
- cp->stage_stride = std::max(cp->stage_stride, flow_stride());
- cp->S = cp->ring_n;
- if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;
- cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);
- if (!fl) continue;
- cudaSetDevice(devs[di].id);
- cp->flow = true;
-#if defined(STRATA_METAL)
- cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;
-#endif
- cp->dev = (int) di;
- cp->io = std::make_unique<IoQueue>(n_readers);
- cp->stage_item.assign((size_t) cp->S, -1);
- cp->stage_ev.resize((size_t) cp->S);
- cp->ring_fill.resize((size_t) cp->ring_n);
- cp->ring_free.resize((size_t) cp->ring_n);
- for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})
- for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
- // (a region is written again only bc_n batches later - more than a layer's items, so after that
- // layer's bounds wait, which every copy from it precedes)
- cp->bc_n = n_exp_max + 2;
- cp->bc_w = cp->ring_n + 1;
- if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||
- !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))
- return false;
- }
- // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one
- // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each
- // window, a page fault at a time) - DS4_CHUNK_MIN still overrides
- bool all_flow = true;
- for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);
- if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;
- }
- for (Layer& l : L) {
- if (l.mtp) continue;
- for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)
- Layer& x = member(l, m);
- if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;
- if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;
- }
- }
- }
- {
- void* m = nullptr;
- if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory for the decode path's requests";
- return false;
- }
- host_allocs.push_back(m);
- reqs = (Req*) m;
- if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
- err = "host-mapped memory for the decode path's answers";
- return false;
- }
- host_allocs.push_back(m);
- resps = (Resp*) m;
- std::memset((void*) reqs, 0, sizeof(Req) * kRing);
- std::memset((void*) resps, 0, sizeof(Resp) * kRing);
- for (auto& x : layer_of) x.store(-1);
- ahead_fl.assign(L.size() * 1024, 0);
- ahead_rank.assign(L.size() * 1024, 0);
- ahead_for.assign(L.size(), 0);
- const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;
- service = std::thread([this, core] {
- if (core >= 0) pin_self(core);
- serve_lane();
- });
- if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });
- }
- if (o.expert_cache) {
- size_t maxb = 0;
- for (const Layer& l : L) maxb = std::max(maxb, l.blob);
- const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));
- sw_bufs.resize((size_t) nt);
- for (SwapBufs& B : sw_bufs) {
- if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStream_t cs = nullptr;
- CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
- B.st.push_back(cs);
- }
- }
- for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- CK(cudaStreamSynchronize(d.st));
- }
- if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)
- for (size_t di = 0; di < devs.size(); ++di) {
- workers.push_back(std::make_unique<Worker>());
- workers.back()->start();
- }
- calibrate_paths();
- init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)
- preload_store();
- if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
- reset();
- return true;
- }
-
-
- // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed
- // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)
- size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }
- size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
- size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
-
- // ---- the chunked prompt path
- size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
- size_t chunk_hand_bytes() const {
- return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
- }
-
- bool alloc_chunks(std::string& err) {
- chunks.resize(devs.size());
- const int D = g.dim, H = g.hc, HD = g.hd;
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- for (size_t di = 0; di < devs.size(); ++di) {
- Dev& d = devs[di];
- cudaSetDevice(d.id);
- chunks[di] = std::make_unique<Chunk>();
- Chunk& c = *chunks[di];
- const int nm = (int) groups[(size_t) d.grp].mem.size();
- const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);
- int FF = 0, ff0_ = 0;
- split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
- const size_t C = (size_t) chunk;
- c.C = chunk;
- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);
- // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
- c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
- c.w16_elems = (size_t) 8 << 20;
- bool has_eg = false, has_first = false;
- size_t stride = 0;
- for (const Layer& l : L)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di && !l.mtp) {
- has_eg = has_eg || l.eg >= 0;
- has_first = has_first || l.il == 0;
- stride = std::max(stride, align256(unit(member(l, m))));
- }
- const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;
- bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&
- dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&
- dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&
- dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&
- dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&
- dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&
- dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&
- dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&
- dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&
- dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&
- dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&
- dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&
- dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&
- dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&
- dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&
- dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
- dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
- dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
- dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
- halloc(c.bounds_pin, 2048, err);
- if (!ok) return false;
- if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
- if (has_eg) {
- const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);
- if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||
- !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))
- return false;
- }
- if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches
- const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;
- c.ring_stride = stride;
- c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);
- if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;
- }
- CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));
- for (int i = 0; i < Chunk::kParts; ++i) {
- CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));
- }
- if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {
- err = "cuBLAS on CUDA" + std::to_string(d.id);
- return false;
- }
- for (int sl = 0; sl < 2; ++sl) {
- if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;
- if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;
- if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;
- CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));
- CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));
- }
- if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;
- }
- return true;
- }
-
- // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS
- bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
- std::string& err) {
- if (rows <= 0) return true;
- const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
- for (int c0 = 0; c0 < m.n; c0 += cols) {
- const int nc = std::min(cols, m.n - c0);
- if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
- else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
- const float one = 1.0f;
- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
- m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,
- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
- if (st != CUBLAS_STATUS_SUCCESS) {
- err = "cublasGemmEx: status " + std::to_string((int) st);
- return false;
- }
- }
- return true;
- }
- // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)
- bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,
- float* y, int ldy, std::string& err) {
- const float zero = 0.0f;
- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,
- x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT_TENSOR_OP);
- if (st != CUBLAS_STATUS_SUCCESS) {
- err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);
- return false;
- }
- return true;
- }
- // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale
- bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {
- const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
-#if defined(STRATA_METAL)
- if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass
- hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);
- return true;
- }
-#endif
- const float one = 1.0f, zero = 0.0f;
- if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=
- CUBLAS_STATUS_SUCCESS) {
- err = "cublasSgemm (mHC mixes)";
- return false;
- }
- hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);
- return true;
- }
-
- // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full
- // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file
- // region read into pinned staging)
- bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,
- const uint8_t* src = nullptr) {
- if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into
- // pinned staging as its blob) or from the file's mapping
- auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };
- bool ok = true;
- if (l.nm == 1) {
- if (src != nullptr)
- ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- else
- for (int r = 0; r < 3; ++r)
- ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),
- cudaMemcpyHostToDevice, cs) == cudaSuccess;
- } else {
- const int ffh = g.ff / l.nm;
- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
- for (int r = 0; r < 2; ++r)
- ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,
- (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,
- (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- }
- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- return ok;
- }
- const auto& parts = l.hh[(size_t) e];
- const int D = g.dim, FF = g.ff;
- auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {
- return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- };
- auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {
- return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;
- };
- bool ok = true;
- if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets
- const uint8_t* p = src != nullptr ? src : parts[0];
- if (l.nm == 1) {
- for (int r = 0; r < 3; ++r)
- ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&
- cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&
- cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);
- } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;
- ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,
- nt * tile, nth * tile, (size_t) D / 16) &&
- cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&
- cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];
- ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,
- (size_t) (fn / 16) * row) &&
- cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&
- cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);
- }
- } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh
- for (int X = 0; X < 2; ++X) {
- ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,
- (size_t) (k / 16));
- ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);
- }
- ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
- }
- } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of
- // both parts' columns
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)
- for (int X = 0; X < 2; ++X) {
- const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);
- if (a >= b) continue;
- ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,
- (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&
- cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,
- (size_t) (b - a) * 2);
- }
- ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);
- }
- const size_t tile = (size_t) 32 * l.lay.K[2];
- const int nh = D / 2;
- for (int X = 0; X < 2; ++X) {
- ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,
- parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,
- (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));
- ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);
- }
- ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- return ok;
- }
-
- // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk
- // in layer order; the copies run at most kParts batches ahead of the compute
- struct Batch {
- int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range
- };
- struct Streamer {
- std::vector<Batch> b;
- size_t cp = 0, kp = 0; // the next batch to copy, to compute
- std::atomic<size_t> done{0}; // cp, for read_ahead's thread
- };
- // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel
- // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -
- // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone
- void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {
- static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *
- 1048576.0;
- std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied
- double ahead = 0;
- for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {
- const size_t done = S.done.load(std::memory_order_relaxed);
- while (!issued.empty() && issued.front().first < done) {
- ahead -= issued.front().second;
- issued.pop_front();
- }
- if (ahead > cap) {
- std::this_thread::sleep_for(std::chrono::microseconds(500));
- continue;
- }
- const Batch& bt = S.b[q];
- const Layer& l0 = L[(size_t) bt.il];
- double nb = 0;
- if (l0.dev == di)
- for (int i = 0; i < bt.n; ++i) {
- const int e = l0.host_list[(size_t) (bt.j0 + i)];
- if (l0.lay.fmt == gg::kFmtMXFP4) {
- if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);
- nb += (double) l0.blob;
- continue;
- }
- os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);
- nb += (double) l0.region_bytes[(size_t) e];
- }
- issued.push_back({q++, nb});
- ahead += nb;
- }
- }
- bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {
- const int P = Chunk::kParts;
- while (S.cp < S.b.size() && S.cp < S.kp + P) {
- const Batch& bt = S.b[S.cp];
- const int half = (int) (S.cp % P);
- const Layer& l0 = L[(size_t) bt.il];
- int mi = 0;
- for (int m = 0; m < l0.nm; ++m)
- if (member(l0, m).dev == di) mi = m;
- if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));
- uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots
- if (c.stage != nullptr) {
- if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)
- sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;
- std::atomic<bool> bad{false};
- c.readers->run(bt.n, [&](int i) {
- const int e = l0.host_list[(size_t) (bt.j0 + i)];
- if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)
- : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],
- l0.region_bytes[(size_t) e],
- sb + (size_t) i * c.stage_stride))
- bad.store(true);
- });
- if (bad.load()) {
- err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";
- return false;
- }
- }
- for (int i = 0; i < bt.n; ++i) {
- uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;
- if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,
- sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))
- return false;
- c.stream_bytes += (double) unit(member(l0, mi));
- }
- CK(cudaEventRecord(c.filled[half], c.cs));
- ++S.cp;
- S.done.store(S.cp, std::memory_order_relaxed);
- }
- return true;
- }
-
- // ---- the flow (see Chunk)
- // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute
- // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing
- static void kick(cudaStream_t s) {
-#if defined(_WIN32) && defined(STRATA_USE_HIP)
- (void) cudaStreamQuery(s);
- (void) cudaGetLastError();
-#else
- (void) s;
-#endif
- }
- size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
- return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
- }
- size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }
- static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }
- // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from
- // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of
- // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole
- size_t mx_read_off(const Layer& l, int m, int r) const {
- return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : 0;
- }
- size_t mx_read_len(const Layer& l, int r) const {
- return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
- : mx_part_bytes(l, r);
- }
- // the device's member index in L0's group (0 alone)
- int flow_mi(const Chunk& c, const Layer& L0) const {
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == c.dev) return m;
- return 0;
- }
- void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {
- c.items.emplace_back();
- Chunk::Item& it = c.items.back();
- it.il = l.il;
- it.mi = mi;
- it.k = k;
- it.e = e;
- it.v = v;
- bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;
- const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)
- if (!in_ram && store && l.nm == 1) {
- const int ss = store->find(l.il, e);
- if (ss >= 0) {
- store_blob = store->blob(ss);
- in_ram = true;
- }
- }
- if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)
- const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];
- for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);
- ++c.it_ram;
- return;
- }
- if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)
- it.ram = store_blob != nullptr ? store_blob
- : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];
- bool hit = false;
- if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after
- const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;
- if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again
- c.lc_il = l.il; // once that layer's items are all read - `after` below)
- c.lc_used = 0;
- c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);
- c.lc_first = i;
- std::fill(c.lc_of.begin(), c.lc_of.end(), -1);
- }
- const int j = c.lc_of[(size_t) e];
- if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)
- it.ram = c.lc + (size_t) j * stride;
- it.after = c.lc_writer[(size_t) j];
- hit = true;
- } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)
- const int jn = c.lc_used++;
- c.lc_of[(size_t) e] = jn;
- c.lc_writer[(size_t) jn] = (int64_t) i;
- it.keep = c.lc + (size_t) jn * stride;
- it.after = (int64_t) c.lc_first - 1;
- }
- }
- if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)
- it.direct = true;
- it.stage = -1;
- it.left.store(3, std::memory_order_relaxed);
- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
- return;
- }
- it.stage = (int) (c.n_file++ % (uint64_t) c.S);
- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
- for (int r = 0; r < 3; ++r) {
- const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);
- it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);
- sb += flow_region(r);
- }
- it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);
- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
- }
- // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance
- // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the
- // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -
- // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is
- // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer
- // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the
- // rest of its picks are streamed in)
- // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same
- // profile and the same routing - and each reads only its part of every expert)
- void flow_spec(Chunk& c, int k, int il, int T) {
- static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;
- Layer& l = L[(size_t) il];
- const int mi = flow_mi(c, l);
- int* vs = member(l, mi).vs_pin;
- c.spec_k = k;
- c.spec_il = il;
- c.spec_begin = c.base + c.items.size();
- const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += pr[e];
- std::vector<std::pair<double, int>> cand;
- const double picks = (double) l.k_exp * T;
- // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE
- // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole
- // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376
- // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and
- // 21 of 1568, the early reads 9-12% faster there)
- static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;
- const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;
- for (int e : l.host_list) {
- if (!early) break;
- const double p = sum > 0 ? pr[e] / sum : 0.0;
- const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));
- if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {
- return a.first > b.first;
- });
- if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));
- std::vector<char> is((size_t) l.n_exp, 0);
- int j = 0;
- for (const auto& x : cand) {
- vs[x.second] = l.n_slots + j++;
- is[(size_t) x.second] = 1;
- }
- for (int e : l.host_list)
- if (!is[(size_t) e]) vs[e] = l.n_slots + j++;
- for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);
- c.n_spec = (int) cand.size();
- c.it_spec += cand.size();
- }
- // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones
- void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {
- const Layer& l = L[(size_t) il];
- const int mi = flow_mi(c, l);
- const int* vs = member(l, mi).vs_pin;
- for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {
- const int v = fit(c, i).v;
- if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;
- }
- for (int e : l.host_list) { // (index order: their virtual slots ascending)
- const int v = vs[e];
- if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
- }
- }
- // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)
- bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
- const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
- const size_t i = (size_t) (it - c.fl_layers.begin());
- if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
- if (k + 1 < (int) c.fl_T.size()) {
- k2 = k + 1;
- il2 = il;
- return true;
- }
- if (i + 1 >= c.fl_layers.size()) return false;
- k2 = 0;
- il2 = c.fl_layers[i + 1];
- return true;
- }
- if (i + 1 < c.fl_layers.size()) {
- k2 = k;
- il2 = c.fl_layers[i + 1];
- return true;
- }
- if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {
- k2 = k + 1;
- il2 = c.fl_layers[0];
- return true;
- }
- return false;
- }
- // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each
- // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now
- bool flow_pump(Chunk& c, std::string& err) {
- const size_t end = c.base + c.items.size();
- IoQueue::Job jobs[6];
- while (c.n_read < end) {
- Chunk::Item& it = fit(c, c.n_read);
- if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot
- // written again once the layer before's items are all read (none still copies out of it)
- bool wait = false;
- if (it.keep == nullptr)
- wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;
- else {
- while (c.lc_chk < c.n_read &&
- (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))
- ++c.lc_chk;
- wait = c.lc_chk <= (size_t) it.after;
- }
- if (wait) break;
- }
- if (it.direct) {
- const size_t R = (size_t) c.ring_n, r = c.n_read % R;
- if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it
- if (c.n_comp <= c.n_read - R) break;
- CK(cudaEventSynchronize(c.ring_free[r]));
- }
- const Layer& l = L[(size_t) it.il];
-#if defined(STRATA_METAL)
- uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)
-#else
- uint8_t* dst = c.ring + r * c.ring_stride;
-#endif
- const x3::ExpertLayout& lay = l.lay;
- int nj = 0;
- for (int q = 0; q < 3; ++q) {
- IoQueue::Job& j = jobs[nj++];
- j = IoQueue::Job{};
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];
- j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];
- j.len = mx_part_bytes(l, q);
- j.dst = dst + lay.trellis[q];
- j.exact = true;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;
- j.map = pk.at(l.mx_file[q], j.off);
- j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;
- }
- c.io->push(jobs, nj);
- } else if (it.stage >= 0) {
- const int64_t prev = c.stage_item[(size_t) it.stage];
- if (prev >= 0) {
- if ((size_t) prev >= c.n_dma) break;
- const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);
- if (q == cudaErrorNotReady) break;
- CK(q);
- }
- c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;
- const Layer& l = L[(size_t) it.il];
- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
- const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)
- int nj = 0;
- for (int r = 0; r < 3; ++r) {
- if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X) {
- IoQueue::Job& j = jobs[nj++];
- j = IoQueue::Job{};
- j.len = nb;
- j.dst = sb + (size_t) X * nb;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
- }
- sb += flow_region(r);
- continue;
- }
- IoQueue::Job& j = jobs[nj++];
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
- j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);
- j.len = mx_read_len(l, r);
- j.dst = sb;
- j.left = &it.left;
- j.bad = &it.bad;
- j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;
- j.map = pk.at(l.mx_file[r], j.off);
- j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;
- sb += flow_region(r);
- }
- c.io->push(jobs, nj);
- }
- ++c.n_read;
- }
- const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;
- struct Kick { // (every way out: what was queued is submitted)
- cudaStream_t s;
- const size_t &a, b;
- ~Kick() {
- if (a != b) kick(s);
- }
- } kq{c.cs, c.n_dma, dma0};
- while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {
- Chunk::Item& it = fit(c, c.n_dma);
- if (it.left.load(std::memory_order_acquire) != 0) break;
- if (it.bad.load(std::memory_order_acquire)) {
- err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +
- " from the model's files failed";
- return false;
- }
- const size_t r = c.n_dma % R;
- if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));
- const Layer& l = L[(size_t) it.il];
- uint8_t* dst = c.ring + r * c.ring_stride;
- const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;
- if (it.direct) { // (already in the slot)
- CK(cudaEventRecord(c.ring_fill[r], c.cs));
- c.stream_bytes += (double) unit(l);
- ++c.n_dma;
- continue;
- }
- for (int p = 0; p < 2; ++p)
- CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));
- if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)
- const int t2 = gg::expert_type(l.lay, 2);
- const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);
- CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,
- cudaMemcpyHostToDevice, c.cs));
- } else
- CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));
- CK(cudaEventRecord(c.ring_fill[r], c.cs));
- if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));
- c.stream_bytes += (double) unit(l);
- ++c.n_dma;
- }
- return true;
- }
- // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM
- // ones in shares between them (the device works while the reads run)
- bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {
- cudaStream_t s = d.st;
- const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)
- if (c.spec_k != k || c.spec_il != L0.il) {
- err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";
- return false;
- }
- flow_rest(c, k, L0.il, b);
- const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
- int k2 = 0, il2 = 0;
- if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);
- const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
- int v_done = 0;
- while (c.n_comp < le) {
- // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what
- // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so
- // the sums come out the same on every run whatever the reads' timing
- const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});
- if (!flow_pump(c, err)) return false;
- if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy
- const double tw = now_ms();
- kick(s);
- if (c.n_read > c.n_dma) c.io->wait_any(500);
- else {
- const Chunk::Item& it = fit(c, c.n_read);
- if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));
- }
- c.wait_ms += now_ms() - tw;
- continue;
- }
- const int n = (int) (i1 - i0);
- int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
- int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
- ++c.n_batch;
- // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)
- for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];
- bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];
- if (bc[n] > bc[0]) {
- CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));
- CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));
- if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))
- return false;
- }
- for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));
- kick(s);
- c.n_comp = i1;
- if (!flow_pump(c, err)) return false;
- const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));
- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;
- v_done = v_to;
- }
- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
- while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)
- c.items.pop_front();
- ++c.base;
- }
- return true;
- }
-
- // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)
- // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order
- bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,
- const int* b, int i0, int i1, int T, std::string& err) {
- const int D = g.dim, FF = l.ffn;
- while (i0 < i1) {
- int j = i0 + 1;
- while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;
- const int a = b[i0], nrows = b[j] - a;
- int max_rows = 0;
- for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);
- if (nrows > 0) {
- x3::SetArgs sa;
- sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)
- sa.stride = stride;
- sa.lay = l.nm > 1 ? l.hlay : l.lay;
- sa.n_exp = j - i0;
- sa.bounds = d_bounds + i0;
- sa.row0 = a;
- sa.max_rows = max_rows;
- sa.hbounds = b + i0;
- sa.total_rows = nrows;
- sa.n_embd = D;
- sa.n_ff = FF;
- sa.limit = g.lim;
- if (sa.lay.fmt == gg::kFmtMXFP4) {
- gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
- gg::set_down(sa, c.h16, FF, c.y, D, d.st);
- } else {
- x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
- x3::set_down(sa, c.h16, FF, c.y, D, d.st);
- }
- if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {
- std::vector<int> bb(b + i0, b + j + 1);
- std::vector<uint16_t> hv((size_t) nrows * FF);
- std::vector<float> yv((size_t) nrows * D);
- CK(cudaStreamSynchronize(d.st));
- CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&
- wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};
- std::fwrite(hdr, 8, 6, dump_y);
- std::fwrite(bb.data(), 4, bb.size(), dump_y);
- std::fwrite(hv.data(), 2, hv.size(), dump_y);
- std::fwrite(yv.data(), 4, yv.size(), dump_y);
- std::fflush(dump_y);
- }
- moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);
- }
- i0 = j;
- }
- (void) err;
- return true;
- }
-
- bool load_dense(int di, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- size_t* acct = &d.weights;
- for (Layer& l0 : L)
- for (int mi = 0; mi < l0.nm; ++mi) {
- Layer& l = member(l0, mi);
- if (l.dev != di) continue;
- // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,
- // of the shared expert's FFN; the rest whole)
- const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);
- const std::string& P = l.name;
- const std::string A = P + "attn.";
- bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&
- (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&
- mat(d, A + "wkv", l.wkv, err, acct) &&
- (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&
- f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&
- f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&
- f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&
- f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&
- f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&
- f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&
- f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&
- f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&
- f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&
- f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&
- f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&
- (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||
- f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&
- mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&
- mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&
- mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);
- if (!ok) return false;
- l.sink += (size_t) l.ah0;
- for (int gi = 0; gi < l.ogn; ++gi)
- if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;
- if (l.kv_src) {
- const std::string C = A + "compressor.";
- if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;
- if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;
- if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||
- !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))
- return false;
- }
- if (l.ix_src) {
- const std::string I = A + "indexer.";
- if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))
- return false;
- if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))
- return false;
- }
- if (l.eg >= 0) {
- if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;
- if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already
- if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;
- } else {
- float *qw = nullptr, *kw = nullptr;
- if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))
- return false;
- std::vector<float> a((size_t) g.hc * g.dim), b(a.size());
- CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));
- for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product
- CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));
- l.eg_w = qw;
- }
- }
- if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;
- if (g.n_mtp > 0 && !l.mtp) {
- if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;
- if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||
- !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))
- return false;
- }
- if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;
- }
- if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns
- const int hn = head_nm();
- if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))
- return false;
- if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))
- return false;
- if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;
- if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head
- const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");
- const int R = g.d_rank;
- head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)
- // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same
- // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)
- if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {
- std::vector<uint16_t> rows((size_t) d.mk_vn * R);
- if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {
- err = "the markov head: read failed";
- return false;
- }
- if (th->dtype == "BF16")
- for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));
- if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||
- !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))
- return false;
- CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));
- }
- }
- }
- if (di == head_dev) {
- head = d.head;
- norm_w = d.norm_w;
- d_norm = d.d_norm;
- if (g.n_mtp > 0) {
- const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";
- if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))
- return false;
- const Tensor* te = pk.get(Z + "markov_head.embed.weight");
- const Tensor* th = pk.get(Z + "markov_head.head.weight");
- if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {
- err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";
- return false;
- }
- if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
- !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))
- return false;
- }
- }
- if (g.n_mtp > 0 && di == d_dev) {
- if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||
- !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))
- return false;
- }
- return true;
- }
-
-#if defined(STRATA_METAL)
- // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays
- // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and
- // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most
- // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.
- double metal_expert_cap() const {
- uint64_t phys = 0;
- size_t len = sizeof phys;
- sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);
- unsigned long long pages = 0;
- size_t pl = sizeof pages;
- double floor_b = 0;
- if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;
- size_t fr = 0, tot = 0;
- cudaMemGetInfo(&fr, &tot);
- const double gb = 1e9, P = (double) phys;
- double keep = std::max(7.0 * gb, 0.15 * P);
- if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;
- double share = 0.40;
- if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);
- double leave = keep;
- if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});
- const double cap = std::min((double) tot, P - leave);
- task_vm_info_data_t ti;
- mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;
- double foot = 0;
- if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;
- return std::max(4.0 * gb, cap - foot - 1.0 * gb);
- }
-#endif
-#if defined(STRATA_METAL)
- // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it
- // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).
- bool zc_try(Layer& l, int e) {
- static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)
- if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;
- if (l.vm[(size_t) e][0].raw != nullptr) return true;
- uint64_t a[3];
- for (int r = 0; r < 3; ++r) {
- a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));
- if (a[r] == 0) return false;
- }
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m = x3::Mat{};
- m.K = gg::expert_type(l.lay, r);
- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
- m.raw = (const void*) a[r];
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- }
- return true;
- }
-#endif
- void set_vm(Layer& l, int e, int slot) {
- uint8_t* b = l.vram_base + (size_t) slot * l.blob;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m = x3::Mat{};
- m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)
- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
- m.raw = b + l.lay.trellis[r];
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- }
- return;
- }
- for (int r = 0; r < 3; ++r) {
- x3::Mat& m = l.vm[(size_t) e][(size_t) r];
- m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);
- m.suh = (const uint16_t*) (b + l.lay.suh[r]);
- m.svh = (const uint16_t*) (b + l.lay.svh[r]);
- m.k = r < 2 ? g.dim : g.ff;
- m.n = r < 2 ? g.ff : g.dim;
- m.K = l.lay.K[r];
- m.cb = 2;
- m.ld = 0;
- }
- }
- // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)
- void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- const uint8_t* tr = blob + l.lay.trellis[r];
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- (size_t) nth * tile);
- std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);
- std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
- }
- }
- void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {
- for (int r = 0; r < 3; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r];
- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
- uint8_t* tr = blob + l.lay.trellis[r];
- for (int X = 0; X < NH; ++X)
- for (int sl = 0; sl < k / 16; ++sl)
- std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
- parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);
- std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
- for (int X = 0; X < NH; ++X)
- std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],
- (size_t) l.hl.n[r] * 2);
- }
- }
-
- // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back
- void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
- if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
- std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,
- blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);
- std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);
- std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
- }
- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
- std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);
- std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
- std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);
- }
- void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
- if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);
- const Layer& x = member(l, m);
- const int f0 = x.ff0, fn = x.ffn;
- for (int r = 0; r < 2; ++r) {
- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
- std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,
- src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);
- if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);
- std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);
- }
- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
- std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);
- std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);
- if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);
- }
-
- bool place_experts(std::string& err) {
- // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)
- double room = 1e300, kept = 0;
- for (const Dev& d : devs)
- if (commit_charged(d)) room = commit_room();
- const double room0 = room;
- // per group: what its tightest member has left (each member holds its part of every VRAM expert of the
- // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round
- // the members layer by layer, so each member's total comes to about the mean's)
- auto ucount = [](const Layer& l) {
- double s = 0;
- for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));
- return s / l.nm;
- };
- for (size_t gi = 0; gi < groups.size(); ++gi) {
- double budget = 1e300;
- bool charged = false;
- for (int m : groups[gi].mem) {
- cudaSetDevice(devs[(size_t) m].id);
- size_t fr = 0, tot = 0;
- CK(cudaMemGetInfo(&fr, &tot));
- budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *
- o.vram_experts);
-#if defined(STRATA_METAL)
- if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {
- const double cap = metal_expert_cap();
- if (cap < budget) {
- std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",
- cap / 1e9, cap / 1e9);
- budget = cap;
- }
- }
-#endif
- charged = charged || commit_charged(devs[(size_t) m]);
- }
- if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)
- double all = 0;
- for (const Layer& l : L)
- if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;
- const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);
- kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;
- budget = capped;
- room = std::max(0.0, room - std::min(budget, all) * nm);
- }
- double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)
- for (Layer& l : L)
- if (l.grp == (int) gi && l.mtp) {
- l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));
- budget_t -= (double) l.n_vram * ucount(l);
- }
- double want = 0;
- for (const Layer& l : L)
- if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;
- const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
- for (Layer& l : L) {
- if (l.grp != (int) gi || l.mtp) continue;
- if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)
- l.n_vram = 0;
- continue;
- }
- l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
- // never more than fits: the floor of each layer's share can still round up the total
- while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
- }
- double used = 0;
- for (Layer& l : L)
- if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
- // what the floors left: whole experts, layer by layer
- for (Layer& l : L) {
- if (l.grp != (int) gi || l.cpu_only) continue;
- while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
- ++l.n_vram;
- used += (double) ucount(l);
- }
- }
- }
- if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it
- const os::Commit c = os::commit();
- const double gib = (double) (1ull << 30);
- const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;
- commit_kept = kept;
- pagefile_hint = std::ceil((pf + kept) / gib);
- std::fprintf(stderr,
- "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "
- "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "
- "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "
- "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",
- kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);
- } else if (room0 < 1e300 && o.verbose) {
- std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "
- "programs)\n", room0 / 1e9, commit_margin() / 1e9);
- }
- for (Layer& l : L) {
- // the hottest experts of the profile first, then by index
- std::vector<int> order((size_t) l.n_exp);
- std::iota(order.begin(), order.end(), 0);
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });
- l.slot.assign((size_t) l.n_exp, -1);
- l.host.assign((size_t) l.n_exp, nullptr);
- // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps
- l.n_slots = std::min(l.n_vram, l.n_exp);
- if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)
- l.n_vram = l.n_slots - 1;
- l.spare_vram = l.n_slots - 1;
- } else {
- l.n_vram = l.n_slots;
- l.spare_vram = -1;
- }
- for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;
- for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {
- Layer& x = member(l, m);
- Dev& d = devs[(size_t) x.dev];
- cudaSetDevice(d.id);
- void* v = nullptr;
- if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {
- err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +
- " experts on CUDA" + std::to_string(d.id) + " failed";
- return false;
- }
- d.allocs.push_back(v);
- d.experts += (size_t) l.n_slots * unit(x);
- x.vram_base = (uint8_t*) v;
- }
- l.heat.assign((size_t) l.n_exp, 0.0f);
- { // the profile as the starting heat: its counts scaled to ~512 hits a layer
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- const double sc = sum > 0 ? 512.0 / sum : 0.0;
- for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);
- }
- const int n_host = l.n_exp - l.n_vram;
- { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;
- l.hl.k[r] = k;
- l.hl.n[r] = n;
- l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;
- off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);
- l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;
- off += align256((size_t) k * 2);
- l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;
- off += align256((size_t) n * 2);
- }
- l.hl.bytes = off;
- }
- l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});
- if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);
- } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)
- for (int e = 0; e < l.n_exp; ++e)
- if (pk.map(l.region_file[(size_t) e]) == nullptr) {
- err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";
- return false;
- }
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) {
- l.hh[(size_t) e][0] = file_part(l, e);
- l.host[(size_t) e] = l.hh[(size_t) e][0];
- }
- } else if (n_host > 0) {
- const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)
- const size_t bytes = (size_t) n_parts * l.hl.bytes;
- for (int X = 0; X < NH; ++X) {
- void* m = node_map(bytes, X, NH);
- if (m == nullptr) {
- err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +
- " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";
- return false;
- }
- host_maps.emplace_back(m, bytes);
- int hi = 0;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;
- if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;
- }
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];
- }
- l.vm.assign((size_t) l.n_exp, {});
- if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);
- if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0)
- tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));
- cudaSetDevice(devs[(size_t) x.dev].id);
- CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));
- }
- }
- return true;
- }
-
- // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the
- // file: the mapping reads them on first use)
- bool load_experts_mx(std::string& err) {
- struct Job {
- int layer, e;
- };
- std::vector<Job> jobs;
- size_t maxb = 0;
- for (const Layer& l : L) {
- maxb = std::max(maxb, l.blob);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
- });
- std::atomic<size_t> next{0};
- std::atomic<bool> failed{false};
- std::mutex emu;
- std::string ferr;
- std::atomic<int64_t> done_bytes{0};
- const double t0 = now_ms();
- std::vector<std::thread> th;
- for (int t = 0; t < 12; ++t)
- th.emplace_back([&] {
- uint8_t *stage = nullptr, *stage2 = nullptr;
- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
- for (;;) {
- const size_t i = next.fetch_add(1);
- if (i >= jobs.size() || failed.load()) break;
- const Layer& l = L[(size_t) jobs[i].layer];
- const int e = jobs[i].e;
- bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);
- for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- if (l.nm > 1) blob_to_half(l, stage, m, stage2);
- ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,
- unit(l), cudaMemcpyHostToDevice) == cudaSuccess;
- }
- if (!ok) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- failed.store(true);
- break;
- }
- const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
- if (o.verbose && (i % 1024) == 0)
- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
- db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- if (stage) cudaFreeHost(stage);
- if (stage2) cudaFreeHost(stage2);
- });
- for (auto& t : th) t.join();
- if (failed.load()) {
- err = ferr;
- return false;
- }
- // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the
- // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them
- struct Warm {
- int layer, e;
- double share;
- };
- std::vector<Warm> warm;
- for (const Layer& l : L) {
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- for (int e = 0; e < l.n_exp; ++e) {
- if (l.slot[(size_t) e] >= 0) {
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);
- } else {
- warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});
- }
- }
- }
- std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });
- if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left
- std::vector<std::pair<int, int>> hot;
- for (const Warm& w : warm) hot.push_back({w.layer, w.e});
- if (!copy_experts(hot, err)) return false;
- if (copy_mode) {
- warm.erase(std::remove_if(warm.begin(), warm.end(),
- [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),
- warm.end());
- const uint64_t av = ram_avail();
- if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;
- }
- }
- if (ram_tier && warm_bytes > 0 && !warm.empty()) {
- // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for
- // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not
- // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on
- // Windows - the engine's own pinned buffers come after this
- const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));
- while (ram_slots.size() < want + 1) {
- const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());
- uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);
- if (c == nullptr) break;
- ram_chunks.push_back(c);
- ram_chunk_bytes.push_back(n * maxb);
- for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);
- }
- ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;
- for (Layer& l : L)
- if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);
- std::atomic<size_t> ri{0};
- std::atomic<bool> rfail{false};
- std::vector<std::thread> rt;
- for (int t = 0; t < 12 && ram_held > 0; ++t)
- rt.emplace_back([&] {
- for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {
- const Layer& l = L[(size_t) warm[i].layer];
- if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);
- for (int r = 0; r < 3; ++r)
- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],
- mx_part_bytes(l, r), false);
- }
- });
- for (auto& t : rt) t.join();
- if (rfail.load()) {
- err = "reading the RAM tier's experts from the model's files failed";
- return false;
- }
- for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;
- if (ram_held > 0) ram_free.push_back(ram_held);
- if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });
- if (o.verbose)
- std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "
- "read from the model's files (%.1f s)\n",
- ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,
- (now_ms() - t0) / 1000.0);
- return true;
- }
- size_t nw = 0;
- for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)
- b += (double) L[(size_t) warm[nw].layer].blob;
- std::atomic<size_t> wi{0};
- std::vector<std::thread> wt;
- for (int t = 0; t < 12 && nw > 0; ++t)
- wt.emplace_back([&] {
- std::vector<uint8_t> buf(maxb);
- for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());
- });
- for (auto& t : wt) t.join();
- if (o.verbose && nw > 0)
- std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,
- nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
- return true;
- }
-
- // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU
- // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the
- // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:
- // none - read in place: the page cache follows a conversation's experts where a static set (the profile's
- // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the
- // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file
- // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what
- // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place
- bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {
- if (o.mmap_experts > 0 || hot.empty()) return true;
- const double t0 = now_ms();
- auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };
- double all = 0;
- {
- std::vector<char> seen(L.size(), 0);
- for (const auto& h : hot) {
- const Layer& l = L[(size_t) h.first];
- all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));
- }
- }
- const uint64_t avail = ram_avail();
- // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
- const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),
- commit_room());
- // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
- // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
- // on 64 GB and on 32 GB of RAM)
- double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;
- if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;
- if (o.mmap_experts == 0 && all > can)
- std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "
- "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "
- "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
- if (budget <= 0) {
- if (o.verbose)
- std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "
- "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",
- all / 1e9, can / 1e9, kRamMargin / 1e9);
- return true;
- }
- // NUMA parts: whole rows of each matrix, so any row count the nodes divide
- int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));
- if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)
- nh = std::atoi(v) >= 2 ? 2 : 1;
- if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;
- NH = nh;
- cp_lay.assign(L.size(), {0, 0, 0, 0});
- for (const Layer& l : L) {
- size_t off = 0;
- for (int r = 0; r < 3; ++r) {
- cp_lay[(size_t) l.il][(size_t) r] = off;
- off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));
- }
- cp_lay[(size_t) l.il][3] = off;
- }
- // the hottest that fit the budget (a layer's first one brings its spare)
- std::vector<int> n_copy(L.size(), 0);
- double used = 0;
- size_t take = 0;
- for (; take < hot.size(); ++take) {
- const Layer& l = L[(size_t) hot[take].first];
- const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));
- if (used + add > budget) break;
- used += add;
- ++n_copy[(size_t) l.il];
- }
- // each layer's slots: one block per node of (copies + spare) parts
- std::vector<int> first(L.size(), -1);
- copy_spare.assign(L.size(), -1);
- for (Layer& l : L) {
- l.ram.assign((size_t) l.n_exp, -1);
- const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);
- if (ns == 0) continue;
- const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
- std::array<uint8_t*, 2> base = {nullptr, nullptr};
- bool ok = true;
- for (int X = 0; X < NH && ok; ++X) {
- base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
- ok = base[(size_t) X] != nullptr;
- if (ok) {
- ram_chunks.push_back(base[(size_t) X]);
- ram_chunk_bytes.push_back(bytes);
- }
- }
- if (!ok) {
- std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,
- ns, (double) bytes * NH / 1e9);
- n_copy[(size_t) l.il] = 0;
- continue;
- }
- first[(size_t) l.il] = (int) ram_part.size();
- for (int i = 0; i < ns; ++i)
- ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});
- if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;
- }
- struct Job {
- int layer, e, slot;
- };
- std::vector<Job> jobs;
- {
- std::vector<int> next = first;
- for (size_t i = 0; i < take; ++i) {
- const int il = hot[i].first;
- if (n_copy[(size_t) il] == 0) continue;
- jobs.push_back({il, hot[i].second, next[(size_t) il]++});
- }
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
- });
- std::atomic<size_t> ji{0};
- std::atomic<bool> bad{false};
- std::atomic<int64_t> done_b{0};
- std::vector<std::thread> th;
- for (int t = 0; t < 12 && !jobs.empty(); ++t)
- th.emplace_back([&] {
- for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {
- const Job& j = jobs[i];
- const Layer& l = L[(size_t) j.layer];
- for (int r = 0; r < 3; ++r) {
- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
- for (int X = 0; X < NH; ++X)
- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,
- ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))
- bad.store(true);
- }
- const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
- if (o.verbose && i % 2048 == 0)
- std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
- db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- });
- for (auto& t : th) t.join();
- if (bad.load()) {
- err = "copying the CPU lane's experts from the model's files failed";
- return false;
- }
- for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;
- copy_mode = true;
- ram_held = (int) jobs.size();
- double cb = 0;
- for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;
- std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",
- jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,
- jobs.size() < hot.size() ? "; the others read in place" : "");
- return true;
- }
-
- bool load_experts(std::string& err) {
- if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);
- // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place
- // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their
- // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are
- // dropped from it again
- struct Job {
- int layer, e;
- };
- std::vector<Job> jobs;
- for (const Layer& l : L)
- for (int e = 0; e < l.n_exp; ++e)
- if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
- uint64_t warm = 0;
- if (file_experts) {
- std::vector<std::pair<double, Job>> cand;
- for (const Layer& l : L) {
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0;
- for (int e = 0; e < l.n_exp; ++e) sum += p[e];
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
- for (const auto& c : cand) {
- const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];
- if (warm + b > warm_bytes) break;
- warm += b;
- jobs.push_back(c.second);
- }
- }
- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
- if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])
- return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];
- return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];
- });
- std::atomic<size_t> next{0};
- std::atomic<bool> failed{false};
- std::mutex emu;
- std::string ferr;
- const int nth = 12;
- size_t maxr = 0, maxb = 0;
- for (const Layer& l : L) {
- maxb = std::max(maxb, l.blob);
- for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);
- }
- std::atomic<int64_t> done_bytes{0};
- const double t0 = now_ms();
- std::vector<std::thread> th;
- for (int t = 0; t < nth; ++t)
- th.emplace_back([&] {
- std::vector<uint8_t> region(maxr);
- uint8_t *stage = nullptr, *stage2 = nullptr;
- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
- int cur_dev = -1;
- for (;;) {
- const size_t i = next.fetch_add(1);
- if (i >= jobs.size() || failed.load()) break;
- const Layer& l = L[(size_t) jobs[i].layer];
- const int e = jobs[i].e;
- if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],
- region.data())) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
- failed.store(true);
- break;
- }
- if (l.slot[(size_t) e] >= 0) {
- repack(l, region.data(), stage);
- bool up = true;
- for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)
- const Layer& x = member(l, m);
- const Dev& d = devs[(size_t) x.dev];
- if (cur_dev != d.id) {
- cudaSetDevice(d.id);
- cur_dev = d.id;
- }
- if (l.nm > 1) blob_to_half(l, stage, m, stage2);
- up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,
- unit(x), cudaMemcpyHostToDevice) == cudaSuccess;
- }
- if (!up) {
- std::lock_guard<std::mutex> lk(emu);
- ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);
- failed.store(true);
- break;
- }
- if (file_experts)
- os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);
- } else if (!file_experts) {
- for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);
- }
- const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +
- (int64_t) l.region_bytes[(size_t) e];
- if (o.verbose && (i % 1024) == 0)
- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),
- db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));
- }
- if (stage) cudaFreeHost(stage);
- if (stage2) cudaFreeHost(stage2);
- });
- for (auto& t : th) t.join();
- if (failed.load()) {
- err = ferr;
- return false;
- }
- if (file_experts)
- std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",
- (now_ms() - t0) / 1000.0, warm / 1e9);
- return true;
- }
-
- std::string describe() const {
- std::ostringstream s;
- for (size_t di = 0; di < devs.size(); ++di) {
- const Dev& d = devs[di];
- int a = -1, b = -1, ev = 0, et = 0;
- for (const Layer& l : L)
- for (int m = 0; m < l.nm; ++m)
- if (member(l, m).dev == (int) di) {
- if (a < 0) a = l.il;
- b = l.il;
- ev += l.n_vram;
- et += l.n_exp;
- }
- size_t fr = 0, tot = 0;
- cudaSetDevice(d.id);
- cudaMemGetInfo(&fr, &tot);
- const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();
- char tp[64] = "";
- if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);
- char line[384];
- std::snprintf(line, sizeof line,
- "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "
- "%.2f GB, %.0f MiB free\n",
- d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,
- d.caches / 1e9, fr / 1048576.0);
- s << line;
- }
- int nh = 0;
- size_t hb_ = 0;
- for (const Layer& l : L) {
- nh += l.n_exp - l.n_vram;
- hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;
- }
- if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)
- uint64_t h = 1469598103934665603ull;
- for (const Layer& l : L)
- for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }
- char hl[96];
- std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);
- s << hl;
- }
- char line[200];
- if (copy_mode)
- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "
- "parts), the rest in place in the model's files; %d threads\n",
- nh, hb_ / 1e9, ram_held, NH, cpu_threads());
- else if (file_experts)
- std::snprintf(line, sizeof line,
- "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",
- nh, hb_ / 1e9, cpu_threads());
- else
- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,
- hb_ / 1e9, NH, cpu_threads());
- s << line;
- return s.str();
- }
-
- double store_gb() const {
- static double cached = -2.0; // (decided once: the layout of the blobs depends on it)
- if (cached > -1.5) return cached;
- double gb = o.ssd_cache_gb;
- if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);
- if (!pk.gguf) gb = 0.0;
- else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most
- const uint64_t avail = ram_avail();
- const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;
- gb = room >= 4.0 ? std::min(48.0, room) : 0.0;
- }
- return cached = gb;
- }
- void init_store() {
- const double gb = store_gb();
- if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;
- std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- ExpertStore::LayerGeo& gl = geo[(size_t) l.il];
- gl.n_exp = l.n_exp;
- for (int r = 0; r < 3; ++r) {
- gl.file[r] = l.mx_file[r];
- gl.off[r] = l.mx_off[r];
- gl.stride[r] = l.mx_stride[r];
- gl.bytes[r] = mx_part_bytes(l, r);
- gl.tr[r] = l.lay.trellis[r];
- }
- gl.blob = l.blob;
- }
- ExpertStore::Cfg cfg;
- cfg.budget_bytes = (size_t) (gb * 1e9);
- cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
- cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
- cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
- cfg.ram_avail = [] { return ram_avail(); };
- cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
- cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;
- // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a
- // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane
- // took which expert, the CPU lane's are the mmap path's byte for byte)
- const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&
- L[0].nm == 1;
- if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)
- const int dev0 = devs[0].id;
- cfg.map_chunk = [dev0](void* h, size_t n) -> void* {
- cudaSetDevice(dev0);
- if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {
- (void) cudaGetLastError();
- return nullptr;
- }
- void* d = nullptr;
- if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {
- (void) cudaGetLastError();
- cudaHostUnregister(h);
- return nullptr;
- }
- if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);
- return d;
- };
- cfg.unmap_chunk = [dev0](void* h) {
- cudaSetDevice(dev0);
- cudaHostUnregister(h);
- };
- }
- bool direct = true;
- for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
- auto st = std::make_unique<ExpertStore>();
- if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {
- std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
- return;
- }
- store = std::move(st);
- for (Layer& l : L)
- if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);
- std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
- store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
- (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
- if (want_zc) {
- if (!store->gpu_mapped()) {
- std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");
- for (Layer& l : L) l.zc.clear();
- } else {
- zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;
- cudaSetDevice(devs[0].id);
- if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {
- (void) cudaGetLastError();
- zc_stage_buf = nullptr;
- for (Layer& l : L) l.zc.clear();
- } else {
- for (Layer& l : L)
- if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);
- else if (!l.zc.empty()) {
- (void) cudaGetLastError();
- l.zc.clear();
- }
- }
- }
- std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",
- zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
- }
- }
- // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)
- void preload_store() {
- if (!store) return;
- if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;
- std::vector<std::pair<double, std::pair<int, int>>> cand;
- double tot = 0;
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e) tot += p[e];
- }
- if (tot <= 0) return; // (no profile yet: the first tokens fill it)
- for (const Layer& l : L) {
- if (l.il >= g.nl) continue;
- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
- }
- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
- std::vector<std::pair<int, int>> list;
- list.reserve(cand.size());
- for (const auto& c : cand) list.push_back(c.second);
- const double t0 = now_ms();
- const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;
- const size_t n = store->preload(list, leave);
- std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,
- (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
- }
- // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
- // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
- void store_sync() {
- if (!store || !store->gpu_mapped()) return;
- std::vector<ExpertStore::Vis> ev;
- store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });
- if (ev.empty()) return;
- std::vector<char> dirty(L.size(), 0);
- for (const auto& v : ev) {
- Layer& l = L[(size_t) v.layer];
- if (l.zc.empty()) continue;
- l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;
- dirty[(size_t) v.layer] = 1;
- }
- for (Layer& l : L) {
- if (!dirty[(size_t) l.il]) continue;
- if (l.zc_tab == nullptr) continue;
- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
- for (int e = 0; e < l.n_exp; ++e)
- if (l.zc[(size_t) e]) {
- const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;
- tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;
- if (s < 0) l.zc[(size_t) e] = 0;
- }
- cudaSetDevice(devs[0].id);
- cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);
- }
- }
-
- int cpu_threads() const {
- int n = 0;
- for (const auto& pl : pools) n += pl->size() - 1;
- return n + 1;
- }
- // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over
- // every node's cores (the pages lie where the cache put them)
- void make_pools() {
- std::vector<std::vector<int>> cs = cores;
- if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)
- std::vector<int> all = cs[0];
- cs.assign((size_t) NH, {});
- for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);
- }
- for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {
- std::vector<int> c = cs[(size_t) X];
- if (file_experts && NH == 1)
- for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());
- if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)
- int nw = (int) c.size();
- if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));
- pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)
- }
- (void) gg::mx_tier(); // (measured here, not on the first token)
- if (o.verbose) {
- int nthr = 0;
- for (auto& p : pools) nthr += p->size();
- std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),
- gg::mx_tier_report());
- }
- }
-
- void reset() {
- pos = 0;
- toks.clear();
- ctoks.clear();
- ck[0].pos = ck[1].pos = -1;
- for (Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m) {
- Layer& l = member(l0, m);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);
- if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);
- if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- }
-
- // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
- // state is indexed by position, and what lies past a restored position is written again)
- void checkpoint(int slot) {
- Check& c = ck[slot & 1];
- c.pos = pos;
- c.win.resize(L.size());
- c.comp.resize(L.size() * 2);
- for (size_t i = 0; i < L.size(); ++i) {
- Layer& l = L[i];
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- c.win[i].resize((size_t) g.win * win_rb());
- cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);
- if (l.c_kv) {
- c.comp[2 * i].resize((size_t) l.ratio * g.hd);
- c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
- cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);
- cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,
- cudaMemcpyDeviceToHost, d.st);
- }
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- }
-
- // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
- int best_check(int64_t limit) const {
- int b = -1;
- for (int i = 0; i < 2; ++i)
- if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
- return b;
- }
- int64_t restorable(int64_t limit) const {
- const int b = best_check(limit);
- return b < 0 ? -1 : ck[b].pos;
- }
-
- bool restore(int64_t limit) {
- const int b = best_check(limit);
- if (b < 0) return false;
- const Check& c = ck[b];
- for (size_t i = 0; i < L.size(); ++i)
- for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)
- Layer& l = member(L[i], m);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);
- if (l.c_kv) {
- cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);
- cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,
- cudaMemcpyHostToDevice, d.st);
- }
- }
- for (Dev& d : devs) {
- cudaSetDevice(d.id);
- cudaStreamSynchronize(d.st);
- }
- pos = c.pos;
- toks.resize((size_t) pos);
- ctoks.resize((size_t) pos);
- for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
- if (o2.pos > pos) o2.pos = -1;
- return true;
- }
-
- size_t bounce_bytes() const {
- return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
- }
- size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }
- size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }
- size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }
- static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need
-
- // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the
- // host; a prompt's pictures as their spans' rows (set_images)
- VisionTower vt;
- VisionGeo vg;
- int img_tok = -1;
- std::string vision_err; // why the tower did not load (pictures off), or empty
- std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]
- std::vector<Image> imgs; // the pictures of the prompt being fed, by start
- // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst
- bool read_bf16(const Tensor& t, size_t n, void* dst) {
- if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);
- if (t.dtype != "F32" || t.bytes != n * 4) return false;
- std::vector<float> f(n);
- if (!pk.read(t, f.data())) return false;
- uint16_t* o = (uint16_t*) dst;
- for (size_t i = 0; i < n; ++i) {
- uint32_t u;
- std::memcpy(&u, &f[i], 4);
- o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);
- }
- return true;
- }
- bool load_vision(std::string& err) {
- vg.layers = (int) pk.num("vision.num_hidden_layers", 0);
- const int tok = (int) pk.num("image_token_id", -1);
- if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none
- vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);
- vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);
- vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);
- vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;
- const char* mark[3] = {"image_start", "image_newline", "image_end"};
- for (int i = 0; i < 3; ++i) {
- const Tensor* t = pk.get(mark[i]);
- img_mark[i].resize((size_t) g.dim);
- if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {
- err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";
- return false;
- }
- }
- const double t0 = now_ms();
- Dev& d = devs[0];
- // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which
- // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)
- auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {
- const Tensor* t = pk.get(n);
- if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {
- const std::string pre = n.substr(0, n.size() - 13);
- const Tensor* tg = pk.get(pre + "mlp.gate.weight");
- const Tensor* tu = pk.get(pre + "mlp.up.weight");
- if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;
- shape = {tg->shape[0] * 2, tg->shape[1]};
- const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];
- return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));
- }
- if (t == nullptr) return false;
- shape = t->shape;
- size_t n_el = 1;
- for (int64_t x : t->shape) n_el *= (size_t) x;
- return dst == nullptr || read_bf16(*t, n_el, dst);
- };
- if (!vt.load(vg, d.id, read, err)) return false;
- img_tok = tok;
- CK(cudaSetDevice(d.id));
- CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)
- if (o.verbose)
- std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",
- vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),
- (now_ms() - t0) / 1000.0);
- return true;
- }
- bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {
- if (img_tok < 0) {
- err = "this model has no vision tower (or it is off)";
- return false;
- }
- std::vector<uint16_t> rows;
- int nh = 0, nw = 0;
- if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;
- const size_t D = (size_t) g.dim;
- span.resize(((size_t) nh * (nw + 1) + 2) * D);
- uint16_t* o = span.data();
- auto put = [&](const uint16_t* r) {
- std::memcpy(o, r, D * 2);
- o += D;
- };
- put(img_mark[0].data());
- for (int i = 0; i < nh; ++i) {
- for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);
- put(img_mark[1].data());
- }
- put(img_mark[2].data());
- return true;
- }
- // the row of a picture's span at position p (null: p is in none)
- const uint16_t* image_row(int64_t p) const {
- if (imgs.empty()) return nullptr;
- auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });
- if (it == imgs.begin()) return nullptr;
- const Image& im = *(it - 1);
- const int64_t r = p - im.start;
- if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;
- return im.rows->data() + (size_t) r * g.dim;
- }
- // token tk's embedding row at position p: inside a picture's span, the picture's
- const uint16_t* emb_row(int64_t p, int tk) const {
- const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;
- return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;
- }
- bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }
- // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)
- int32_t ctok(int64_t p, int tk) const {
- if (tk == img_tok && image_row(p) != nullptr) return -1;
- return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;
- }
- // a chunk's image rows (route()'s bits); false: none
- bool vl_bits(int64_t p0, int T, VlBits& b) const {
- if (imgs.empty()) return false;
- std::memset(&b, 0, sizeof b);
- bool any = false;
- for (int t = 0; t < T && t < kVlRows; ++t)
- if (in_image(p0 + t)) {
- b.w[t >> 5] |= 1u << (t & 31);
- any = true;
- }
- return any;
- }
-
- // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]
- // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a
- // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,
- // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.
- void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;
- const size_t per = (size_t) cols * (hd + hd / 32);
- b.clear();
- b.reads.reserve((size_t) T * cols);
- const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
- const Tensor* tw = pk.get(P + "weight");
- const Tensor* ts = pk.get(P + "scale");
- const int fmt = eg_fmt[(size_t) li];
- const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)
- const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
- const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)
- std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
- int64_t hits = 0;
- {
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- for (int t = 0; t < T; ++t) {
- const int64_t p = p0 + t;
- uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;
- uint8_t* scales = rows + (size_t) cols * hd;
- if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)
- std::memset(rows, 0, per);
- continue;
- }
- int64_t prod[8];
- bool blocked = false;
- for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)
- blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;
- const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];
- prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];
- }
- int64_t rolling = prod[0];
- for (int n = 1; n < pk.eg_ngram; ++n) {
- rolling ^= prod[n];
- for (int h = 0; h < pk.eg_heads; ++h) {
- const int c = (n - 1) * pk.eg_heads + h;
- const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];
- uint8_t* rd = rows + (size_t) c * hd;
- uint8_t* sd = scales + (size_t) c * (hd / 32);
- const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;
- if (const uint8_t* hit = eg_cache.find(key)) {
- std::memcpy(rd, hit, (size_t) hd);
- std::memcpy(sd, hit + hd, (size_t) hd / 32);
- ++hits;
- continue;
- }
- const auto fi = first.find(key);
- if (fi != first.end()) {
- b.dups.push_back({fi->second, {rd, sd}});
- continue;
- }
- first.emplace(key, (int) b.reads.size());
- EgRead r;
- r.file = tw->file;
- r.dst = rd;
- r.scales = sd;
- r.key = key;
- if (mx8) {
- r.mx8 = true;
- r.n = (uint32_t) (hd / 32 * 33);
- r.off = tw->off + (uint64_t) id * r.n;
- } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)
- r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);
- r.off = tw->off + (uint64_t) id * r.n;
- } else {
- r.n = (uint32_t) hd;
- r.off = tw->off + (uint64_t) id * hd;
- r.file2 = ts->file;
- r.n2 = (uint32_t) (hd / 32);
- r.off2 = ts->off + (uint64_t) id * (hd / 32);
- }
- b.reads.push_back(r);
- }
- }
- }
- }
- eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();
- eg_hits_n += hits + (int64_t) b.dups.size();
- eg_reads_n += (int64_t) b.reads.size();
- eg_io.submit(b);
- }
- // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)
- void eg_finish(EgBatch& b) {
- if (b.post.load(std::memory_order_acquire) == 2) return;
- const int64_t t0 = now_ns();
- eg_io.wait(b);
- int want = 0;
- if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {
- const int hd = pk.eg_hd;
- for (const auto& d : b.dups) {
- const EgRead& r = b.reads[(size_t) d.first];
- std::memcpy(d.second.first, r.dst, (size_t) hd);
- std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
- }
- if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite
- for (size_t i = 0; i < b.reads.size(); ++i) {
- const EgRead& r = b.reads[i];
- for (int k = 0; k < hd / 32; ++k) {
- uint16_t hb;
- std::memcpy(&hb, r.dst + (size_t) k * 18, 2);
- if ((hb & 0x7c00) == 0x7c00)
- std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),
- r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);
- }
- }
- }
- {
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- for (size_t i = 0; i < b.reads.size(); ++i) {
- const EgRead& r = b.reads[i];
- if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)
- std::memset(r.dst, 0, (size_t) hd);
- std::memset(r.scales, 0, (size_t) hd / 32);
- continue;
- }
- if (uint8_t* c = eg_cache.put(r.key)) {
- std::memcpy(c, r.dst, (size_t) hd);
- std::memcpy(c + hd, r.scales, (size_t) hd / 32);
- }
- }
- }
- eg_io_ns += b.t1.load() - b.t0;
- b.live = false;
- b.post.store(2, std::memory_order_release);
- } else {
- while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();
- }
- eg_wait_ns += now_ns() - t0;
- }
- void eg_stats() {
- st.eg_rows = eg_rows_n.load();
- st.eg_hits = eg_hits_n.load();
- st.eg_reads = eg_reads_n.load();
- st.eg_io_ms = (double) eg_io_ns.load() / 1e6;
- std::lock_guard<std::mutex> lk(eg_cache_mu);
- st.eg_cached = (int64_t) eg_cache.size();
- st.eg_cache_cap = (int64_t) eg_cache.capacity();
- }
- // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)
- void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {
- EgBatch b;
- eg_issue(p0, T, only, dst, b);
- eg_finish(b);
- }
-
- // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its
- // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,
- // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's
- // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down
- // pieces; the down bands). yd[slot] gets each row's expert output.
- struct Ent {
- int e = 0, n = 0, slot0 = 0;
- int tok[kc::kExl3CpuMaxN] = {};
- };
- void run_nodes(int n, const std::function<void(int)>* f) {
- for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);
- pools[0]->help();
- for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();
- }
-
- // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each
- // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),
- // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits
- // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds
- // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever
- // number of entries take it, and each job its band of h straight into the down's int8 input.
- // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)
- void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
- if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);
- constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
- const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);
- const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);
- const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)
- const float lim = g.lim;
- static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");
- { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at
- // once, so the SSD reads them with large requests side by side while the first entries compute
- // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost
- // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
- std::vector<std::pair<const void*, size_t>> want;
- for (const Ent& en : ents)
- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
- os::prefetch_cold(want);
- }
- // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token
- // index past the window (none on the decode or chunk paths) gets a row of its own after them
- uint8_t* xq = (uint8_t*) c_xh.data();
- uint8_t* hq = xq + (size_t) kWin * xa;
- uint8_t* xq_extra = hq + (size_t) kSlots * ha;
- auto x_act = [&](const Ent& en, int i) -> const void* {
- const int t = en.tok[i];
- return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;
- };
- {
- bool done[kWin] = {};
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i) {
- const int t = en.tok[i];
- if (t < kWin && done[t]) continue;
- if (t < kWin) done[t] = true;
- gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));
- }
- }
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];
- float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = x_act(en, i);
- gp[i] = yg[i];
- up[i] = yu[i];
- }
- // (part X's rows, from where they are: a copy's part on node X, or the file's)
- gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);
- gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);
- for (int i = 0; i < en.n; ++i) {
- float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;
- for (int r = r0; r < r1; ++r) {
- float gv = yg[i][r - r0], uv = yu[i][r - r0];
- if (lim > 0.0f) {
- gv = std::min(gv, lim);
- uv = std::min(std::max(uv, -lim), lim);
- }
- h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;
- }
- if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);
- }
- };
- run_nodes(ne * GB, f);
- if (!band_q)
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i)
- gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float* yp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = hq + (size_t) (en.slot0 + i) * ha;
- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;
- }
- gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);
- };
- run_nodes(ne * DB, f);
- }
- // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input
- // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same
-#if defined(STRATA_METAL)
- // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD
- // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN
- // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the
- // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per
- // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).
- static bool la_enabled() { return la_depth() > 0; }
- static int la_depth() {
- static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)
- return std::min(d, 2);
- }
- struct LaPool {
- std::mutex m;
- std::condition_variable cv;
- std::deque<std::pair<const void*, size_t>> q;
- LaPool() {
- const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;
- for (int i = 0; i < nth; ++i)
- std::thread([this] {
- for (;;) {
- std::pair<const void*, size_t> r;
- {
- std::unique_lock<std::mutex> lk(m);
- cv.wait(lk, [&] { return !q.empty(); });
- r = q.front();
- q.pop_front();
- while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)
- }
- const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;
- const size_t len = r.second + ((uintptr_t) r.first - a);
- std::vector<unsigned char> vec((len + 16383) / 16384);
- bool cold = true;
- if (mincore((void*) a, len, (char*) vec.data()) == 0) {
- cold = false;
- for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }
- }
- if (cold) os::prefetch({r});
- }
- }).detach();
- }
- void push(const void* p, size_t n) {
- { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }
- cv.notify_one();
- }
- };
- std::mutex la_mu;
- std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed
- long la_hit = 0, la_all = 0, la_issued = 0;
- void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {
- if (l.nm != 1) return;
- static LaPool* pool = nullptr;
- static bool reg = false;
- if (pool == nullptr) {
- la_guess.assign(L.size(), {});
- pool = new LaPool();
- }
- if (std::getenv("DS4_CPU_LOG") && !reg) {
- reg = true;
- static Impl* self = this;
- std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });
- }
- for (const Ent& en : ents) {
- ++la_all;
- const auto& gs = la_guess[(size_t) l.il];
- if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;
- }
- if (la_n <= 0) return;
- cudaEventSynchronize(ev2);
- static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;
- for (int a = 1; a <= la_n; ++a) {
- Layer& nx = L[(size_t) (l.il + a)];
- std::vector<int> cpu;
- for (int r = 0; r < nx.k_exp; ++r) {
- const int e = h_idx_a[(a - 1) * kPicks + r];
- if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);
- }
- la_guess[nx.il] = cpu;
- for (int e : cpu)
- for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }
- }
- }
-#endif
- void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
- constexpr int RB = 64; // rows a job
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
- gg::IqCpu fm;
- std::string ferr;
- if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||
- fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {
- static std::atomic<bool> said{false};
- if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());
- for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);
- return;
- }
- static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;
- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }
- ++n_calls;
- double tp0 = now_ms();
- const float lim = g.lim;
- { // (os::prefetch, as above)
- std::vector<std::pair<const void*, size_t>> want;
- for (const Ent& en : ents)
- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
- if (std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)
- static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;
- if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }
- for (const auto& w : want) {
- const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;
- unsigned char vec[64];
- const size_t len = std::min<size_t>(w.second, 64 * 16384);
- bool cold = false;
- if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;
- const double t0 = now_ms();
- os::prefetch({w});
- (cold ? cold_ms : warm_ms) += now_ms() - t0;
- (cold ? cold_n : warm_n)++;
- }
- } else if (want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {
- // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of
- // all the call's experts at once keep the SSD busy)
- std::function<void(int)> pf[4];
- for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };
- run_nodes((int) want.size(), pf);
- } else
- os::prefetch(want);
- }
- double tp1 = now_ms(); t_pre += tp1 - tp0;
- // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of
- // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or
- // chunk paths), then (x no longer needed) the slot's h
- uint8_t* aq = (uint8_t*) c_xh.data();
- auto x_act = [&](const Ent& en, int i) -> uint8_t* {
- const int t = en.tok[i];
- return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;
- };
- {
- bool done[kWin] = {};
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i) {
- const int t = en.tok[i];
- if (t < kWin && done[t]) continue;
- if (t < kWin) done[t] = true;
- gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
- }
- }
- double tq1 = now_ms(); t_q1 += tq1 - tp1;
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
- const void* ap[gg::kMxCpuMaxN];
- float* hp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- ap[i] = x_act(en, i);
- hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)
- }
- // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any
- // block size of the type)
- gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,
- r0 - X * FFh, r1 - X * FFh, lim);
- };
- run_nodes(ne * GB, f);
- double tr1 = now_ms(); t_r1 += tr1 - tq1;
- for (const Ent& en : ents)
- for (int i = 0; i < en.n; ++i)
- gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
- aq + (size_t) (kWin + en.slot0 + i) * E * 4);
- double tq2 = now_ms(); t_q2 += tq2 - tr1;
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
- const void* hp[gg::kMxCpuMaxN];
- float* yp[gg::kMxCpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;
- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)
- }
- gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
- };
- run_nodes(ne * DB, f);
- t_r2 += now_ms() - tq2;
- }
-
- // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
- struct StoreHold {
- Impl* m = nullptr;
- int layer = 0;
- std::vector<int> es;
- std::vector<char> ok;
- StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
- if (!m_->store) return;
- m = m_;
- layer = l.il;
- for (const Ent& en : ents)
- if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
- m->store->ensure(layer, es.data(), (int) es.size(), ok);
- }
- ~StoreHold() {
- if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
- }
- };
- void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
- const int* ord = nullptr) {
- if (ldx == 0) ldx = (size_t) g.dim;
- if (l.lay.fmt == gg::kFmtMXFP4) {
- StoreHold hold(this, l, ents);
- return cpu_experts_mx(l, x, ents, ldx, ord);
- }
- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
- const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
- const int ne = (int) ents.size();
- const float lim = g.lim;
- auto part = [&](int e, int X, int r) {
- const uint8_t* b = l.hh[(size_t) e][(size_t) X];
- kc::Exl3Part p;
- p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);
- p.suh = (const uint16_t*) (b + l.hl.suh[r]);
- p.svh = (const uint16_t*) (b + l.hl.svh[r]);
- p.k = l.hl.k[r];
- p.n = l.hl.n[r];
- p.K = l.lay.K[r];
- return p;
- };
- auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };
- auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };
- auto acc = [&](int X, int sl, int r, int q) {
- return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;
- };
- auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };
- auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };
- auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };
- std::function<void(int)> f[4];
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / (2 * GQ))];
- const int r = (j / GQ) % 2, q = j % GQ;
- const kc::Exl3Part p = part(en.e, X, r);
- const float* xp[kc::kExl3CpuMaxN];
- float* ap[kc::kExl3CpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- const float* xt = x + (size_t) en.tok[i] * ldx;
- for (int b = 2 * q; b < 2 * q + 2; ++b)
- xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);
- xp[i] = xh(X, sl, r);
- ap[i] = acc(X, sl, r, q);
- }
- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);
- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);
- };
- run_nodes(ne * 2 * GQ, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / GB)];
- const int b = j % GB;
- const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- float y[2][128], a[128], hb[128];
- for (int r = 0; r < 2; ++r) {
- for (int c = 0; c < 128; ++c) a[c] = 0.0f;
- for (int q = 0; q < GQ; ++q) {
- const float* aq = acc(X, sl, r, q) + b * 128;
- for (int c = 0; c < 128; ++c) a[c] += aq[c];
- }
- double xs = 0.0;
- for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];
- kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);
- }
- for (int c = 0; c < 128; ++c) {
- float gv = y[0][c], uv = y[1][c];
- if (lim > 0.0f) {
- gv = std::min(gv, lim);
- uv = std::min(std::max(uv, -lim), lim);
- }
- hb[c] = gv / (1.0f + std::exp(-gv)) * uv;
- }
- const int gb = X * GB + b;
- xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);
- }
- };
- run_nodes(ne * GB, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / DQ)];
- const int q = j % DQ;
- const kc::Exl3Part p = part(en.e, X, 2);
- const float* xp[kc::kExl3CpuMaxN];
- float* ap[kc::kExl3CpuMaxN];
- for (int i = 0; i < en.n; ++i) {
- xp[i] = xhd(en.slot0 + i);
- ap[i] = accd(X, en.slot0 + i, q);
- }
- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);
- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);
- };
- run_nodes(ne * DQ, f);
- for (int X = 0; X < NH; ++X)
- f[X] = [&, X](int j) {
- const Ent& en = ents[(size_t) (j / DB)];
- const int b = j % DB;
- const kc::Exl3Part pd = part(en.e, X, 2);
- for (int i = 0; i < en.n; ++i) {
- const int sl = en.slot0 + i;
- float a[128];
- for (int c = 0; c < 128; ++c) a[c] = 0.0f;
- for (int q = 0; q < DQ; ++q) {
- const float* aq = accd(X, sl, q) + b * 128;
- for (int c = 0; c < 128; ++c) a[c] += aq[c];
- }
- double xs = 0.0;
- for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];
- kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);
- }
- };
- run_nodes(ne * DB, f);
- }
-
- // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it
- // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs
- // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's
- // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:
- // the CPU lane reads it from the file again.
- void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot
- for (;;) {
- RamSwap rs;
- {
- std::unique_lock<std::mutex> lk(sw_mu);
- ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });
- if (quit.load()) return;
- rs = ram_queue.front();
- ram_queue.pop_front();
- }
- const Layer& l = L[(size_t) rs.layer];
- if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {
- for (int r = 0; r < 3; ++r) // (one copy of it in RAM)
- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],
- mx_part_bytes(l, r), false);
- } else {
- rs.in = -1; // (failed: the slot goes back)
- }
- std::lock_guard<std::mutex> lk(sw_mu);
- ram_ready.push_back(rs);
- }
- }
-
- void cache_loop(SwapBufs& B) {
- for (;;) {
- Swap sw;
- {
- std::unique_lock<std::mutex> lk(sw_mu);
- sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });
- if (quit.load()) return;
- sw = sw_queue.front();
- sw_queue.pop_front();
- }
- Layer& l = L[(size_t) sw.layer];
- if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)
- swap_in_mx(l, sw, B);
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_ready.push_back(sw);
- continue;
- }
- parts_to_blob(l, l.hh[(size_t) sw.in], B.a);
- if (l.nm == 1) {
- const Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t cs = B.st[(size_t) l.dev];
- cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);
- if (!file_experts)
- cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,
- cudaMemcpyDeviceToHost, cs);
- cudaStreamSynchronize(cs);
- } else { // a group: each member's part in, each member's part out
- for (int m = 0; m < l.nm; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = B.st[(size_t) x.dev];
- blob_to_half(l, B.a, m, B.h);
- cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);
- cudaStreamSynchronize(cs);
- if (file_experts) continue;
- cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,
- cudaMemcpyDeviceToHost, cs);
- cudaStreamSynchronize(cs);
- half_to_blob(l, B.h, m, B.b);
- }
- }
- if (!file_experts)
- for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);
- if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte
- std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);
- bool ok = true;
- for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)
- const int e = pass == 0 ? sw.in : sw.out;
- reg.resize(l.region_bytes[(size_t) e]);
- pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());
- if (pass == 0) {
- repack(l, reg.data(), blob.data());
- std::vector<uint8_t> dev(l.blob);
- if (l.nm == 1)
- cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);
- else
- for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)
- std::vector<uint8_t> h(member(l, m).hblob);
- cudaSetDevice(devs[(size_t) member(l, m).dev].id);
- cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,
- cudaMemcpyDeviceToHost);
- half_to_blob(l, h.data(), m, dev.data());
- }
- for (int r = 0; r < 3; ++r)
- ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&
- std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&
- std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;
- } else {
- for (int X = 0; X < NH; ++X) {
- repack_part(l, reg.data(), X, part.data());
- for (int r = 0; r < 3; ++r) {
- const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];
- ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&
- std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&
- std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;
- }
- }
- }
- }
- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,
- ok ? "both match the model file" : "MISMATCH");
- }
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_ready.push_back(sw);
- }
- }
-
- // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);
- // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)
- // from the layer's spare RAM slot, which it is copied into from VRAM here
- void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {
- uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)
- const std::vector<cudaStream_t>& copy_st = B.st;
- bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till
- // this swap's switch)
- for (int m = 0; m < l.nm && ok; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = copy_st[(size_t) x.dev];
- if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);
- ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),
- cudaMemcpyHostToDevice, cs) == cudaSuccess &&
- cudaStreamSynchronize(cs) == cudaSuccess;
- }
- if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot
- bool got = ok;
- for (int m = 0; m < l.nm && got; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaStream_t cs = copy_st[(size_t) x.dev];
- got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),
- unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&
- cudaStreamSynchronize(cs) == cudaSuccess;
- if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);
- }
- if (got) blob_to_ram(l, sw_b, sw.ram);
- else sw.ram = -1; // (back to the file's pages then)
- }
- if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte
- std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);
- for (int m = 0; m < l.nm && ok; ++m) {
- const Layer& x = member(l, m);
- cudaSetDevice(devs[(size_t) x.dev].id);
- cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),
- cudaMemcpyDeviceToHost);
- if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());
- }
- ok = ok && read_mx_file(l, sw.in, file.data());
- for (int r = 0; r < 3 && ok; ++r)
- ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
- bool ok_out = true;
- if (sw.ram >= 0) {
- ok_out = read_mx_file(l, sw.out, file.data());
- ram_to_blob(l, sw.ram, dev.data());
- for (int r = 0; r < 3 && ok_out; ++r)
- ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
- }
- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,
- sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");
- } else if (!ok) {
- std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);
- }
- }
-
- void cache_tick(int T) {
- cache_tick_swaps(T);
- store_sync();
- store_watch();
- }
- // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time
- // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free
- void store_watch() {
- if (!store) return;
- static int tick = 0;
- if (++tick % 16 != 0) return;
- const uint64_t avail = ram_avail();
- if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;
- const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));
- const size_t n = store->shrink_chunks(chunks);
- if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());
- }
- void cache_tick_swaps(int T) {
- if (!o.expert_cache || cache_ths.empty()) return;
- std::vector<Swap> ready;
- std::vector<RamSwap> rready;
- {
- std::lock_guard<std::mutex> lk(sw_mu);
- ready.swap(sw_ready);
- rready.swap(ram_ready);
- }
- for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)
- --ram_inflight;
- Layer& l = L[(size_t) rs.layer];
- if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {
- ram_free.push_back(rs.slot);
- continue;
- }
- l.ram[(size_t) rs.in] = rs.slot;
- if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {
- ram_free.push_back(l.ram[(size_t) rs.out]);
- l.ram[(size_t) rs.out] = -1;
- }
- ++st.ram_moves;
- }
- for (const Swap& sw : ready) {
- Layer& l = L[(size_t) sw.layer];
- const int old_slot = l.slot[(size_t) sw.out];
- const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];
- if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)
- const int s_in = l.ram[(size_t) sw.in];
- l.ram[(size_t) sw.in] = -1;
- if (sw.ram >= 0) {
- l.ram[(size_t) sw.out] = sw.ram;
- copy_spare[(size_t) l.il] = s_in;
- } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)
- --ram_held;
- if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;
- }
- } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)
- ram_free.push_back(l.ram[(size_t) sw.in]);
- l.ram[(size_t) sw.in] = -1;
- }
- l.slot[(size_t) sw.in] = l.spare_vram;
- if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)
- l.zc[(size_t) sw.in] = 0;
- store->unmark(l.il, sw.in);
- }
- if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
- l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
- l.host[(size_t) sw.in] = nullptr;
- l.slot[(size_t) sw.out] = -1;
- if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)
- l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);
- } else if (file_experts) { // (back to its pages in the file)
- l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};
- l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];
- } else {
- l.hh[(size_t) sw.out] = l.spare_host;
- l.host[(size_t) sw.out] = l.spare_host[0];
- l.spare_host = old_parts;
- }
- l.spare_vram = old_slot;
- for (int m = 0; m < l.nm; ++m) {
- Layer& x = member(l, m);
- const Dev& dm = devs[(size_t) x.dev];
- l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));
- l.tab_pin[2 * m + 1] = 0ull;
- cudaSetDevice(dm.id); // (stream order: before the next window)
- cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);
- cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);
- }
- l.swap_pending = false;
- --sw_inflight;
- ++st.swaps;
- }
- decay_tokens += T;
- if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens
- decay_tokens = 0;
- for (Layer& l : L)
- for (float& h : l.heat) h *= c_decay;
- }
- bool queued = false, rqueued = false;
- for (Layer& l : L) {
- if (l.cpu_hits.empty()) continue;
- std::vector<int> hits;
- hits.swap(l.cpu_hits);
- int best = -1, vin = -1;
- for (int e : hits)
- if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;
- if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in
- int cold = -1;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;
- if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {
- l.swap_pending = true;
- ++sw_inflight;
- vin = best;
- Swap sw{l.il, best, cold};
- if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)
- std::lock_guard<std::mutex> lk(sw_mu);
- sw_queue.push_back(sw);
- queued = true;
- }
- }
- if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;
- int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)
- for (int e : hits)
- if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&
- (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))
- rb = e;
- if (rb < 0) continue;
- int rc = -1;
- for (int e = 0; e < l.n_exp; ++e)
- if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;
- const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)
- if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;
- RamSwap rs;
- rs.layer = l.il;
- rs.in = rb;
- rs.out = room ? -1 : rc;
- rs.slot = ram_free.back();
- ram_free.pop_back();
- ++ram_inflight;
- std::lock_guard<std::mutex> lk(sw_mu);
- ram_queue.push_back(rs);
- rqueued = true;
- }
- if (queued) sw_cv.notify_all();
- if (rqueued) ram_cv.notify_one();
- }
-
- // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes
- void prefetch_loop() {
- std::vector<std::pair<const void*, size_t>> b;
- for (;;) {
- {
- std::unique_lock<std::mutex> lk(pf_mu);
- pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });
- if (quit.load(std::memory_order_relaxed)) return;
- b.swap(pf_q);
- }
- os::prefetch_cold(b); // (the resident ones would only cost the page walk)
- b.clear();
- }
- }
- // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)
- static bool file_read(const Layer& l, int e) {
- return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);
- }
- // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4
- // experts read in place, some of them from the files now (none: nothing to read early - null)
- const Layer* ahead_layer(const Layer& l0, int a) const {
- const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;
- if (j >= end || j >= (int) L.size()) return nullptr;
- const Layer& n = L[(size_t) j];
- if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||
- (int) n.slot.size() < n.n_exp)
- return nullptr;
- for (int e = 0; e < n.n_exp; ++e)
- if (file_read(n, e)) return &n;
- return nullptr;
- }
- // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the
- // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts
- std::vector<std::pair<const void*, size_t>> pf_want;
- void ahead_ask(int li, const Req& r, unsigned seq, int T) {
- pf_want.clear();
- for (int a = 1; a <= 2; ++a) {
- const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;
- if (m <= 0 || j >= (int) L.size()) continue;
- const Layer& n = L[(size_t) j];
- uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;
- uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;
- if (ahead_for[(size_t) j] != seq + (unsigned) a) {
- std::memset(fl, 0, 1024);
- std::memset(rk, 0, 1024);
- ahead_for[(size_t) j] = seq + (unsigned) a;
- }
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < m; ++i) {
- const int e = r.ahead[t][(a - 1) * kAhead1 + i];
- if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;
- fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);
- if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
- if ((fl[e] & 8) || !file_read(n, e)) continue;
- fl[e] |= 8;
- if (store) store->prefetch(j, e);
- else
- for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
- ++st.ahead_pf;
- }
- }
- if (pf_want.empty()) return;
- {
- std::lock_guard<std::mutex> lk(pf_mu);
- pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());
- }
- pf_cv.notify_one();
- }
-
- // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's
- // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.
- void serve_lane() {
- unsigned next = 1;
- auto last = std::chrono::steady_clock::now();
- std::vector<Ent> ents;
- while (!quit.load(std::memory_order_relaxed)) {
- Req& r = reqs[next % kRing];
- if (*(volatile unsigned*) &r.seq != next) {
- cpu_relax();
- if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))
- std::this_thread::sleep_for(std::chrono::microseconds(20));
- continue;
- }
- std::atomic_thread_fence(std::memory_order_acquire);
- const int li = layer_of[next % kRing].load(std::memory_order_acquire);
- Resp& rp = resps[next % kRing];
- const int T = std::max(1, std::min(kWin, r.T));
- for (int t = 0; t < T; ++t) rp.cpu[t] = 0;
- if (li >= 0 && li < (int) L.size()) {
- Layer& l = L[(size_t) li];
- uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
- ents.clear();
- if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)
- // lookahead: the guesses made for this layer (when they are for this request), counted on decode
- const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
- ? ahead_fl.data() + (size_t) li * 1024 : nullptr;
- // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row
- float wv[kWin * kPicks];
- int tok_of[kWin * kPicks];
- int nslots = 0;
- std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
- if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids
- // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the
- // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's
- // predecessors (the request carries the guesses for the layers after it)
- static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;
- uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};
- std::fwrite(hd, 1, 2, trace_f);
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < l.k_exp; ++i) {
- const int16_t v = (int16_t) r.ids[t][i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- if (with_ahead) {
- const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};
- std::fwrite(na, 1, 2, trace_f);
- for (int t = 0; t < T; ++t) {
- for (int i = 0; i < na[0]; ++i) {
- const int16_t v = (int16_t) r.ahead[t][i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- for (int i = 0; i < na[1]; ++i) {
- const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- }
- }
- }
- const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
- for (int t = 0; t < T; ++t) {
- float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)
- if (dsk)
- for (int i = 0; i < l.k_exp; ++i)
- if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];
- for (int i = 0; i < l.k_exp; ++i) {
- const int e = r.ids[t][i];
- if (e < 0 || e >= l.n_exp) continue;
- ++pr[e];
- l.heat[(size_t) e] += 1.0f;
- if (fl != nullptr) {
- ++st.ahead_picks;
- st.ahead_hit_k += fl[e] & 1;
- st.ahead_hit += (fl[e] & 6) != 0;
- st.ahead_hit2 += (fl[e] >> 2) & 1;
- }
- if (l.slot[(size_t) e] < 0) {
- l.cpu_hits.push_back(e);
- if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot
- store->touch(li, e);
- ++st.store_zc;
- continue;
- }
- if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
- ++st.skipped;
- continue;
- }
- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;
- else if (fl != nullptr) {
- ++st.ahead_cold;
- st.ahead_cold_hit += (fl[e] >> 3) & 1;
- }
- picks.push_back({e, t * kPicks + i});
- }
- }
- }
- if (fl != nullptr) { // the guesses by rank: how many were picked
- const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;
- for (int e = 0; e < l.n_exp && e < 1024; ++e) {
- if (rk[e] == 0 || rk[e] > 16) continue;
- bool hit = false;
- for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;
- ++st.ahead_rank_n[rk[e] - 1];
- st.ahead_rank_hit[rk[e] - 1] += hit;
- }
- }
- if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);
- std::sort(picks.begin(), picks.end());
- for (size_t a = 0; a < picks.size();) {
- Ent en;
- en.e = picks[a].first;
- en.slot0 = nslots;
- en.n = 0;
- while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {
- const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;
- en.tok[en.n++] = t;
- tok_of[nslots] = t;
- wv[nslots] = r.w[t][i];
- ++nslots;
- ++a;
- }
- ents.push_back(en);
- }
- // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for
- // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)
- int ord[kWin * kPicks];
- const int* po = nullptr;
- if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&
- ents.size() <= (size_t) (kWin * kPicks)) {
- const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;
- int n = 0;
- for (int pass = 0; pass < 2; ++pass)
- for (size_t q = 0; q < ents.size(); ++q) {
- const int e = ents[q].e;
- const bool early = !file_read(l, e) || (fa[e] & 8) != 0;
- if (early == (pass == 0)) ord[n++] = (int) q;
- }
- po = ord;
- }
- if (!ents.empty()) {
- const double tc = now_ms();
- cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);
- const int D = g.dim;
- for (int t = 0; t < T; ++t) {
- bool any = false;
- for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;
- if (!any) continue;
- float* o = rp.part[t];
- for (int e = 0; e < D; ++e) o[e] = 0.0f;
- for (int sl = 0; sl < nslots; ++sl) {
- if (tok_of[sl] != t) continue;
- const float* y = c_yd.data() + (size_t) sl * D;
- for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];
- }
- rp.cpu[t] = 1;
- }
- st.cpu_ms += now_ms() - tc;
- st.cpu_experts += nslots;
- }
- }
- std::atomic_thread_fence(std::memory_order_release);
- *(volatile unsigned*) &rp.seq = next;
- ++next;
- last = std::chrono::steady_clock::now();
- }
- }
-
- // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's
- static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {
- bool g = false;
- for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;
- return g ? gg::mv(j, n, s) : x3::mv(j, n, s);
- }
- bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {
- x3::MvJob j[x3::kMaxMvJobs];
- int n = 0;
- for (const x3::MvJob& x : jobs) j[n++] = x;
- if (!mv_any(j, n, d.st)) {
- err = "exl3 mv failed";
- return false;
- }
- return true;
- }
- bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {
- for (int a = 0; a < n; a += x3::kMaxMvJobs)
- if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {
- err = "exl3 mv failed";
- return false;
- }
- return true;
- }
- // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs
- static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,
- float alpha = 1.0f) {
- x3::MvJob j;
- j.m = &m;
- j.xf = x;
- j.y = y;
- j.alpha = alpha;
- j.ncol = T;
- j.xf_cs = x_cs;
- j.y_cs = y_cs;
- return j;
- }
-
- // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:
- // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm
- // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x
- // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),
- // the trunk's head, the markov bias token by token (greedy) and the confidence head.
- bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {
- if (g.n_mtp == 0 || n <= 0) return true;
- if (n > kWin * kPicks) {
- err = "seeding the drafter with " + std::to_string(n) + " rows";
- return false;
- }
- Dev& d = devs[(size_t) d_dev];
- cudaSetDevice(d.id);
- const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;
- if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps
- for (Dev& e : devs) {
- cudaSetDevice(e.id);
- CK(cudaStreamSynchronize(e.st));
- }
- cudaSetDevice(d.id);
- const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};
- std::fwrite(hdr, 8, 4, dump_d);
- std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);
- std::fflush(dump_d);
- }
- for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)
- const int na = std::min(kWin, n - a);
- CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,
- cudaMemcpyHostToDevice, d.st));
- if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;
- }
- CK(cudaEventRecord(ev_seed, d.st));
- Finish fm;
- fm.w = d_main_norm;
- fm.eps = g.eps;
- finish(d.ed, n, D, D, fm, d.st);
- for (int i = 0; i < g.n_mtp; ++i) {
- Layer& l = L[(size_t) (g.nl + i)];
- if (l.dev != d_dev) {
- err = "the DSpark blocks are split across devices";
- return false;
- }
- for (int a = 0; a < n; a += kWin) {
- const int na = std::min(kWin, n - a);
- if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) (p0 + a);
- fk.freqs = d.freqs_win;
- fk.quant = 1;
- fk.dst = d.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(d.kvr, na, HD, HD, fk, d.st);
- ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);
- }
- }
- return true;
- }
-
- bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,
- int top_k, float min_p, uint64_t seed, uint64_t counter) {
- const double t0 = now_ms();
- drafts_q = temperature > 0.0f && sample_drafts;
- dq_temp = temperature;
- dq_top_p = top_p;
- dq_top_k = top_k;
- dq_min_p = min_p;
- dq_seed = seed;
- dq_counter = counter;
- const bool ok = draft_(y, out, conf, err);
- if (!ok) drafts_q = false;
- if (max_drafts > 0 && (int) out.size() > max_drafts) {
- out.resize((size_t) max_drafts);
- conf.resize((size_t) max_drafts);
- }
- st.draft_ms += now_ms() - t0;
- ++st.drafts_rounds;
- return ok;
- }
- bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {
- out.clear();
- conf.clear();
- const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;
- const int64_t p0 = pos;
- if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;
- int tk[kWin];
- tk[0] = y;
- for (int t = 1; t < T; ++t) tk[t] = g.d_noise;
- Dev& d = devs[(size_t) d_dev];
- cudaSetDevice(d.id);
- for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, d.st);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float hh[4] = {0, 0, 0, 0};
- cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, d.st);
- WinPlan dp; // (window attention alone: the mailbox seqs)
- dp.async = !sync_decode && T <= async_max_t;
- for (int i = 0; i < g.n_mtp && dp.async; ++i) {
- const int il = g.nl + i;
- dp.seq[il] = ++seq_issued;
- layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);
- }
- const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;
- if (dump_d != nullptr && h_dump == nullptr &&
- (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))
- return false;
- for (int i = 0; i < g.n_mtp; ++i) {
- Layer& l = L[(size_t) (g.nl + i)];
- if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)
- ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);
- CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));
- }
- if (!layer(l, 0, p0, T, dp, err)) return false;
- if (dump_d != nullptr) {
- cudaSetDevice(d.id);
- CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));
- }
- }
- cudaSetDevice(d.id);
- hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)
- Dev& hd = devs[(size_t) head_dev];
- // (through the host to the head's device, and to the head group's other members - the drafter's device can be
- // the head group's lead itself, a group of every GPU)
- if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {
- CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));
- CK(cudaEventRecord(d.ev_out, d.st));
- }
- if (d_dev != head_dev) {
- cudaSetDevice(hd.id);
- CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));
- CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));
- }
- // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)
- const std::vector<int>& hg = groups[(size_t) hd.grp].mem;
- const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)
- for (int m = (int) hg.size() - 1; m >= 0; --m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- if (m > 0) {
- CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));
- CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));
- } else
- CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));
- Finish fn;
- fn.w = e.d_norm;
- fn.eps = g.eps;
- finish(e.ao, T, D, D, fn, e.st);
- if (!head_share(e, m, e.ao, T, err, !mk)) return false;
- }
- h_idx[0] = y;
- if (mk) {
- // the markov steps over the group: each member its rows of the bias and its share's argmax, every member
- // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,
- // each step ~1/nm of their 66 MB and no gather of the shares
- const int nm = (int) hg.size();
- const unsigned seq = ++mk_seq;
- for (int m = 0; m < nm; ++m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));
- CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));
- }
- for (int i = 0; i < T; ++i)
- for (int m = 0; m < nm; ++m) {
- Dev& e = devs[(size_t) hg[(size_t) m]];
- cudaSetDevice(e.id);
- float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;
- markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);
- argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);
- markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,
- e.st);
- }
- cudaSetDevice(hd.id);
- } else {
- CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));
- strata::dsv41k::SamplerParams sp;
- sp.greedy = true;
- sp.temperature = 0.0f;
- if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)
- sp.greedy = false;
- sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);
- sp.top_p = dq_top_p;
- sp.top_k = dq_top_k;
- sp.min_p = dq_min_p;
- sp.seed = dq_seed;
- }
- for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
- markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
- if (drafts_q) {
- sp.counter = dq_counter + (uint64_t) i;
- strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);
- } else
- strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);
- }
- }
- confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
- int tok[kWin + 1];
- float cf[kWin];
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];
- for (int i = 0; i < T; ++i) cf[i] = h_w[i];
- if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden
- std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream
- CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));
- cudaSetDevice(d.id);
- CK(cudaStreamSynchronize(d.st));
- const int64_t hdr[4] = {'D', p0, y, T};
- std::fwrite(hdr, 8, 4, dump_d);
- std::fwrite(tok + 1, 4, (size_t) T, dump_d);
- std::fwrite(cf, 4, (size_t) T, dump_d);
- std::fwrite(xs.data(), 4, xs.size(), dump_d);
- for (int i = 0; i < g.n_mtp; ++i) {
- std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);
- std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);
- }
- std::fflush(dump_d);
- cudaSetDevice(hd.id);
- }
- for (int i = 0; i < T; ++i) {
- out.push_back(tok[i + 1]);
- conf.push_back(cf[i]);
- }
- return true;
- }
-
- int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,
- const float* bias) {
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- strata::dsv41k::SamplerParams sp;
- sp.temperature = temperature;
- sp.greedy = temperature <= 0.0f;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- sp.logit_bias = bias;
- strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
- int t = -1;
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- return t;
- }
-
- // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.
- // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own
- // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution
- // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the
- // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.
- bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
- uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {
- const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);
- const int T = n + 1, V = g.vocab;
- const int64_t p0 = pos;
- int tk[kWin];
- tk[0] = y;
- for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];
- spec_win = true;
- const double tv = now_ms();
- const bool ok = forward(tk, T, T, err, 0);
- st.verify_ms += now_ms() - tv;
- spec_win = false;
- if (!ok) return false;
- Dev& hd = devs[(size_t) head_dev];
- cudaSetDevice(hd.id);
- int m = 0;
- if (temperature <= 0.0f) {
- strata::dsv41k::SamplerParams sp;
- sp.greedy = true;
- sp.temperature = 0.0f;
- strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
- next = h_idx[m];
- } else {
- // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)
- // (never d itself): each row a token distributed as the model's own draw at that position, p through the
- // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,
- // never kept); past the first rejection the rows do not count
- const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&
- dq_min_p == min_p && dq_seed == seed && dq_counter == counter;
- if (!own) { // the drafter's argmax: q a point at each draft
- for (int i = 0; i < n; ++i) {
- h_specq[(size_t) i].n = 1;
- h_specq[(size_t) i].ids[0] = drafts[(size_t) i];
- h_specq[(size_t) i].p[0] = 1.0;
- }
- CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),
- cudaMemcpyHostToDevice, hd.st));
- }
- for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];
- CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));
- strata::dsv41k::SamplerParams sp;
- sp.greedy = false;
- sp.temperature = temperature;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
- CK(cudaStreamSynchronize(hd.st));
- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
- next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);
- }
- drafts_q = false;
- rollback(p0, T, m + 1);
- if (!seed_drafter(p0, m + 1, 0, err)) return false;
- accepted = m;
- st.drafts_offered += n;
- st.drafts_accepted += m;
- for (int i = 0; i < n && i <= m && i < 8; ++i) {
- ++st.draft_reached[i];
- if (i < m) ++st.draft_kept[i];
- }
- return true;
- }
-
- // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone
- // (the compressed KV and the index keys past them are written again when those positions come)
- void rollback(int64_t p0, int T, int keep) {
- if (keep >= T) return;
- // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor
- // parallel, a millisecond of the host's before the next draft)
- std::vector<RingBatch> rb(devs.size());
- for (int il = 0; il < g.nl; ++il)
- for (int m = 0; m < L[(size_t) il].nm; ++m) {
- Layer& l = member(L[(size_t) il], m);
- RingBatch& b = rb[(size_t) l.dev];
- if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)
- cudaSetDevice(devs[(size_t) l.dev].id);
- ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);
- continue;
- }
- b.ring[b.n] = l.win;
- b.bak[b.n++] = l.ring_bak;
- }
- for (size_t di = 0; di < devs.size(); ++di)
- if (rb[di].n > 0) {
- cudaSetDevice(devs[di].id);
- ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);
- }
- for (int il = 0; il < g.nl; ++il)
- for (int m = 0; m < L[(size_t) il].nm; ++m) {
- Layer& l = member(L[(size_t) il], m);
- Dev& d = devs[(size_t) l.dev];
- if (l.c_snap != nullptr) {
- cudaSetDevice(d.id);
- const size_t ns = (size_t) l.ratio * g.hd * 4;
- cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);
- cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);
- compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,
- g.hd, l.c_norm, g.eps, d.latent, d.st);
- }
- }
- pos = p0 + keep;
- toks.resize((size_t) pos);
- ctoks.resize((size_t) pos);
- }
-
- // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
- bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }
-
-
- // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
- // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
- // place, the CPU lane's streamed in batches)
- bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {
- Layer& l = member(L0, mi);
- Dev& d = devs[(size_t) l.dev];
- Chunk& c = *chunks[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- prof_mark(d, 0);
- const int nm = L0.nm;
- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
- const ArGroup& ab = groups[(size_t) L0.grp].arb;
- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
- // Engram, in slabs of 256 tokens
- if (l.eg >= 0) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- const size_t per = (size_t) cols * (ehd + ehd / 32);
- const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)
- uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;
- CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.eg_ev[slot], s));
- for (int s0 = 0; s0 < T; s0 += 256) {
- const int ns = std::min(256, T - s0);
- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
- x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
- if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
- engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
- }
- }
- // attention
- if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
- dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
- hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
- hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
- dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
- dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);
- dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);
- dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);
- dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);
- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))
- return false;
- if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
- if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
- dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);
- dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
- // the window rows: the ring's (positions p0 - win ..) then the chunk's
- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = c.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(c.kvr, T, HD, HD, fk, s);
- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
- if (snap) {
- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
- }
- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
- const uint8_t* ckv = nullptr;
- const long long ss = (long long) o.max_context + 64;
- if (l.ratio > 0) {
- Src& sr = srcs[(size_t) l.src];
- const int R = l.ratio;
- const int64_t g_first = p0 / R, g_end = (p0 + T) / R;
- const int ng = (int) std::max<int64_t>(0, g_end - g_first);
- const float* latent = c.latent;
- if (l.kv_src) {
- if (R == 1) {
- Finish fc;
- fc.w = l.c_norm;
- fc.eps = g.eps;
- finish(c.ckr, T, HD, HD, fc, s);
- latent = c.ckr;
- } else {
- if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)
- const int64_t ps = p0 + wp.snap_t;
- for (int sl = 0; sl < R; ++sl) {
- const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;
- const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;
- const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;
- CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
- }
- }
- compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);
- }
- }
- const int clen_max = (int) ((p0 + T) / R);
- if (l.ix_src) {
- Src& ks = srcs[(size_t) l.ik_src];
- if (l.owns_k && ng > 0) {
- x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);
- if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;
- Finish fi;
- fi.w = l.ik_norm;
- fi.eps = g.eps;
- fi.rd = g.rd;
- fi.pos = (int) (g_first * R);
- fi.pos_step = R;
- fi.freqs = freqs;
- fi.quant = 3;
- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- fi.pdst_stride = (long long) ik_rb();
- finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
- }
- if (clen_max > 0) {
- if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
- return false;
- for (int a0 = 0; a0 < T; a0 += c.AI) {
- const int na = std::min(c.AI, T - a0);
- const int64_t pa = p0 + a0;
- if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,
- g.ix_heads * g.ix_hd, 0.0f, err))
- return false;
- Finish fi;
- fi.rd = g.rd;
- fi.pos = (int) pa;
- fi.vpp = g.ix_heads;
- fi.freqs = freqs;
- fi.quant = 3;
- finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
- Rows lim;
- lim.gen = 1;
- lim.p0 = pa;
- lim.R = R;
- lim.kcap = g.ix_topk;
- lim.ccap = g.cand_topk;
- lim.blk = std::max(1, g.cand_blk);
- const int nlast = (int) ((pa + na) / R); // the slab's widest query
- int* tk = c.topk + (size_t) a0 * topk_stride();
- int* cd = c.cand + (size_t) a0 * cand_stride();
- const float* iw = c.iw + (size_t) a0 * 64;
- if (nlast <= 0) continue;
- if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {
- Rows cl = lim;
- cl.gen = 2;
- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,
- cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);
- Rows tr = lim;
- tr.gen = 3;
- topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);
- } else {
- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,
- lim, c.scores, ss, s);
- if (l.cand_src) {
- Rows cr = lim;
- cr.kcap = g.cand_topk;
- cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,
- (int) cand_stride(), s);
- }
- topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);
- }
- }
- }
- }
- if (l.kv_src && ng > 0) {
- Finish fc;
- fc.rd = g.rd;
- fc.pos = (int) (g_first * R);
- fc.pos_step = R;
- fc.freqs = freqs;
- fc.quant = 2;
- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
- fc.pdst_stride = (long long) ckv_rb();
- finish(const_cast<float*>(latent), ng, HD, HD, fc, s);
- if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once
- const int slot = wp.k % 2; // they took the chunk before last's)
- if (wp.k >= 2)
- for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);
- Xfer x;
- x.src[0] = fc.pdst;
- x.dst[0] = sr.stage(slot);
- x.bytes[0] = (size_t) ng * ckv_rb();
- x.n = 1;
- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();
- x.bytes[1] = (size_t) ng * ik_rb();
- x.n = 2;
- }
- xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);
- }
- }
- ckv = sr.ckv[(size_t) l.dev];
- }
- // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
- if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
- for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
- const int na = std::min(c.A, T - a0);
- if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;
- Finish fr;
- fr.rd = g.rd;
- fr.pos = (int) (p0 + a0);
- fr.vpp = NHL;
- fr.freqs = freqs;
- finish(c.q, na * NHL, HD, HD, fr, s);
- if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);
- AttnRows ar;
- ar.gen = 1;
- ar.p0 = p0;
- ar.t0 = a0;
- ar.win = g.win;
- ar.R = l.ratio;
- ar.kcap = g.ix_topk;
- sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
- na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
- if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);
- Finish fo;
- fo.rd = g.rd;
- fo.pos = (int) (p0 + a0);
- fo.vpp = NHL;
- fo.freqs = freqs;
- fo.inverse = 1;
- fo.quant = 1;
- finish(c.o, na * NHL, HD, HD, fo, s);
- if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);
- x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
- const int per = HQ / OG;
- for (int gi = 0; gi < OG; ++gi)
- if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,
- OG * g.o_lora, 0.0f, err))
- return false;
- x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
- if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
- if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);
- }
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);
- hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)
- prof_mark(d, 1);
- // the MoE
- if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
- hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);
- hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);
- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
- gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)
- for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)
- const int nr = std::min(kVlRows, T - r0);
- VlBits vb;
- const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);
- route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,
- c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);
- }
- const int n_vs = L0.n_slots + (int) L0.host_list.size();
- CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)
- moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);
- CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));
- // the shared expert into acc while the host waits for the bounds
- if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))
- return false;
- swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);
- x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);
- if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;
- prof_mark(d, 2);
- CK(cudaStreamSynchronize(s));
- std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);
- if (dump_r != nullptr && mi == 0) {
- const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;
- std::vector<int> ri(np), rt(nk);
- std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);
- CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));
- CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
- cudaMemcpyDeviceToHost));
- CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};
- std::fwrite(hdr, 8, 8, dump_r);
- std::fwrite(ri.data(), 4, np, dump_r);
- std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);
- std::fwrite(b.data(), 4, b.size(), dump_r);
- std::fwrite(rt.data(), 4, nk, dump_r);
- std::fwrite(gl.data(), 4, gl.size(), dump_r);
- std::fwrite(gb.data(), 4, gb.size(), dump_r);
- std::fflush(dump_r);
- }
- if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids
- std::vector<int> ri((size_t) T * kPicks);
- CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));
- std::lock_guard<std::mutex> lk(dump_mu);
- const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};
- std::fwrite(hd, 1, 3, trace_f);
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < l.k_exp; ++i) {
- const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];
- std::fwrite(&v, 2, 1, trace_f);
- }
- }
- if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- for (int e = 0; e < l.n_exp; ++e) {
- const int v = L0.vs_pin[e];
- pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);
- }
- st.experts += (int64_t) T * l.k_exp;
- }
- if (c.flow) {
- if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 3);
- return true;
- }
- // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones
- // in shares between them (the device works while the copies run)
- const int di = l.dev, P = Chunk::kParts;
- int nb = 0;
- for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;
- int v_done = 0;
- for (int bi = 0; bi < nb; ++bi) {
- const Batch& bt = S.b[S.kp];
- const int half = (int) (S.kp % P);
- CK(cudaStreamWaitEvent(s, c.filled[half], 0));
- const int v0 = L0.n_slots + bt.j0;
- if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,
- b.data(), v0, v0 + bt.n, T, err))
- return false;
- CK(cudaEventRecord(c.freed[half], s));
- ++S.kp;
- if (!copy_ahead(d, c, S, di, err)) return false;
- const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);
- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;
- v_done = v_to;
- }
- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 3);
- return true;
- }
-
- // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)
- void plan_streams() {
- for (Layer& l : L) {
- if (l.mtp || l.vs_pin == nullptr) continue;
- l.host_list.clear();
- for (int e = 0; e < l.n_exp; ++e)
- if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);
- for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)
- Layer& x = member(l, m);
- int j = 0;
- for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;
- }
- }
- }
-
- // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows
- Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {
- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- void* dv[4] = {c.h, c.pre, c.topk, c.cand};
- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
- const size_t nb[4] = {nh, np, nk, nc};
- Xfer x;
- x.n = 4;
- for (int i = 0; i < 4; ++i) {
- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
- x.dst[i] = put ? (void*) hv[i] : dv[i];
- x.bytes[i] = nb[i];
- }
- return x;
- }
-
- // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the
- // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows
- // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across
- // layers and chunks
- struct Run {
- std::vector<int64_t> p0;
- std::vector<int> T;
- std::vector<unsigned> seq;
- int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
- };
- // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
- // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
- // layers and back before the next - so a layer's experts are read from the files once for the whole run (the
- // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk
- bool run_dev_layers(int di, const Run& run, std::string& err) {
- Dev& d = devs[(size_t) di];
- Chunk& c = *chunks[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc, nk = (int) run.T.size();
- Streamer S; // (none: the flow streams)
- const size_t hb = chunk_hand_bytes();
- // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while
- // the one before runs (slot u % 2, once unit u - 2's copy of it is done)
- std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
- for (int il : c.fl_layers)
- if (L[(size_t) il].eg >= 0)
- for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});
- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
- auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
- if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
- const int li = egu[u].first, k = egu[u].second;
- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);
- };
- size_t eg_u = 0;
- std::future<void> eg_next;
- if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);
- for (size_t li = 0; li < c.fl_layers.size(); ++li) {
- const int il = c.fl_layers[li];
- Layer& L0 = L[(size_t) il];
- const bool last_l = li + 1 == c.fl_layers.size();
- for (int k = 0; k < nk; ++k) {
- const int64_t p0 = run.p0[(size_t) k];
- const int T = run.T[(size_t) k], slot = k % 2;
- const bool last_k = k + 1 == nk;
- WinPlan wp;
- wp.par = true;
- wp.async = true;
- wp.win = run.seq[(size_t) k];
- wp.k = k;
- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
- if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way
- const double te = now_ms();
- eg_next.get();
- st.engram_ms += now_ms() - te;
- wp.eg_slot = (int) (eg_u % 2);
- if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);
- }
- if (li == 0) { // the chunk's input: its embedding rows
- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
- for (int t = 0; t < T; ++t)
- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
- (size_t) D * 2);
- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.emb_ev[slot], s));
- embed(c.emb, T, c.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = c.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(c.pre, T, s);
- } else { // or its state after the layer before
- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
- for (int i = 0; i < x.n; ++i)
- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));
- }
- if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (it != g.d_targets.end()) {
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int nr = std::min(T, tap_cap);
- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
- CK(cudaStreamWaitEvent(s, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
- (size_t) nr, cudaMemcpyDeviceToHost, s));
- }
- }
- if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;
- if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
- if (!last_l) { // the chunk's state kept for the next layer
- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
- for (int i = 0; i < x.n; ++i)
- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));
- continue;
- }
- // through every layer: the chunk's end, and the logits
- if ((size_t) k < chunk_done.size()) {
- CK(cudaEventRecord(chunk_done[(size_t) k], s));
- chunk_rec[(size_t) k].store(1, std::memory_order_release);
- }
- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
- for (int r0 = 0; r0 < T; r0 += kWin) {
- const int nr = std::min(kWin, T - r0);
- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, nr, err)) return false;
- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaStreamSynchronize(s));
- }
- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
- const int t0 = T - n_logits;
- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
- }
- }
- lm_layers.store((int) li + 1, std::memory_order_release);
- }
- return true;
- }
- bool run_dev_chunks(int di, const Run& run, std::string& err) {
- Dev& d = devs[(size_t) di];
- Chunk& c = *chunks[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc, nk = (int) run.T.size();
- Streamer S;
- const int B = std::max(1, c.ring_n / Chunk::kParts);
- if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)
- CK(cudaStreamSynchronize(s));
- CK(cudaStreamSynchronize(c.cs));
- c.items.clear();
- c.base = c.n_read = c.n_dma = c.n_comp = 0;
- c.n_file = 0;
- std::fill(c.stage_item.begin(), c.stage_item.end(), -1);
- c.fl_layers.clear();
- for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)
- for (int m = 0; m < L[(size_t) il].nm; ++m)
- if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
- c.fl_T = run.T;
- c.spec_k = c.spec_il = -1;
- c.wait_ms = 0;
- c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
- c.lc_il = -1;
- c.lc_used = 0;
- c.lc_first = c.lc_chk = 0;
- c.io_d0 = c.io->direct_bytes();
- c.io_b0 = c.io->buffered_bytes();
- if (!c.fl_layers.empty()) {
- flow_spec(c, 0, c.fl_layers[0], run.T[0]);
- if (!flow_pump(c, err)) return false;
- }
- if (c.lm) return run_dev_layers(di, run, err);
- }
- for (int k = 0; k < nk && !c.flow; ++k)
- for (int il = 0; il < g.nl; ++il) {
- const Layer& L0 = L[(size_t) il];
- bool here = false;
- for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
- if (!here) continue;
- for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)
- S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});
- }
- if (!S.b.empty() && c.ring == nullptr) {
- err = "no slots to stream experts into on CUDA" + std::to_string(d.id);
- return false;
- }
- std::atomic<bool> ra_stop{false};
- std::thread ra;
- if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });
- struct Join { // (every way out of the run: the read-ahead ends before S does)
- std::atomic<bool>& stop;
- std::thread& th;
- ~Join() {
- stop.store(true);
- if (th.joinable()) th.join();
- }
- } ra_join{ra_stop, ra};
- if (!copy_ahead(d, c, S, di, err)) return false;
- // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once
- // chunk k - 2's copy of it is done)
- std::vector<int> egs;
- for (const Layer& l0 : L)
- for (int m = 0; m < l0.nm; ++m)
- if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);
- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
- auto read_eg = [this, &c, &run, &egs, eg_per](int k) {
- if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);
- for (int li : egs)
- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);
- };
- std::future<void> eg_next;
- if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
- for (int k = 0; k < nk; ++k) {
- const int64_t p0 = run.p0[(size_t) k];
- const int T = run.T[(size_t) k], slot = k % 2;
- const bool last_k = k + 1 == nk;
- if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way
- const double te = now_ms();
- eg_next.get();
- if (d.mi == 0) st.engram_ms += now_ms() - te;
- if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);
- }
- WinPlan wp;
- wp.par = true;
- wp.async = true;
- wp.win = run.seq[(size_t) k];
- wp.k = k;
- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
- bool started = false;
- std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
- Layer& L0 = L[(size_t) il];
- int mi = -1;
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == di) mi = m;
- if (mi < 0) continue;
- if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off
- started = true;
- if (il == 0) {
- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
- for (int t = 0; t < T; ++t)
- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
- (size_t) D * 2);
- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
- CK(cudaEventRecord(c.emb_ev[slot], s));
- embed(c.emb, T, c.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = c.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(c.pre, T, s);
- } else {
- Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
- xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);
- set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);
- }
- }
- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
- if (sidx < 0 || got[(size_t) sidx]) continue;
- got[(size_t) sidx] = 1;
- Src& sr = srcs[(size_t) sidx];
- if (L[(size_t) sr.layer].grp == L0.grp) continue;
- const int64_t g_first = p0 / sr.ratio;
- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
- Xfer x;
- if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {
- x.src[x.n] = sr.stage(slot);
- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
- x.bytes[x.n++] = (size_t) ng * ckv_rb();
- }
- if (ng > 0 && sr.ik[(size_t) di] != nullptr) {
- x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();
- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
- x.bytes[x.n++] = (size_t) ng * ik_rb();
- }
- if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);
- set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)
- }
- if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (it != g.d_targets.end()) {
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int nr = std::min(T, tap_cap);
- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
- CK(cudaStreamWaitEvent(s, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
- (size_t) nr, cudaMemcpyDeviceToHost, s));
- }
- }
- if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;
- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
- if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
- const int nnext = L[(size_t) il + 1].nm;
- if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);
- xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);
- }
- }
- if (di == head_dev && (size_t) k < chunk_done.size()) {
- CK(cudaEventRecord(chunk_done[(size_t) k], s));
- chunk_rec[(size_t) k].store(1, std::memory_order_release);
- }
- if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits
- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
- for (int r0 = 0; r0 < T; r0 += kWin) {
- const int nr = std::min(kWin, T - r0);
- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, nr, err)) return false;
- if (di != head_dev) continue;
- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaStreamSynchronize(s));
- }
- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
- const int t0 = T - n_logits;
- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
- }
- }
- }
- return true;
- }
-
- // ---- a prompt's path: the flow or windows of 16
- // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a
- // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in
- // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the
- // flow, as before)
- void calibrate_paths() {
- if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;
- if (const char* v = std::getenv("DS4_PATH_PICK"))
- if (std::atoi(v) == 0) return;
- int di = -1;
- for (size_t i = 0; i < chunks.size() && di < 0; ++i)
- if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;
- if (di < 0) return;
- Chunk& c = *chunks[(size_t) di];
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- const double t0 = now_ms();
- // the DMA: staging -> the ring (once to warm up, once timed)
- const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});
- double ms = 0;
- for (int rep = 0; rep < 2; ++rep) {
- const double a = now_ms();
- if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||
- cudaStreamSynchronize(c.cs) != cudaSuccess) {
- cudaGetLastError();
- return;
- }
- ms = now_ms() - a;
- }
- pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);
- // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are
- auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {
- const double a = now_ms();
- left.store((int) jobs.size());
- c.io->push(jobs.data(), (int) jobs.size());
- while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
- return now_ms() - a;
- };
- {
- const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;
- std::vector<uint8_t> src(part * 8, 1);
- std::vector<IoQueue::Job> jobs(8);
- std::atomic<int> left{0};
- std::atomic<bool> bad{false};
- for (int i = 0; i < 8; ++i) {
- jobs[(size_t) i].mem = src.data() + (size_t) i * part;
- jobs[(size_t) i].dst = c.stage + (size_t) i * part;
- jobs[(size_t) i].len = part;
- jobs[(size_t) i].left = &left;
- jobs[(size_t) i].bad = &bad;
- }
- run_jobs(jobs, left);
- const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);
- pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);
- }
- // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:
- // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane
- // three times (from the disk, then in RAM: the faster of two)
- int best = -1, most = 0;
- for (int il = 0; il < g.nl; ++il) {
- int n = 0;
- for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;
- if (n > most) most = n, best = il;
- }
- if (best < 0) return;
- const Layer& l = L[(size_t) best];
- std::vector<int> cold;
- const int n_read = std::min(8, c.S), n_cpu = 8;
- for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {
- if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;
- bool res = false;
- for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);
- if (!res) cold.push_back(e);
- }
- if ((int) cold.size() >= n_read + n_cpu) {
- std::vector<IoQueue::Job> jobs;
- std::atomic<int> left{0};
- std::atomic<bool> bad{false};
- for (int i = 0; i < n_read; ++i) {
- uint8_t* sb = c.stage + (size_t) i * c.stage_stride;
- for (int r = 0; r < 3; ++r) {
- IoQueue::Job j;
- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
- j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];
- j.len = mx_part_bytes(l, r);
- j.dst = sb;
- j.left = &left;
- j.bad = &bad;
- j.map = pk.at(l.mx_file[r], j.off);
- jobs.push_back(j);
- sb += flow_region(r);
- }
- }
- const double rd = run_jobs(jobs, left);
- if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);
- }
- { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM
- const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;
- std::vector<Ent> ents;
- for (int i = 0; i < n_cpu; ++i) {
- Ent en;
- en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;
- if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)
- for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)
- if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;
- if (en.e < 0) break;
- en.n = 1;
- en.slot0 = i;
- en.tok[0] = 0;
- ents.push_back(en);
- }
- if (ents.empty()) return;
- std::vector<float> x((size_t) g.dim);
- for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);
- const double bytes = (double) ents.size() * (double) l.blob;
- double a = now_ms();
- cpu_experts(l, x.data(), ents, (size_t) g.dim);
- const double t_first = now_ms() - a;
- double t_warm = 1e30;
- for (int rep = 0; rep < 2; ++rep) {
- a = now_ms();
- cpu_experts(l, x.data(), ents, (size_t) g.dim);
- t_warm = std::min(t_warm, now_ms() - a);
- }
- pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);
- if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);
- }
- pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;
- if (o.verbose) {
- char dk[32] = "not measured", cc[32] = "not measured";
- if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);
- if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);
- std::fprintf(stderr,
- "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "
- "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",
- pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);
- }
- }
- // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's
- // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)
- double pm_rate(const Layer& l, int& n_cold) const {
- const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- double sum = 0, cold = 0;
- n_cold = 0;
- for (int e = 0; e < l.n_exp; ++e) {
- sum += pr[e];
- if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;
- }
- if (n_cold == 0) return 0.0;
- const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;
- return pm.m * (double) l.k_exp * share / n_cold;
- }
- // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))
- double pm_picks(int n) const {
- double d = 0;
- for (int il = 0; il < g.nl; ++il) {
- int nc = 0;
- const double a = pm_rate(L[(size_t) il], nc);
- d += nc * (1.0 - std::exp(-a * n));
- }
- return d;
- }
- // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation
- // that (bisection over its log; a running mean of the log)
- void pm_learn(int n, double picked) {
- if (!pm.on || n < 2 || picked < 1) return;
- const double m0 = pm.m;
- double lo = std::log(1e-3), hi = std::log(1e3);
- for (int it = 0; it < 40; ++it) {
- pm.m = std::exp(0.5 * (lo + hi));
- (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);
- }
- const double obs = 0.5 * (lo + hi);
- pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);
- ++pm.n_m;
- }
- // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's
- // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -
- // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:
- // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at
- // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
- // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
- // faster (by half, with less than kPickRes in RAM); said with --verbose
- bool pick_path(int n) {
- pm.last = -1;
- if (!pm.on || n >= kPickMax) return true;
- double f_res = 0, f_cold = 0, w = 0;
- const int nw = n / kWin, rem = n % kWin;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- int nc = 0;
- const double a = pm_rate(l, nc);
- if (nc == 0) continue;
- int sampled = 0, res = 0, seen = 0;
- const int step = std::max(1, nc / 32);
- for (int e = 0; e < l.n_exp; ++e) {
- if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;
- bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;
- if (!r) {
- r = true;
- for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);
- }
- ++sampled;
- res += r;
- }
- const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;
- auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };
- f_res += share * u(n) * b;
- f_cold += (1.0 - share) * u(n) * b;
- w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;
- }
- const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;
- const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);
- const double t_flow = f_res / r_fr + f_cold / r_fc;
- const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;
- const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;
- // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the
- // fixed costs of every layer on every GPU again, which a pass of the flow pays once)
- const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;
- // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane
- // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would
- // cost the most; windows win where they are all in RAM)
- const double res_share = f_res / std::max(1.0, f_res + f_cold);
- const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));
- pm.last = flow ? 1 : 0;
- pm.last_cls = cls;
- pm.last_nw = n_win;
- pm.last_pred = flow ? t_flow : t_win;
- if (o.verbose)
- std::fprintf(stderr,
- "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "
- "experts' %.1f GB in RAM)\n",
- n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,
- 100.0 * res_share, (f_res + f_cold) / 1e9);
- return flow;
- }
- // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more
- static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }
- // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs
- // is mostly the fixed costs a window pays again and again)
- double pm_k(int c) const {
- for (int dc = 0; dc < 4; ++dc)
- for (int cc : {c - dc, c + dc})
- if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];
- return 1.0;
- }
- // n tokens through windows of 16, the last one's logits_rows rows of logits
- bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {
- logits_all = false;
- for (int a = 0; a < n; a += kWin) {
- const int T = std::min(kWin, n - a);
- if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;
- }
- return true;
- }
- // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken
- // / predicted), windows' time a window - running means
- void path_done(double ms) {
- if (pm.last < 0 || pm.last_pred <= 0) return;
- if (pm.last == 0) {
- const double w = ms / 1000.0 / std::max(1, pm.last_nw);
- pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;
- ++pm.win_n;
- } else {
- const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));
- double& k = pm.k[0][pm.last_cls];
- int64_t& n = pm.kn[0][pm.last_cls];
- k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;
- ++n;
- }
- pm.last = -1;
- }
- // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the
- // first would read from the files again more than the chunks' states moved to host memory and back at every layer
- // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,
- // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB
- // sets it) - at least one expert, or not layer-major
- bool lm_begin(const Run& run) {
- static const char* env = std::getenv("DS4_LAYER_MAJOR");
- if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;
- Chunk& c = *chunks[0];
- if (!c.flow) return false;
- size_t need = 0, n_exp = 0, blob_max = 0;
- double file_b = 0;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- size_t ne = 0;
- for (int e : l.host_list)
- if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;
- const size_t stride = (l.blob + 63) & ~(size_t) 63;
- need = std::max(need, ne * stride);
- n_exp = std::max(n_exp, (size_t) l.n_exp);
- blob_max = std::max(blob_max, stride);
- file_b += (double) ne * (double) l.blob;
- }
- int64_t rows = 0;
- for (int T : run.T) rows += T;
- const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;
- const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;
- if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;
- size_t lc_bytes = need;
- if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);
- else {
- const uint64_t avail = ram_avail();
- if (avail != ~uint64_t{0})
- lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);
- }
- if (lc_bytes < blob_max) return false;
- void* st_p = nullptr;
- c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;
- if (!c.lm_pinned) {
- cudaGetLastError();
- st_p = os::anon_alloc(state_b, -1, 1);
- if (st_p == nullptr) return false;
- }
- c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);
- if (c.lc == nullptr) {
- if (c.lm_pinned) cudaFreeHost(st_p);
- else os::anon_free(st_p, state_b);
- return false;
- }
- c.lm_state = (uint8_t*) st_p;
- c.lm_state_bytes = state_b;
- c.lc_bytes = lc_bytes;
- c.lc_of.assign(n_exp, -1);
- c.lc_writer.assign(n_exp, -1);
- c.lm = true;
- lm_layers.store(0);
- return true;
- }
- void lm_end() {
- for (size_t di = 0; di < chunks.size(); ++di) {
- Chunk& c = *chunks[di];
- if (!c.lm) continue;
- cudaSetDevice(devs[di].id);
- cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's
- cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)
- for (size_t i = c.base; i < c.n_read; ++i)
- while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
- if (c.lm_pinned) cudaFreeHost(c.lm_state);
- else os::anon_free(c.lm_state, c.lm_state_bytes);
- os::anon_free(c.lc, c.lc_bytes);
- c.lm_state = c.lc = nullptr;
- c.lm_state_bytes = c.lc_bytes = 0;
- c.lm = c.lm_pinned = false;
- }
- }
-
- // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
- // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
- bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
- if (n < 1) return true;
- if (pos + n > o.max_context) {
- err = "the context is full";
- return false;
- }
- for (int t = 0; t < n; ++t)
- if (tk[t] < 0 || tk[t] >= g.vocab) {
- err = "token " + std::to_string(tk[t]) + " out of range";
- return false;
- }
- const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);
- Run run;
- for (int a = 0; a < n; a += per) {
- run.p0.push_back(pos + a);
- run.T.push_back(std::min(per, n - a));
- run.seq.push_back(++win_seq);
- }
- logits_all = logits_rows > kWin;
- if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
- for (int t = 0; t < n; ++t) {
- toks.push_back(tk[t]);
- ctoks.push_back(ctok(pos + t, tk[t]));
- }
- plan_streams();
- if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way
- run.snap_pos = pos + n - 1;
- for (int il = 0; il < g.nl; ++il) {
- Layer& l = L[(size_t) il];
- Dev& d = devs[(size_t) l.dev];
- if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;
- if (l.c_kv != nullptr && l.snap_kv == nullptr &&
- (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))
- return false;
- }
- }
- const int T_last = run.T.back();
- n_logits = logits_all ? 0 : std::min(logits_rows, T_last);
- const double tl = now_ms();
- { // each chunk's end on the head device, for the progress reports
- cudaSetDevice(devs[(size_t) head_dev].id);
- while (chunk_done.size() < run.T.size()) {
- cudaEvent_t e = nullptr;
- CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
- chunk_done.push_back(e);
- }
- chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);
- for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);
- }
- prof_chunk = prof_on;
- const bool lm = lm_begin(run);
- struct LmEnd { // (every way out: its host memory freed once the device is done with it)
- Impl* p;
- bool on;
- ~LmEnd() {
- if (on) p->lm_end();
- }
- } lm_end_{this, lm};
- std::vector<std::string> errs(devs.size());
- for (size_t di = 0; di < devs.size(); ++di)
- workers[di]->submit([this, di, &run, &errs] {
- std::string e;
- if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
- });
- size_t reported = 0;
- for (;;) {
- bool all = true;
- for (auto& w : workers) all = all && w->idle();
- if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
- const int nd = lm_layers.load(std::memory_order_acquire);
- if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
- } else if (progress) { // chunks through every layer (the last layer's device past their event)
- while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&
- cudaEventQuery(chunk_done[reported]) == cudaSuccess)
- ++reported;
- if (reported > 0 && reported < run.T.size())
- progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);
- }
- if (all) break;
- std::this_thread::sleep_for(std::chrono::milliseconds(50));
- }
- for (const std::string& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- st.layer_ms += now_ms() - tl;
- eg_stats();
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- const double th = now_ms();
- CK(cudaStreamSynchronize(d.st));
- st.head_ms += now_ms() - th;
- if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed
- for (Dev& dv : devs) {
- cudaSetDevice(dv.id);
- CK(cudaStreamSynchronize(dv.st));
- }
- cudaSetDevice(d.id);
- prof_collect();
- std::fprintf(stderr,
- "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "
- "(with the waits for streamed ones), between layers %.0f ms\n",
- n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);
- for (double& v : cprof_ms) v = 0;
- prof_chunk = false;
- }
- if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {
- const double ms = now_ms() - tl;
- for (size_t di = 0; di < chunks.size(); ++di) {
- const Chunk& c = *chunks[di];
- if (!c.flow) continue;
- const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;
- const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;
- const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)
- char lcs[160] = "";
- if (c.lm)
- std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "
- "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);
- std::fprintf(stderr,
- "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "
- "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "
- "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "
- "%.0f ms for reads\n",
- devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),
- (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,
- (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,
- (unsigned long long) c.it_spec_unused, c.wait_ms);
- }
- }
- if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)
- bool tp = false;
- for (const Layer& l : L) tp = tp || l.nm > 1;
- double picked = 0;
- for (const auto& cp : chunks)
- if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;
- if (!tp) pm_learn(n, picked);
- }
- if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)
- pos += n;
- st.tokens += n;
- cache_tick(n);
- const int nr = std::min(T_last, tap_cap);
- if (!seed_drafter(pos - nr, nr, 0, err)) return false;
- if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -
- // the next feed seeds them again from its own rows, past this position)
- Check& ckp = ck[snap_slot & 1];
- ckp.pos = run.snap_pos;
- ckp.win.resize(L.size());
- ckp.comp.resize(L.size() * 2);
- for (size_t i = 0; i < L.size(); ++i) {
- Layer& l = L[i];
- Dev& dv = devs[(size_t) l.dev];
- cudaSetDevice(dv.id);
- const bool kept = !l.mtp && l.snap_win != nullptr;
- ckp.win[i].resize((size_t) g.win * win_rb());
- CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),
- cudaMemcpyDeviceToHost, dv.st));
- if (l.c_kv) {
- ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);
- ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
- CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,
- ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));
- CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,
- ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));
- }
- }
- for (Dev& dv : devs) {
- cudaSetDevice(dv.id);
- CK(cudaStreamSynchronize(dv.st));
- }
- }
- return true;
- }
-
- // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)
- void plan_window(int64_t p0, int T, WinPlan& wp) {
- wp.par = par;
- wp.async = par || (!sync_decode && T <= async_max_t);
- wp.vl = 0;
- for (int t = 0; t < T && !imgs.empty(); ++t)
- if (in_image(p0 + t)) wp.vl |= 1u << t;
- for (int il = 0; il < g.nl; ++il) {
- const Layer& l = L[(size_t) il];
- if (l.ratio > 0) {
- const int R = l.ratio;
- int n[kWin], k[kWin], clen_max = 0;
- for (int t = 0; t < T; ++t) {
- n[t] = (int) ((p0 + t + 1) / R);
- k[t] = std::min(g.ix_topk, n[t]);
- clen_max = std::max(clen_max, n[t]);
- }
- if (l.ix_src) {
- for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];
- if (clen_max > 0 && l.cand_src)
- for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);
- for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];
- }
- for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];
- }
- if (wp.async) {
- wp.seq[il] = ++seq_issued;
- layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);
- }
- }
- }
-
- int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }
- // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's
- // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows
- void head_cols(int nm, int mi, int& v0, int& vn) const {
- const Tensor* tw = pk.get("head.weight");
- if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {
- vn = g.vocab / nm;
- v0 = mi * vn;
- } else
- split128(g.vocab, nm, mi, v0, vn);
- }
- // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's
- // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two
- // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather
- // false: each member keeps its share (the drafter's markov steps over the group, mk_split)
- bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {
- const int V = g.vocab, D = g.dim, nm = head_nm();
- int v0 = 0, vn = 0, Vmax = 0, z = 0;
- head_cols(nm, mi, v0, vn);
- head_cols(nm, 0, z, Vmax);
- Group& G = groups[(size_t) d.grp];
- float* out = mi == 0 ? d.logits : d.logits_part;
- const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;
- cudaSetDevice(d.id);
- // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4
- // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)
- const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head)); // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)
- for (int c = 0; c < rows; c += per)
- if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))
- return false;
- if (nm == 1 || !gather) return true;
- const unsigned seq = ++d.head_seq;
- const int slot = (int) (seq % 2);
- auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };
- auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };
- if (mi > 0) {
- if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);
- Xfer x;
- x.n = 1;
- x.src[0] = d.logits_part;
- x.dst[0] = stage(mi);
- x.bytes[0] = (size_t) rows * vn * 4;
- xfer_put(x, flag(mi), seq, d.xcnt, d.st);
- return true;
- }
- for (int m = 1; m < nm; ++m) {
- int m0 = 0, mn = 0;
- head_cols(nm, m, m0, mn);
- Xfer x;
- x.n = 1;
- x.src[0] = stage(m);
- x.dst[0] = d.logits_part;
- x.bytes[0] = (size_t) rows * mn * 4;
- xfer_get(x, flag(m), seq, d.st);
- CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,
- (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));
- }
- set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);
- return true;
- }
-
- // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)
- bool head_rows_dev(int di, int T, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- const int D = g.dim, H = g.hc;
- if (n_logits <= 0) return true;
- const int t0 = T - n_logits;
- hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);
- return head_share(d, d.mi, d.x, n_logits, err);
- }
-
- // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read
- // h_taps)
- bool tap(Dev& d, int il, int T, std::string& err) {
- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
- if (g.n_mtp == 0 || it == g.d_targets.end()) return true;
- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
- const int D = g.dim;
- cudaSetDevice(d.id);
- hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);
- CK(cudaStreamWaitEvent(d.st, ev_seed, 0));
- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,
- cudaMemcpyDeviceToHost, d.st));
- return true;
- }
-
- bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {
-#if defined(STRATA_METAL)
- struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)
- bool on;
- BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }
- ~BusyAll() { if (on) ds4_busy_end(); }
- } busy_all;
-#endif
- if (T < 1 || T > kWin) {
- err = "a window of " + std::to_string(T) + " tokens";
- return false;
- }
- if (pos + T > o.max_context) {
- err = "the context is full";
- return false;
- }
- for (int t = 0; t < T; ++t)
- if (tk[t] < 0 || tk[t] >= g.vocab) {
- err = "token " + std::to_string(tk[t]) + " out of range";
- return false;
- }
- const int64_t p0 = pos;
- for (int t = 0; t < T; ++t) {
- toks.push_back(tk[t]);
- ctoks.push_back(ctok(pos + t, tk[t]));
- }
- // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is
- // left of their reads by then is the window's Engram time
- const double te = now_ms();
- const int64_t eg_w0 = eg_wait_ns.load();
- for (int li = 0; li < pk.eg_layers; ++li) {
- EgBatch& b = *eg_win[(size_t) li];
- if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)
- eg_issue(p0, T, li, nullptr, b);
- }
- st.engram_ms += now_ms() - te;
- const int D = g.dim;
- for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)
- std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);
- plan_window(p0, T, wplan);
- prof_window = prof_on && T == 1;
- n_logits = std::min(logits_rows, T);
- const double tl = now_ms();
- if (par) {
- if (!forward_par(p0, T, err)) return false;
- } else if (!forward_serial(p0, T, err)) return false;
- st.layer_ms += now_ms() - tl;
- for (auto& b : eg_win)
- if (b->live) eg_finish(*b);
- st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;
- eg_stats();
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- const double th = now_ms();
- CK(cudaStreamSynchronize(d.st));
- st.head_ms += now_ms() - th;
- prof_collect();
- pos += T;
- st.tokens += T;
- cache_tick(T);
- if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;
- return true;
- }
-
- // one thread: the layers device after device, the hand-offs through pinned memory and events
- void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer
- ds4_metal_flush();
- const float* hp = (const float*) strata::gpu::host_view(dev);
- size_t nf = 0, first = 0; float mx = 0.0f;
- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }
- std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);
- }
- void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains
- ds4_metal_flush();
- const float* hp = (const float*) strata::gpu::host_view(d.h);
- float mx = 0.0f; size_t nf = 0;
- const size_t n = (size_t) T * g.hc * g.dim;
- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }
- std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);
- }
- bool forward_serial(int64_t p0, int T, std::string& err) {
- const int D = g.dim, H = g.hc;
- int cur = L[0].dev;
- if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();
- {
- Dev& d = devs[(size_t) cur];
- cudaSetDevice(d.id);
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, d.st);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, d.st);
- if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);
- }
- std::vector<float> dump;
- if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
- for (int il = 0; il < g.nl; ++il) {
- Layer& l = L[(size_t) il];
- if (l.dev != cur) {
- Dev& a = devs[(size_t) cur];
- Dev& b = devs[(size_t) l.dev];
- cudaSetDevice(a.id);
- const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- uint8_t* bb = a.bounce;
- CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));
- CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));
- CK(cudaEventRecord(a.ev_out, a.st));
- cudaSetDevice(b.id);
- CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));
- CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));
- CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));
- cur = l.dev;
- }
- if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
- if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);
- if (!layer(l, 0, p0, T, wplan, err)) return false;
- if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
- Dev& dd = devs[(size_t) l.dev];
- std::vector<float> hv((size_t) T * H * D);
- cudaSetDevice(dd.id);
- CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));
- CK(cudaStreamSynchronize(dd.st));
- for (int t = 0; t < T; ++t)
- std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,
- (size_t) H * D * 4);
- }
- }
- if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);
- if (cur != head_dev) {
- err = "the head is not on the last layer's GPU";
- return false;
- }
- return head_rows_dev(head_dev, T, err);
- }
-
- // threads: each device's thread enqueues its layers (a group's members each their share) at once, the
- // devices waiting on each other through host-mapped flags (the window's seq)
- bool forward_par(int64_t p0, int T, std::string& err) {
- wplan.win = ++win_seq;
- std::vector<std::string> errs(devs.size());
- for (size_t di = 0; di < devs.size(); ++di)
- workers[di]->submit([this, di, p0, T, &errs] {
- std::string e;
- if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
- });
- for (auto& w : workers) w->wait();
- for (const std::string& e : errs)
- if (!e.empty()) {
- err = e;
- return false;
- }
- return true;
- }
-
- // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer
- Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {
- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
- nc = (size_t) T * cand_stride() * 4;
- void* dv[4] = {d.h, d.pre, d.topk, d.cand};
- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
- const size_t nb[4] = {nh, np, nk, nc};
- Xfer x;
- x.n = 4;
- for (int i = 0; i < 4; ++i) {
- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
- x.dst[i] = put ? (void*) hv[i] : dv[i];
- x.bytes[i] = nb[i];
- }
- return x;
- }
-
- bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {
- Dev& d = devs[(size_t) di];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int D = g.dim, H = g.hc;
- bool started = false;
- std::vector<char> got(srcs.size(), 0);
- for (int il = 0; il < g.nl; ++il) {
- Layer& L0 = L[(size_t) il];
- int mi = -1;
- for (int m = 0; m < L0.nm; ++m)
- if (member(L0, m).dev == di) mi = m;
- if (mi < 0) continue;
- if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
- started = true;
- if (il == 0) {
- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
- embed(d.emb_rows, T, d.h, D, H, s);
- if (std::getenv("DS4_DBG_EMB")) {
- ds4_metal_flush();
- float *hp = d.h;
- float hh[4] = {0,0,0,0};
- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
- }
- init_pre(d.pre, T, s);
- } else {
- const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
- xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);
- }
- }
- // the compressed KV / index keys another group's source layer wrote this window
- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
- if (sidx < 0 || got[(size_t) sidx]) continue;
- got[(size_t) sidx] = 1;
- Src& sr = srcs[(size_t) sidx];
- if (L[(size_t) sr.layer].grp == L0.grp) continue;
- const int64_t g_first = p0 / sr.ratio;
- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
- if (ng == 0) continue;
- Xfer x;
- if (sr.ckv[(size_t) di] != nullptr) {
- x.src[x.n] = sr.x_stage;
- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
- x.bytes[x.n++] = (size_t) ng * ckv_rb();
- }
- if (sr.ik[(size_t) di] != nullptr) {
- x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();
- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
- x.bytes[x.n++] = (size_t) ng * ik_rb();
- }
- if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);
- }
- if (mi == 0 && !tap(d, il, T, err)) return false;
- if (!layer(L0, mi, p0, T, wp, err)) return false;
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);
- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
- if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);
- }
- if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);
- return true;
- }
-
- // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,
- // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it
- bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,
- size_t bytes, int from, std::string& err) {
- bool any = false;
- for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);
- if (!any) return true;
- Dev& d = devs[(size_t) from];
- CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));
- CK(cudaEventRecord(ev, d.st));
- for (size_t od = 0; od < devs.size(); ++od) {
- if (copies[od] == nullptr || (int) od == from) continue;
- Dev& o2 = devs[od];
- cudaSetDevice(o2.id);
- CK(cudaStreamWaitEvent(o2.st, ev, 0));
- CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));
- }
- cudaSetDevice(d.id);
- return true;
- }
-
- // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;
- // the partial sums all-reduced across the group)
- bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {
- Layer& l = member(L0, mi);
- Dev& d = devs[(size_t) l.dev];
- cudaSetDevice(d.id);
- cudaStream_t s = d.st;
- const int nm = L0.nm;
- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
- const ArGroup& ag = groups[(size_t) L0.grp].ar;
- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
- if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's
- ds4_metal_flush();
- int shown = 0;
- for (int e = 0; e < l.n_exp && shown < 3; ++e) {
- if (l.slot[(size_t) e] < 0) continue;
- std::vector<uint8_t> want(l.blob, 0);
- read_mx_file(l, e, want.data());
- const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));
- size_t bad = 0, first = 0;
- for (size_t i = 0; i < l.blob; ++i)
- if (want[i] != have[i]) { if (!bad) first = i; ++bad; }
- std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);
- ++shown;
- }
- }
- if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);
- if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
- std::vector<float> hv((size_t) T * H * D);
- cudaSetDevice(d.id);
- ds4_metal_flush();
- {
- const float* hvp = (const float*) strata::gpu::host_view(d.h);
- if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);
- }
- int nf = 0, nzv = 0;
- float mx = 0.0f;
- size_t firstnan = 0, lastnan = 0;
- for (size_t i = 0; i < hv.size(); ++i) {
- const float v = hv[i];
- if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }
- if (v != 0.0f) ++nzv;
- if (std::fabs(v) > mx) mx = std::fabs(v);
- }
- std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",
- l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],
- hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);
- }
- prof_mark(d, 0);
- // Engram
- if (l.eg >= 0) {
- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
- const size_t per = (size_t) cols * (ehd + ehd / 32);
- eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)
- uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;
- for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]
- CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,
- cudaMemcpyHostToDevice, s));
- uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;
- for (int t = 0; t < T; ++t)
- CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
- (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
- dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);
- dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);
- if (std::getenv("DS4_DBG_EG")) {
- dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
- const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
- for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)
- if (!std::isfinite(xv[i])) {
- const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;
- const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;
- const uint8_t* hv = src + t * per + c * ehd;
- std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);
- for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);
- std::fprintf(stderr, "\n");
- break;
- }
- }
- if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
- if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
- dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);
- dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);
- engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
- dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
- }
- dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));
- // attention
- hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
- dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);
- dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);
- dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);
- dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);
- hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
- dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);
- dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
- {
- x3::MvJob j[4];
- int nj = 0;
- j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);
- j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);
- if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);
- if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);
- if (!mv_n(d, j, nj, err)) return false;
- }
- dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
- dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);
- Finish fq;
- fq.w = l.q_norm;
- fq.eps = g.eps;
- finish(d.qr, T, g.q_lora, g.q_lora, fq, s);
- dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
- if (NHL > 0) {
- if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;
- Finish fr;
- fr.rd = g.rd;
- fr.pos = (int) p0;
- fr.vpp = NHL;
- fr.freqs = freqs;
- finish(d.q, T * NHL, HD, HD, fr, s);
- }
- if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);
- // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark
- // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of
- // that ring and to the whole block, and leaves the ring as it was
- ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)
- Finish fk;
- fk.w = l.kv_norm;
- fk.eps = g.eps;
- fk.rd = g.rd;
- fk.pos = (int) p0;
- fk.freqs = freqs;
- fk.quant = 1;
- fk.dst = d.ext + (size_t) g.win * HD;
- fk.dst_stride = HD;
- finish(d.kvr, T, HD, HD, fk, s);
- if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
- dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);
- dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)
- AttnRows ar;
- for (int t = 0; t < T; ++t) {
- if (l.mtp) {
- ar.win0[t] = 0;
- ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);
- ar.nsel[t] = T;
- continue;
- }
- const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)
- ar.win0[t] = first;
- ar.nwin[t] = g.win + t - first + 1;
- ar.nsel[t] = 0;
- }
- const void* ckv = nullptr;
- int ckv_fmt = kKvFp4;
- if (l.ratio > 0) {
- Src& sr = srcs[(size_t) l.src];
- const int R = l.ratio;
- // the groups this window completes: g_first .. g_first + ng - 1
- const int64_t g_first = p0 / R; // the first group whose last position is >= p0
- const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1
- const int ng = (int) std::max<int64_t>(0, g_end - g_first);
- if (l.kv_src) {
- if (R == 1) {
- Finish fc;
- fc.w = l.c_norm;
- fc.eps = g.eps;
- finish(d.ckr, T, HD, HD, fc, s);
- CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));
- } else {
- if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected
- const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;
- CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));
- CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
- }
- compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
- dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);
- dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);
- }
- }
- Rows lim;
- int clen_max = 0;
- for (int t = 0; t < T; ++t) {
- lim.n[t] = (int) ((p0 + t + 1) / R);
- lim.k[t] = std::min(g.ix_topk, lim.n[t]);
- clen_max = std::max(clen_max, lim.n[t]);
- }
- if (l.ix_src) {
- Src& ks = srcs[(size_t) l.ik_src];
- if (l.owns_k && ng > 0) {
- if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;
- Finish fi;
- fi.w = l.ik_norm;
- fi.eps = g.eps;
- fi.rd = g.rd;
- fi.pos = (int) (g_first * R);
- fi.pos_step = R;
- fi.freqs = freqs;
- fi.quant = 3;
- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- fi.pdst_stride = (long long) ik_rb();
- finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
- if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),
- (size_t) ng * ik_rb(), l.dev, err))
- return false;
- }
- if (clen_max > 0) {
- if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;
- Finish fi;
- fi.rd = g.rd;
- fi.pos = (int) p0;
- fi.vpp = g.ix_heads;
- fi.freqs = freqs;
- fi.quant = 3;
- finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
- // (iw rows are 64 apart)
- gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,
- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);
- const long long ss = (long long) o.max_context + 64;
- bool restrict_ = false;
- for (int t = 0; t < T; ++t)
- restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;
- if (l.uses_cand && restrict_) {
- Rows cl; // the candidate positions of each query: its blocks x blk
- int nmax = 0;
- for (int t = 0; t < T; ++t) {
- cl.n[t] = lim.n[t];
- cl.k[t] = wp.cand[l.il][t];
- nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);
- }
- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,
- (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);
- Rows tr;
- for (int t = 0; t < T; ++t) {
- tr.n[t] = wp.cand[l.il][t] * g.cand_blk;
- tr.k[t] = lim.k[t];
- }
- topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);
- } else {
- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,
- lim, d.scores, ss, s);
- if (l.cand_src) {
- Rows cr;
- for (int t = 0; t < T; ++t) {
- cr.n[t] = lim.n[t];
- cr.k[t] = g.cand_topk;
- }
- cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,
- (int) cand_stride(), s);
- }
- topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);
- }
- }
- }
- if (l.kv_src && ng > 0) {
- Finish fc;
- fc.rd = g.rd;
- fc.pos = (int) (g_first * R);
- fc.pos_step = R;
- fc.freqs = freqs;
- fc.quant = 2;
- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
- fc.pdst_stride = (long long) ckv_rb();
- finish(d.latent, ng, HD, HD, fc, s);
- if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),
- (size_t) ng * ckv_rb(), l.dev, err))
- return false;
- if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers
- Xfer x;
- x.src[0] = fc.pdst;
- x.dst[0] = sr.x_stage;
- x.bytes[0] = (size_t) ng * ckv_rb();
- x.n = 1;
- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
- x.dst[1] = sr.x_stage + x_cap() * ckv_rb();
- x.bytes[1] = (size_t) ng * ik_rb();
- x.n = 2;
- }
- xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
- }
- }
- if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);
- dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);
- for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
- ckv = sr.ckv[(size_t) l.dev];
- }
- if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection
- ckv = d.ext + (size_t) g.win * HD;
- ckv_fmt = kKvBf16;
- iota_rows(d.topk, T, T, (int) topk_stride(), s);
- }
- if (NHL > 0) {
- sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,
- 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);
- dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);
- Finish fo;
- fo.rd = g.rd;
- fo.pos = (int) p0;
- fo.vpp = NHL;
- fo.freqs = freqs;
- fo.inverse = 1;
- fo.quant = 1;
- finish(d.o, T * NHL, HD, HD, fo, s);
- dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);
- x3::MvJob j[16];
- const int per = HQ / OG;
- for (int gi = 0; gi < OG; ++gi)
- j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);
- if (!mv_n(d, j, OG, err)) return false;
- if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;
- } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)
- dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);
- if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
- hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
- dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- prof_mark(d, 1);
- // the MoE
- hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
- hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);
- hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
- dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);
- dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);
- dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);
- dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
- if (wp.async) {
- // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members
- // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the
- // router made), and waits for the answer only before the experts' sum
- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
- const unsigned sq = wp.seq[l.il];
- const int slot = (int) (sq % kRing);
- const bool lead = mi == 0;
- int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below
- for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {
- const Layer* nx = ahead_layer(L0, a);
- if (nx == nullptr) break;
- const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);
- float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;
- gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN
- // weights instead: the same hit rate)
- route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);
- na[a - 1] = m;
- }
- route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,
- d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],
- na[1]);
- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;
- swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);
- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;
- prof_mark(d, 2);
- const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;
- if (T == 1 && moe_token_path) {
- x3::MoeArgs ma;
- ma.plan_ptr = d.plan_ptr;
- ma.plan_w = d.rw;
- ma.cpu_part = lead ? resps[slot].part[0] : nullptr;
- ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;
- ma.lay = lay;
- ma.k = l.k_exp;
- ma.n_embd = D;
- ma.n_ff = FF;
- ma.limit = g.lim;
- const bool mx = lay.fmt == gg::kFmtMXFP4;
- if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)
- zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);
- if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
- else x3::moe_gate_up(ma, d.x, d.xg, s);
- prof_mark(d, 3);
- if (lead) wait_seq(&resps[slot].seq, sq, s);
- prof_mark(d, 4);
- if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
- else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
- } else { // each distinct expert read once for the window's tokens
- gf::MoeWin mw;
- mw.plan_ptr = d.plan_ptr;
- mw.T = T;
- mw.k = l.k_exp;
- mw.grp_ptr = d.grp_ptr;
- mw.grp_start = d.grp_start;
- mw.grp_ent = d.grp_ent;
- mw.n_groups = d.n_groups;
- gf::moe_win_group(mw, s);
- x3::WinArgs wa;
- wa.grp_ptr = d.grp_ptr;
- wa.grp_start = d.grp_start;
- wa.grp_ent = d.grp_ent;
- wa.n_groups = d.n_groups;
- wa.plan_ptr = d.plan_ptr;
- wa.plan_w = d.rw;
- wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;
- wa.cpu_stride = kMaxDim;
- wa.cpu_flag = lead ? resps[slot].cpu : nullptr;
- wa.wait_ptr = lead ? &resps[slot].seq : nullptr;
- wa.wait_val = sq;
- wa.x = d.x;
- wa.x_stride = (size_t) D;
- wa.h = d.xg;
- wa.part = d.ed;
- wa.sh_out = d.sh_out;
- wa.out = d.moe_out;
- wa.out_stride = (size_t) D;
- wa.lay = lay;
- wa.T = T;
- wa.k = l.k_exp;
- wa.n_embd = D;
- wa.n_ff = FF;
- wa.limit = g.lim;
- if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {
- err = "exl3 win_experts failed";
- return false;
- }
- }
- if (lead) st.experts += (int64_t) T * l.k_exp;
- if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- prof_mark(d, 5);
- return true;
- }
- if (nm > 1) {
- err = "a tensor-parallel layer on the host-routed path";
- return false;
- }
- // (glog rows are 1024 apart: route reads them so)
- static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes
- static double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;
- const double ph0 = phases ? now_ms() : 0.0;
- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
- VlBits vb = {};
- vb.w[0] = l.mtp ? 0u : wp.vl;
- route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,
- vb.w[0] != 0 ? &vb : nullptr);
- CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));
- CK(cudaEventRecord(d.ev, s));
- // the shared expert while the host looks at the picks
- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;
-#if defined(STRATA_METAL)
- int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer
- if (T == 1 && la_enabled()) {
- for (int a = 1; a <= la_depth(); ++a) {
- const int il = l.il + a;
- if (il >= g.nl) break;
- const Layer& nx = L[(size_t) il];
- if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;
- gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);
- route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,
- d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);
- CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));
- la_n = a;
- }
- if (la_n > 0) CK(cudaEventRecord(d.ev2, s));
- }
-#endif
- if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);
- std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }
- swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);
- if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);
- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;
- const double tw = now_ms();
- CK(cudaEventSynchronize(d.ev));
- st.gpu_wait_ms += now_ms() - tw;
- const double ph2 = phases ? now_ms() : 0.0;
- dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);
- dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);
- dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);
- dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);
- // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the
- // CPU lane's entries
- Combine cm;
- Gather ga;
- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
- int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];
- int nvex = 0, nrows = 0;
- std::vector<Ent> ents;
- std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)
- std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)
- for (int t = 0; t < T; ++t) {
- cm.n[t] = 0;
- cm.cpu[t] = 0;
- for (int i = 0; i < l.k_exp; ++i) {
- const int e = h_idx[t * kPicks + i];
- if (e < 0 || e >= l.n_exp) {
- err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";
- return false;
- }
- ++pr[e];
- l.heat[(size_t) e] += 1.0f;
- if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
- per[(size_t) e].push_back({t, i});
- }
- }
- int nslots = 0;
- for (int e = 0; e < l.n_exp; ++e) {
- const auto& pe = per[(size_t) e];
- if (pe.empty()) continue;
-#if defined(STRATA_METAL)
- const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));
-#else
- const bool on_gpu = l.slot[(size_t) e] >= 0;
-#endif
- if (on_gpu) {
- vex[nvex] = e;
- vrow0[nvex] = nrows;
- vn[nvex] = (int) pe.size();
- for (const auto& tp : pe) {
- ga.src[nrows] = tp.first;
- const int t = tp.first;
- cm.row[t][cm.n[t]] = nrows;
- cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];
- ++cm.n[t];
- ++nrows;
- }
- ++nvex;
- } else {
- for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {
- Ent en;
- en.e = e;
- en.slot0 = nslots;
- en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);
- for (int i = 0; i < en.n; ++i) {
- en.tok[i] = pe[a + (size_t) i].first;
- cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});
- cm.cpu[en.tok[i]] = 1;
- }
- nslots += en.n;
- ents.push_back(en);
- }
- }
- }
- st.experts += (int64_t) T * l.k_exp;
- st.cpu_experts += nslots;
-#if defined(STRATA_METAL)
- if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);
-#endif
- if (nrows > 0) {
- gather_rows(d.x, ga, nrows, D, d.xe, s);
- dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);
- { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }
- dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }
- x3::MvJob j[2 * kWin * kPicks];
- int nj = 0;
- for (int i = 0; i < nvex; ++i) {
- const auto& vm = l.vm[(size_t) vex[i]];
- j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
- j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
- }
- if (!mv_n(d, j, nj, err)) return false;
- dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);
- dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);
- swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);
- dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);
- nj = 0;
- for (int i = 0; i < nvex; ++i)
- j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,
- vn[i], g.ff, D);
- if (!mv_n(d, j, nj, err)) return false;
- dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);
- }
- const double ph3 = phases ? now_ms() : 0.0;
- if (!ents.empty()) {
- const double tc = now_ms();
-#if defined(STRATA_METAL)
- ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
- ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)
-#endif
- static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
- if (!skip_cpu) cpu_experts(l, h_x, ents);
- else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)
-#if defined(STRATA_METAL)
- ds4_busy_end();
-#endif
- std::fill(h_part, h_part + (size_t) T * D, 0.0f);
- for (int sl = 0; sl < nslots; ++sl) {
- float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
- const float wv = cpu_rows[(size_t) sl].second;
- const float* y = c_yd.data() + (size_t) sl * D;
- for (int e = 0; e < D; ++e) o[e] += wv * y[e];
- }
- { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;
- ms += now_ms() - tc; ++calls; exps += (long) ents.size();
- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }
- (void) 0; (void) ms; }
- st.cpu_ms += now_ms() - tc;
- dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);
- CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));
- }
- combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);
- dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);
- if (std::getenv("DS4_DBG_MOE")) {
- float hp = 0.0f;
- for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));
- std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);
- dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);
- dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);
- }
- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
- std::swap(d.h, d.h2);
- dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);
- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)
- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
- if (phases && T == 1) {
- const double ph4 = now_ms();
- ph[0] += ph2 - ph0; ph[1] += ph3 - ph2; ph[2] += ph4 - ph3; if (ph_prev > 0 && ph0 - ph_prev < 5.0) ph[3] += ph0 - ph_prev; ph_prev = ph4; ++ph_n;
- static bool reg = false;
- if (!reg) { reg = true; std::atexit([] { std::fprintf(stderr, "phases per layer call (%ld): route enqueue + shared expert + wait %.3f ms, routing analysis + GPU experts enqueue %.3f ms, CPU lane + combine + hc_post %.3f ms, attention enqueue %.3f ms%s", ph_n, ph[0] / ph_n, ph[1] / ph_n, ph[2] / ph_n, ph[3] / ph_n, "\n"); }); }
- }
- // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
- return true;
- }
-
- int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- strata::dsv41k::SamplerParams sp;
- sp.temperature = temperature;
- sp.greedy = temperature <= 0.0f;
- sp.top_p = top_p;
- sp.top_k = top_k;
- sp.min_p = min_p;
- sp.seed = seed;
- sp.counter = counter;
- strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
- sp, d.tok, d.st);
- int t = -1;
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
- cudaStreamSynchronize(d.st);
- return t;
- }
-
- bool logits(std::vector<float>& out) {
- if (logits_all) {
- out = chunk_logits;
- return true;
- }
- Dev& d = devs[(size_t) head_dev];
- cudaSetDevice(d.id);
- out.resize((size_t) std::max(1, n_logits) * g.vocab);
-#if defined(STRATA_METAL)
- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
-#endif
- return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
- }
-
- void save_profile() const {
- if (o.profile.empty()) return;
- const std::string tmp = o.profile + ".tmp";
- std::FILE* f = std::fopen(tmp.c_str(), "wb");
- if (f == nullptr) return;
- const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();
- std::fclose(f);
- // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile
- // never moved on and only .tmp files were left)
- if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());
- }
-
- std::vector<std::string> info_lines() const {
- std::vector<std::string> v;
- int nh = 0;
- for (const Layer& l : L) nh += l.n_exp - l.n_vram;
- char b[512];
- // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)
- std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "
- "cpu_experts=%d ram_experts=%d engine=ds4-0.1",
- (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);
- v.push_back(b);
- if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)
- std::snprintf(b, sizeof b,
- "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "
- "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "
- "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);
- v.push_back(b);
- std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),
- pagefile_hint);
- v.push_back(b);
- }
- if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor
- std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,
- pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);
- v.push_back(b);
- }
- if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)
- std::snprintf(b, sizeof b,
- "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "
- "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",
- vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,
- (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),
- (long long) (vt.device_bytes() >> 20));
- v.push_back(b);
- } else if (!vision_err.empty()) { // (the server offers no pictures and says why)
- std::string why = vision_err;
- for (char& ch : why)
- if (ch == '\n' || ch == '\r') ch = ' ';
- v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");
- v.push_back("INFO vision=off");
- }
- return v;
- }
-};
-
-Model::Model() : p_(new Impl) {}
-Model::~Model() = default;
-bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }
-bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }
-bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
- Impl& p = *p_;
- p.pm.last = -1;
- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);
- if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
- const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
- const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
- : p.feed_windows(tokens, n, logits_rows, err);
- if (ok) p.path_done(now_ms() - t0);
- return ok;
- }
- return p.feed_windows(tokens, n, logits_rows, err);
-}
-bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
- Impl& p = *p_;
- p.pm.last = -1;
- const double t0 = now_ms();
- if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
- const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
- if (ok) p.path_done(now_ms() - t0);
- return ok;
- }
- if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;
- p.checkpoint(slot);
- const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);
- if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)
- return ok;
-}
-int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
-void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
-int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
- return p_->sample(t, tp, tk, mp, seed, counter);
-}
-bool Model::logits(std::vector<float>& out) { return p_->logits(out); }
-int Model::image_token() const { return p_->img_tok; }
-int Model::dim() const { return p_->g.dim; }
-bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {
- return p_->encode_image(rgb, width, height, span, err);
-}
-void Model::set_images(std::vector<Image> images) {
- std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });
- p_->imgs = std::move(images);
-}
-int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }
-bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,
- float mp, uint64_t seed, uint64_t counter) {
- return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);
-}
-bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,
- int& accepted, int& next, std::string& err) {
- return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);
-}
-int64_t Model::pos() const { return p_->pos; }
-int64_t Model::max_context() const { return p_->o.max_context; }
-int Model::vocab() const { return p_->g.vocab; }
-const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
-void Model::reset() { p_->reset(); }
-void Model::checkpoint(int slot) { p_->checkpoint(slot); }
-bool Model::restore(int64_t limit) { return p_->restore(limit); }
-int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
-void Model::save_profile() const { p_->save_profile(); }
-const Stats& Model::stats() const {
- Stats& st = p_->st;
- if (p_->store) { // (the store keeps its own counters: copied in when asked)
- const ExpertStore& x = *p_->store;
- st.store_hits = x.stats().hits.load();
- st.store_misses = x.stats().misses.load();
- st.store_prefetched = x.stats().prefetched.load();
- st.store_pf_used = x.stats().prefetch_used.load();
- st.store_pf_wasted = x.stats().prefetch_wasted.load();
- st.store_evictions = x.stats().evictions.load();
- st.store_bytes = x.stats().bytes_read.load();
- st.store_fails = x.stats().fails.load();
- st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
- st.store_busy_ms = x.busy_ms();
- st.store_zc_visible = (int64_t) x.visible_count();
- st.store_slots = (int64_t) x.slots_allocated();
- st.store_slots_max = (int64_t) x.slots_max();
- }
- return st;
-}
-std::string Model::describe() const { return p_->describe(); }
-std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
-
-} // namespace strata::ds4
+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
+// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).
+//
+// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):
+// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables
+// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)
+// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus
+// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the
+// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b
+// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])
+// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert
+// h = post * moe + comb^T h; pre = f_pre
+// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
+#include "ds4_model.hpp"
+#include "ds4_pick_rule.hpp"
+#include "ds4_experts_rule.hpp"
+#include "ds4_engram_io.hpp"
+
+#include "ds4_gguf.cuh"
+#include "ds4_iq.cuh"
+#include "ds4_gguf_cpu.hpp"
+#include "ds4_kernels.cuh"
+#include "ds4_os.hpp"
+#include "ds4_pool.hpp"
+#include "ds4_store.hpp"
+#include "ds4_vision.cuh"
+#include "strata/dsv41/exl3_cpu.hpp"
+#include "strata/dsv41/exl3.hpp"
+#include "strata/kernels/f16_bits.hpp"
+#include "strata/dsv41/moe_win.hpp"
+#include "strata/dsv41/sampler.hpp"
+#include "wmma_gemm.h" // strata_pf_gemm_f16_ld: the RDNA3 WMMA dense GEMM (gemm() prefers it over hipBLAS on gfx11; -Isrc/prefill)
+
+#include <cublas_v2.h>
+#include <cuda_runtime.h>
+
++#if defined(STRATA_METAL)
++extern "C" void ds4_busy_begin();
++extern "C" void ds4_busy_end();
++#include <sys/sysctl.h>
++#include <mach/mach.h>
++#endif
+#include <algorithm>
+#include <array>
+#include <condition_variable>
+#include <deque>
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <map>
+#include <mutex>
+#include <numeric>
+#include <sstream>
+#include <thread>
+#include <unordered_map>
+
+#if defined(_MSC_VER)
+#define strtok_r strtok_s
+#endif
+
+namespace strata::ds4 {
+namespace x3 = strata::dsv41k::exl3;
+namespace kc = strata::dsv41k::cpu;
+namespace gf = strata::dsv41k::glmf;
+namespace gg = strata::ds4::gguf;
+
+#define CK(call) \
+ do { \
+ cudaError_t e_ = (call); \
+ if (e_ != cudaSuccess) { \
+ err = std::string(#call) + ": " + cudaGetErrorString(e_); \
+ return false; \
+ } \
+ } while (0)
+
+namespace {
+
+// A HIP build (the share-based defaults below are gated on the devices being RDNA4, Impl::rdna4()).
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+constexpr bool kHipBuild = true;
+#else
+constexpr bool kHipBuild = false;
+#endif
+
+// an environment variable as 0 / 1, or -1 when it is not set (the rules in ds4_experts_rule.hpp take this form)
+int env_tri(const char* name) {
+ const char* v = std::getenv(name);
+ return v != nullptr ? (std::atoi(v) != 0 ? 1 : 0) : -1;
+}
+
+double now_ms() {
+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
+}
+
+size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }
+
+// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:
+// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):
+// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before
+void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {
+ const int b = whole / 128, q = b / np, r = b % np;
+ auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };
+ int at = 0;
+ for (int i = 0; i < p; ++i) at += blocks(i);
+ off = 128 * at;
+ n = 128 * blocks(p);
+}
+
+std::string read_file(const std::string& p) {
+ std::ifstream f(p);
+ std::stringstream ss;
+ ss << f.rdbuf();
+ return ss.str();
+}
+std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"
+ std::vector<int> v;
+ std::stringstream ss(s);
+ std::string part;
+ while (std::getline(ss, part, ',')) {
+ if (part.empty() || part == "\n") continue;
+ const size_t dash = part.find('-');
+ const int a = std::atoi(part.c_str());
+ const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);
+ for (int c = a; c <= b; ++c) v.push_back(c);
+ }
+ return v;
+}
+// each NUMA node's physical cores (the first logical CPU of each core), in order
+std::vector<std::vector<int>> node_cores() {
+#if defined(_WIN32)
+ static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)
+ const char* v = std::getenv("DS4_CPU_SMT");
+ return v != nullptr && std::atoi(v) != 0;
+ }();
+ std::vector<std::vector<int>> nodes = os::node_cores(smt);
+#else
+ std::vector<std::vector<int>> nodes;
+ for (int n = 0; n < 64; ++n) {
+ const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");
+ if (cl.empty()) break;
+ std::vector<int> cores;
+ for (int c : cpu_list(cl)) {
+ const std::vector<int> sib = cpu_list(
+ read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));
+ if (sib.empty() || sib[0] == c) cores.push_back(c);
+ }
+ if (!cores.empty()) nodes.push_back(cores);
+ }
+#endif
+ if (nodes.empty()) {
+ std::vector<int> all;
+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);
+ nodes.push_back(all);
+ }
+ return nodes;
+}
+// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages
+void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }
+void pin_self(int cpu) { os::pin_self(cpu); }
+// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root
+// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known
+uint64_t ram_avail() {
+#if defined(_WIN32)
+ return os::ram_avail();
+#endif
+ uint64_t avail = ~uint64_t{0};
+ {
+ std::istringstream mi(read_file("/proc/meminfo"));
+ std::string k;
+ uint64_t kb = 0;
+ while (mi >> k >> kb) {
+ if (k == "MemAvailable:") avail = kb << 10;
+ mi.ignore(64, '\n');
+ }
+ }
+ std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."
+ const size_t at = cg.find("0::");
+ if (at == std::string::npos) return avail;
+ cg = cg.substr(at + 3);
+ cg = cg.substr(0, cg.find('\n'));
+ for (;;) {
+ const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);
+ const std::string mx = read_file(dir + "/memory.max");
+ if (!mx.empty() && mx.compare(0, 3, "max") != 0) {
+ const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);
+ const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);
+ uint64_t file = 0;
+ std::istringstream st(read_file(dir + "/memory.stat"));
+ std::string k;
+ uint64_t v = 0;
+ while (st >> k >> v)
+ if (k == "file") file = v;
+ const uint64_t used = cur > file ? cur - file : 0;
+ avail = std::min(avail, lim > used ? lim - used : 0);
+ }
+ if (cg.empty() || cg == "/") break;
+ const size_t sl = cg.rfind('/');
+ cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);
+ }
+ return avail;
+}
+
+float bf16_to_f32(uint16_t b) {
+ uint32_t u = (uint32_t) b << 16;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+float f16_to_f32(uint16_t h) {
+ const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;
+ uint32_t u;
+ if (e == 0) {
+ if (m == 0) u = s;
+ else { // subnormal
+ int ee = -1;
+ uint32_t mm = m;
+ do {
+ ++ee;
+ mm <<= 1;
+ } while ((mm & 0x400) == 0);
+ u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;
+ }
+ } else if (e == 31) u = s | 0x7f800000u | m << 13;
+ else u = s | (e - 15 + 127) << 23 | m << 13;
+ float f;
+ std::memcpy(&f, &u, 4);
+ return f;
+}
+
+// ------------------------------------------------------------------ the pack
+struct Tensor {
+ std::string dtype;
+ int file = -1;
+ uint64_t off = 0, bytes = 0;
+ std::vector<int64_t> shape;
+};
+
+struct Pack {
+ std::string dir, model_dir;
+ std::unordered_map<std::string, std::string> kv;
+ std::vector<std::string> files;
+ std::vector<os::fd_t> fds;
+ std::unordered_map<std::string, Tensor> t;
+ int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;
+ std::vector<int32_t> eg_map;
+ std::vector<int64_t> eg_mult, eg_primes, eg_off;
+ // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the
+ // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)
+ bool gguf = false;
+
+ std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)
+ std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)
+ std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)
+
+ ~Pack() {
+ for (auto& m : maps) os::unmap(m.first, m.second);
+ for (os::fd_t fd : fds) os::close_fd(fd);
+ for (os::fd_t fd : dfds) os::close_fd(fd);
+ for (os::fd_t fd : pfds) os::close_fd(fd);
+ }
+ // file i mapped whole, read-only (once; not thread-safe: called while loading)
+ const uint8_t* map(int file) {
+ if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});
+ auto& m = maps[(size_t) file];
+ if (m.first == nullptr) {
+ const uint64_t size = os::file_size(fds[(size_t) file]);
+ void* p = os::map_ro(fds[(size_t) file], size);
+ if (p == nullptr) return nullptr;
+ m = {(uint8_t*) p, (size_t) size};
+ }
+ return m.first;
+ }
+
+ bool load(const std::string& d, std::string& err) {
+ dir = d;
+ {
+ std::ifstream f(d + "/ds4.txt");
+ if (!f) {
+ err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";
+ return false;
+ }
+ std::string line;
+ while (std::getline(f, line)) {
+ const size_t sp = line.find(' ');
+ if (sp == std::string::npos) continue;
+ kv[line.substr(0, sp)] = line.substr(sp + 1);
+ }
+ }
+ model_dir = str("model_dir");
+ std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");
+ if (f == nullptr) {
+ err = d + "/tensors.txt: not found";
+ return false;
+ }
+ std::vector<char> buf(1 << 16);
+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {
+ char* s = buf.data();
+ if (s[0] == 'f') { // file <i> <name>
+ char name[4096];
+ int i = 0;
+ if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {
+ if ((int) files.size() <= i) files.resize((size_t) i + 1);
+ files[(size_t) i] = name;
+ }
+ continue;
+ }
+ if (s[0] != 't') continue;
+ char* save = nullptr;
+ ::strtok_r(s, " \n", &save);
+ const char* name = ::strtok_r(nullptr, " \n", &save);
+ const char* dt = ::strtok_r(nullptr, " \n", &save);
+ const char* fi = ::strtok_r(nullptr, " \n", &save);
+ const char* off = ::strtok_r(nullptr, " \n", &save);
+ const char* nb = ::strtok_r(nullptr, " \n", &save);
+ if (name == nullptr || nb == nullptr) continue;
+ Tensor x;
+ x.dtype = dt;
+ x.file = std::atoi(fi);
+ x.off = std::strtoull(off, nullptr, 10);
+ x.bytes = std::strtoull(nb, nullptr, 10);
+ while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));
+ t.emplace(name, std::move(x));
+ }
+ std::fclose(f);
+ for (const std::string& fn : files) {
+ const std::string p = model_dir + "/" + fn;
+ const os::fd_t fd = os::open_ro(p);
+ if (fd == os::kBadFd) {
+ err = p + ": cannot open";
+ return false;
+ }
+ fds.push_back(fd);
+ }
+ gguf = str("format") == "gguf";
+ if (gguf)
+ for (size_t i = 0; i < fds.size(); ++i) {
+ if (map((int) i) == nullptr) {
+ err = model_dir + "/" + files[i] + ": cannot map";
+ return false;
+ }
+ dfds.push_back(os::open_direct(model_dir + "/" + files[i]));
+ pfds.push_back(os::open_par(model_dir + "/" + files[i]));
+ }
+ // the Engram tables
+ std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");
+ if (e == nullptr) {
+ err = d + "/engram.bin: not found";
+ return false;
+ }
+ char magic[4];
+ int32_t hdr[6];
+ bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&
+ std::fread(hdr, 4, 6, e) == 6;
+ if (ok) {
+ eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];
+ const int cols = (eg_ngram - 1) * eg_heads;
+ eg_map.resize((size_t) eg_vocab);
+ eg_mult.resize((size_t) eg_layers * eg_ngram);
+ eg_primes.resize((size_t) eg_layers * cols);
+ eg_off.resize((size_t) eg_layers * cols);
+ ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&
+ std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&
+ std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&
+ std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();
+ }
+ std::fclose(e);
+ if (!ok) {
+ err = d + "/engram.bin: unreadable";
+ return false;
+ }
+ return true;
+ }
+
+ std::string str(const std::string& k, const std::string& def = "") const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : it->second;
+ }
+ double num(const std::string& k, double def) const {
+ auto it = kv.find(k);
+ return it == kv.end() ? def : std::atof(it->second.c_str());
+ }
+ std::vector<int> list(const std::string& k) const {
+ std::vector<int> v;
+ std::stringstream ss(str(k));
+ std::string x;
+ while (std::getline(ss, x, ','))
+ if (!x.empty()) v.push_back(std::atoi(x.c_str()));
+ return v;
+ }
+ const Tensor* get(const std::string& n) const {
+ auto it = t.find(n);
+ return it == t.end() ? nullptr : &it->second;
+ }
+ bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)
+ if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)
+ return os::pread_par(pfds[(size_t) file], dst, n, off);
+ return os::pread_full(fds[(size_t) file], dst, n, off);
+ }
+ bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }
+ const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)
+};
+
+// ------------------------------------------------------------------ geometry
+struct Geo {
+ int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,
+ k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,
+ cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;
+ float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;
+ double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;
+ std::vector<int> ratios, kv_src, ix_src, eg_layers;
+ // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks
+ int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;
+ std::vector<int> d_targets;
+
+ void from(const Pack& p) {
+ vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);
+ nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);
+ hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);
+ q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);
+ o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);
+ k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);
+ win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);
+ sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);
+ ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);
+ cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);
+ cand_blk = (int) p.num("candidate_block_size", 0);
+ orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);
+ eos = (int) p.num("eos_token_id", 1);
+ eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);
+ route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);
+ rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);
+ factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);
+ beta_slow = p.num("rope_scaling.beta_slow", 1);
+ ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");
+ ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");
+ d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);
+ d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);
+ d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");
+ // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)
+ n_mtp = d_block > 0 && (int) ratios.size() > nl &&
+ (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)
+ ? (int) ratios.size() - nl : 0;
+ }
+};
+
+// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them
+std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {
+ std::vector<float> f((size_t) dim / 2);
+ for (int i = 0; i < dim / 2; ++i) {
+ const float e = (float) (2 * i) / (float) dim;
+ f[(size_t) i] = 1.0f / std::pow((float) base, e);
+ }
+ if (original > 0) {
+ auto corr = [&](double rot) {
+ return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)
+ };
+ const double low = std::max(std::floor(corr(beta_fast)), 0.0);
+ const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);
+ const float den = (float) std::max(high - low, 1e-3);
+ for (int i = 0; i < dim / 2; ++i) {
+ float ramp = ((float) i - (float) low) / den;
+ ramp = std::min(1.0f, std::max(0.0f, ramp));
+ const float smooth = 1.0f - ramp;
+ f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;
+ }
+ }
+ return f;
+}
+
+// ------------------------------------------------------------------ devices, layers
+struct Dev {
+ int id = 0;
+ cudaStream_t st = nullptr;
- cudaEvent_t ev = nullptr;
++ cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)
++ int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)
++ float* rw_a = nullptr;
+ size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;
+ std::vector<void*> allocs;
+ // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)
+ float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,
+ *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,
+ *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,
+ *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,
+ *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,
+ *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,
+ *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,
+ *ed = nullptr;
+ float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;
+ unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)
+ uint8_t* eg_rows = nullptr;
+ uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows
+ AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)
+ // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the
+ // driver's staging, ~1.7 ms a copy): h | pre | topk | cand
+ uint8_t* bounce = nullptr;
+ cudaEvent_t ev_out = nullptr;
+ float* tap = nullptr; // a window's stream means before a DSpark target layer
+ // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's
+ // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window
+ // MoE's grouping scratch
+ int grp = 0, mi = 0;
+ unsigned ar_seq = 0;
+ unsigned* xcnt = nullptr;
+ unsigned* hccnt = nullptr; // [kWin] hc_mix_split's per-token block counters (zeroed at allocation, left zero)
+ // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;
+ // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)
+ x3::Mat head;
+ float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;
+ unsigned head_seq = 0;
+ // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the
+ // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)
+ uint16_t *mk_share = nullptr, *mk_erow = nullptr;
+ int* mk_ints = nullptr;
+ int mk_v0 = 0, mk_vn = 0;
+ uint8_t* hand = nullptr;
+ unsigned* hand_flag = nullptr;
+ unsigned long long* grp_ptr = nullptr;
+ int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;
+};
+
+// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of
+// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS
+// DS4_PRED_DUMP: a fixed +-1 random projection of the router input (D -> 256), one block a token
+__global__ void pred_proj_k(const float* x, int D, float* out) {
+ const int t = blockIdx.x, j = threadIdx.x; // 256 threads
+ const float* xr = x + (size_t) t * D;
+ float a = 0.0f;
+ for (int d = 0; d < D; ++d) {
+ unsigned h = (unsigned) d * 2654435761u ^ ((unsigned) j + 1u) * 40503u;
+ h ^= h >> 15; h *= 2246822519u; h ^= h >> 13; h *= 3266489917u; h ^= h >> 16;
+ a += (h & 1u) ? xr[d] : -xr[d];
+ }
+ out[(size_t) t * 256 + j] = a * rsqrtf((float) D);
+}
+
+struct Chunk {
+ int C = 0, A = 0, AI = 0;
+ float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,
+ *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,
+ *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,
+ *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,
+ *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;
+ uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,
+ *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;
+ size_t w16_elems = 0;
+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;
+ uint8_t* eg_rows = nullptr;
+ int* bounds_pin = nullptr; // pinned: the sort's bounds, read back
+ // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,
+ // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)
+ static constexpr int kParts = 4;
+ uint8_t* ring = nullptr;
+ size_t ring_stride = 0;
+ int ring_n = 0;
+ cudaStream_t cs = nullptr;
+ cudaEvent_t filled[kParts] = {}, freed[kParts] = {};
+ // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of
+ // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,
+ // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s
+ uint8_t* stage = nullptr;
+ size_t stage_stride = 0;
+ std::unique_ptr<Pool> readers;
+ cublasHandle_t blas = nullptr;
+ // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each
+ // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)
+ uint8_t* hand[2] = {nullptr, nullptr};
+ unsigned* hand_flag[2] = {nullptr, nullptr};
+ unsigned* hand_ack = nullptr;
+ unsigned arb_seq = 0; // the group's chunk all-reduces
+ // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)
+ uint16_t* emb_host[2] = {nullptr, nullptr};
+ uint8_t* eg_host[2] = {nullptr, nullptr};
+ cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};
+ double stream_bytes = 0; // the experts streamed in, in all
+ // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a
+ // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the
+ // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once
+ // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io
+ // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits
+ // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot
+ // i % ring_n; the compute takes the items as they land, in runs of what has.
+ struct Item {
+ int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)
+ int mi = 0; // the device's member index in the layer's group
++ bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA
+ const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read
+ const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)
+ uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there
+ int64_t after = -1; // (layer-major) the items up to this one read first
+ std::atomic<int> left{0}; // parts still being read
+ std::atomic<bool> bad{false};
+ };
+ bool flow = false;
++ bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring
+ int dev = -1; // the device (index into devs) this chunk state belongs to
+ std::unique_ptr<IoQueue> io;
+ std::deque<Item> items; // items[0] is item `base`; the indices below are absolute
+ size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;
+ uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)
+ int S = 0; // staging slots, stage_stride apart
+ std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)
+ std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use
+ int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where
+ size_t spec_begin = 0;
+ std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes
+ int fl_yoco = -1, fl_cut_k = 0, fl_cut_r = 0; // Run::yoco / cut_k / cut_r: what a layer's chunk streams for
+ int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count
+ int *bc_pin = nullptr, *bc_dev = nullptr;
+ uint64_t n_batch = 0;
+ double wait_ms = 0; // the compute waiting for a read
+ // (DS4_PROF: where copy_ahead's time goes - the host staging copy, the DMA issues, the waits for a slot)
+ double dbg_stage_ms = 0, dbg_dma_ms = 0, dbg_evt_ms = 0;
+ uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;
+ uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started
+ double io_busy0 = 0; // io's busy time when the run started (DS4_PROF)
+ // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through
+ // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in
+ // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for
+ // its later chunks, so each is read once a run
+ bool lm = false, lm_pinned = false;
+ uint8_t* lm_state = nullptr;
+ size_t lm_state_bytes = 0;
+ uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side
+ size_t lc_bytes = 0;
+ int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all
+ size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read
+ std::vector<int> lc_of; // expert -> its slot (-1: none)
+ std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it
+ uint64_t it_lc = 0; // items taken from the layer cache
+};
+
+struct Layer {
+ int il = 0, dev = 0, ratio = 0;
+ // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the
+ // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert
+ // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)
+ int grp = 0, nm = 1, mi = 0;
+ std::vector<Layer> mates;
+ x3::ExpertLayout hlay;
+ size_t hblob = 0;
+ int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)
+ // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more
+ // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their
+ // attention adds nothing to the sum
+ int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;
+ // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's
+ // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member
+ int* vs_pin = nullptr;
+ int* vs_dev = nullptr;
+ std::vector<int> host_list;
+ std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."
+ int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)
+ bool mtp = false; // a DSpark block (index stage = il - n_layers)
+ bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;
+ int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)
+ int ik_src = -1; // the kv source whose index keys its indexer reads
+ int eg = -1; // engram layer index, or -1
+ x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;
+ float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,
+ *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,
+ *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,
+ *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)
+ uint16_t* i_wproj = nullptr;
+ float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)
+ uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)
+ float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)
+ // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row
+ uint8_t* snap_win = nullptr;
+ float *snap_kv = nullptr, *snap_sc = nullptr;
+ // routed experts
+ x3::ExpertLayout lay;
+ size_t blob = 0; // bytes of one expert's blob
+ int n_vram = 0;
+ bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's
+ uint8_t* vram_base = nullptr;
+ std::vector<int> slot; // expert -> VRAM slot or -1
+ std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
+ std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)
+ // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),
+ // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)
+ struct Half {
+ size_t trellis[3], suh[3], svh[3], bytes;
+ int k[3], n[3];
+ } hl;
+ std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes
+ // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it
+ // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)
+ int n_slots = 0, spare_vram = -1;
+ std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};
+ std::vector<float> heat;
+ std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked
+ bool swap_pending = false;
+ unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes
+ // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and
+ // the window's raw rows (kv, then score) to replay the accepted ones
+ uint8_t* ring_bak = nullptr;
+ float *c_snap = nullptr, *c_rows = nullptr;
+ unsigned long long* vram_tab = nullptr; // device [n_exp]
+ std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe
+ unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it
+ std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob
+ std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)
+ std::vector<uint64_t> region_bytes;
+ std::vector<int> region_file;
+ size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region
+ size_t piece_bytes[3][3];
+ size_t dense_bytes = 0;
+ // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +
+ // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane
+ int mx_file[3] = {-1, -1, -1};
+ uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};
+};
+
+struct Src {
+ int layer = 0, ratio = 1;
+ int64_t cap = 0;
+ // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)
+ std::vector<uint8_t*> ckv, ik;
+ uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices
+ cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;
+ // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows
+ // (ckv rows, then ik rows at kWin * hd) and its flag
+ bool remote = false;
+ uint8_t* x_stage = nullptr;
+ unsigned* x_flag = nullptr;
+ // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements
+ // [slot][device] (32 apart); readers: the devices outside the source's group that read it
+ uint8_t* x_stage1 = nullptr;
+ unsigned* x_flag1 = nullptr;
+ unsigned* x_ack = nullptr;
+ std::vector<int> readers;
+ uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }
+ unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }
+};
+
+// a host thread that enqueues one device's work (threads mode)
+struct Worker {
+ std::thread th;
+ std::mutex mu;
+ std::condition_variable cv;
+ std::function<void()> job;
+ bool busy = false, quit = false;
+ void start() {
+ th = std::thread([this] {
+ for (;;) {
+ std::function<void()> j;
+ {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return quit || job != nullptr; });
+ if (quit) return;
+ j = std::move(job);
+ job = nullptr;
+ }
+ j();
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ busy = false;
+ }
+ cv.notify_all();
+ }
+ });
+ }
+ void submit(std::function<void()> f) {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ job = std::move(f);
+ busy = true;
+ }
+ cv.notify_all();
+ }
+ void wait() {
+ std::unique_lock<std::mutex> lk(mu);
+ cv.wait(lk, [&] { return !busy; });
+ }
+ bool idle() {
+ std::lock_guard<std::mutex> lk(mu);
+ return !busy;
+ }
+ void stop() {
+ {
+ std::lock_guard<std::mutex> lk(mu);
+ quit = true;
+ }
+ cv.notify_all();
+ if (th.joinable()) th.join();
+ }
+};
+
+struct Group {
+ std::vector<int> mem; // device indices, member order (member 0 the lead)
+ ArGroup ar, arb; // a window's all-reduces, a chunk's
+ // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the
+ // lead's acknowledgements (host-mapped)
+ float* lg_stage = nullptr;
+ unsigned *lg_flag = nullptr, *lg_ack = nullptr;
+};
+
+constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers
+
+// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the
+// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate
+// blocks as each layer sees them, the window's hand-off seq
+struct WinPlan {
+ bool par = false, async = false;
+ unsigned win = 0;
+ int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2
+ int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)
+ int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)
+ unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)
+ unsigned seq[kMaxL] = {};
+ int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};
+};
+
+} // namespace
+
+// ------------------------------------------------------------------ the model
+struct Model::Impl {
+ Options o;
+ Pack pk;
+ Geo g;
+ std::vector<Dev> devs;
+ std::vector<Layer> L;
+ std::vector<Src> srcs;
+ std::vector<Group> groups;
+ std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device
+ bool par = false; // threads mode (the windows: decode)
+ mutable int all_gfx12 = -1; // rdna4(): -1 not asked yet
+ // Every GPU the engine uses is a gfx1200 / gfx1201 (RDNA4: R9700, RX 9000) in a HIP build. The DeepSeek engine's share-based defaults (the small
+ // chunk for multi-card tp runs, the whole-chunk slab, the small staging ring, the pinned copy at share >= 0.5) and the drafter's cumulative cut are
+ // the 4 x R9700 measurements, so only RDNA4 takes them until another AMD part is tested (Strix Halo next).
+ bool rdna4() const {
+ if (all_gfx12 < 0) {
+ bool yes = false;
+#if defined(STRATA_USE_HIP)
+ yes = !devs.empty();
+ for (const auto& d : devs) {
+ cudaDeviceProp prop{};
+ yes = yes && cudaGetDeviceProperties(&prop, d.id) == cudaSuccess &&
+ (std::strncmp(prop.gcnArchName, "gfx1200", 7) == 0 || std::strncmp(prop.gcnArchName, "gfx1201", 7) == 0);
+ }
+#endif
+ all_gfx12 = yes ? 1 : 0;
+ }
+ return all_gfx12 == 1;
+ }
+ bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked
+ unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)
+ // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows
+ int chunk = 0, chunk_min = 512;
+ static constexpr bool kChunkedPromptDefault = true; // the layer-major prompt is the default (gate: KL 0.1685 vs 0.1686, 8K prompt 6x, decode within 2% - ds4 notes-dsv41-r97)
+ static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)
+ // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk
+ // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers
+ // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token
+ static constexpr int kMaxChunk = 4 * kVlRows;
+ std::vector<std::unique_ptr<Chunk>> chunks; // per device
+ size_t eg_cap = kWin; // tokens h_eg holds
+ int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)
+ static constexpr int kYocoMin = 1024; // a long prompt's skip (forward_chunks): the fewest rows worth skipping
+ bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)
+ std::vector<float> chunk_logits;
+ std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)
+ // a Stop while a run of chunks goes through the layers (Model::set_cancel, asked while forward_chunks waits): one
+ // GPU stops after the layer (or the layer's chunk) it is on (halt); several stop at a chunk every one of them
+ // reaches - none starts chunk halt_k or later, halt_k one past the latest any started - so no device waits on a
+ // hand-off that never comes (Project Maya v1.1.20)
+ std::function<bool()> cancel;
+ std::atomic<bool> halt{false};
+ std::mutex halt_mu;
+ int halt_k = INT_MAX, halt_seen = -1;
+ bool halted = false; // the last run ended early on a Stop
+ static constexpr const char* kHalted = "stopped";
+ bool chunk_go(int k) { // chunk k may start on this device (several GPUs)
+ std::lock_guard<std::mutex> lk(halt_mu);
+ if (k >= halt_k) return false;
+ halt_seen = std::max(halt_seen, k);
+ return true;
+ }
+ bool halt_now() const { return devs.size() == 1 && halt.load(std::memory_order_acquire); }
+ // a run's early end on device di: the expert reads still in flight land first (they count down items the next run
+ // clears), then its streams are idle
+ bool halt_end(int di, std::string& err) {
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(devs[(size_t) di].id);
+ if (c.flow)
+ for (size_t i = c.base; i < c.n_read; ++i)
+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ if (c.cs != nullptr) cudaStreamSynchronize(c.cs);
+ cudaStreamSynchronize(devs[(size_t) di].st);
+ err = kHalted;
+ return false;
+ }
+ std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer
+ std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded
+ // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this
+ // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took
+ struct PathModel {
+ bool on = false;
+ double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)
+ // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and
+ // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)
+ double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};
+ int64_t kn[2][4] = {};
+ double win_s = 0; // windows: seconds a window took, so far
+ int64_t win_n = 0;
+ int last_cls = 0, last_nw = 0; // the last pick's size class, its windows
+ double m = 1; // the picks' spread, against the profile's
+ int64_t n_m = 0;
+ int last = -1; // the last pick: 1 the flow, 0 windows
+ double last_pred = 0; // its prediction (s, before its k)
+ } pm;
+ static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)
+ static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)
+ std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through
+ WinPlan wplan;
+ int head_dev = 0;
+ x3::Mat head;
+ float* norm_w = nullptr; // on the head device
+ // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the
+ // confidence head (on the head device: the drafts' logits come from the trunk's head)
+ x3::Mat d_main_proj;
+ float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;
+ uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]
+ uint8_t* d_mk_head_q8 = nullptr; // the markov head as Q8_0 blocks (the same bias, half the bytes a step)
+ // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its
+ // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole
+ // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq
+ bool mk_split = false;
+ uint16_t* h_mk_embed = nullptr;
+ unsigned* mk_slots = nullptr;
+ unsigned mk_seq = 0;
+ int d_dev = 0; // the drafter's first block's device
+ bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)
+ float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps
+ cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it
+ float* d_confv = nullptr; // on the head device
+ int* d_ints = nullptr; // on the head device: drafts in, picks out
+ float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;
+ // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for
+ // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()
+ // checks it is its own)
+ strata::dsv41k::SpecList* d_specq = nullptr;
+ std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);
+ bool drafts_q = false;
+ float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;
+ int dq_top_k = 0;
+ uint64_t dq_seed = 0, dq_counter = 0;
+ // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the
+ // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5
+ // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the
+ // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs
+ // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)
+ bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;
+ float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;
+ int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;
+ std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;
+ float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)
+ uint16_t* h_dump_ring = nullptr;
+
+ uint16_t* embed_pinned = nullptr;
+ std::vector<float> freqs_win, freqs_comp;
+ std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores
+ std::vector<std::vector<int>> cores; // each node's physical cores
+ int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)
+ // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps
+ // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)
+ bool file_experts = false;
+ uint64_t warm_bytes = 0;
+ static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)
+ uint8_t* file_part(const Layer& l, int e) const {
+ return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];
+ }
+ // host side of the CPU lane, a window's rows
+ float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;
+ int* h_idx = nullptr;
++ int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks
+ uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales
+ // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one
+ // an Engram layer: asked for when the window's tokens are known, collected right before that layer)
+ EgReader eg_io;
+ EgRowCache eg_cache;
+ std::mutex eg_cache_mu;
+ std::vector<std::unique_ptr<EgBatch>> eg_win;
+ // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1
+ // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8
+ // codes with their E8M0 scales in a tensor of their own (see eg_issue)
+ std::vector<uint8_t> eg_fmt;
+ std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};
+ uint16_t* h_emb = nullptr; // the window's embedding rows
+ std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;
+ // the state
+ int64_t pos = 0;
+ std::vector<int32_t> toks, ctoks;
+ struct Check { // the state at a position: every window ring, the compressors' partial groups
+ int64_t pos = -1;
+ std::vector<std::vector<uint8_t>> win;
+ std::vector<std::vector<float>> comp;
+ };
+ static constexpr int kChecks = 4;
+ Check ck[kChecks]; // the end of the last prompt, the position before it, the anchor (the position
+ // before the end of the first prompt of the run of prompts that each continued
+ // the last in place: a chat template that drops earlier turns' reasoning at a new
+ // user message re-renders the history from about there) and the prefix (where the
+ // server said the prompt's shared start - system prompt and tools - ends: another
+ // agent behind the same start goes back to it)
+ bool slot_held = false; // slot_save took this state and nothing has changed it since
+ std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits
+ Stats st;
+ int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query
+ // (plan_window's: the windows' plans are made in order)
+ int n_logits = 0; // the last forward's logit rows
+ // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread
+ // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum
+ Req* reqs = nullptr;
+ Resp* resps = nullptr;
+ unsigned seq_issued = 0;
+ std::atomic<int> layer_of[kRing];
+ std::thread service;
+ std::atomic<bool> quit{false};
+ std::thread store_bg; // the RAM store's fill beyond the first 48 GB, read after the model is ready (preload_store)
++#if defined(STRATA_METAL)
++ // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the
++ // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)
++ bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;
++#else
+ bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
++#endif
+ // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
+ // before the wait for the CPU lane - the same bits, no faster)
+ bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
+ int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this
+ // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same
+ // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files
+ // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next
+ // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the
+ // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's
+ // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).
+ int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;
+ int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;
+ // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)
+ bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;
+ std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;
+ // 8 asked for (prefetch)
+ std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)
+ std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for
+ std::thread pf_th;
+ std::mutex pf_mu;
+ std::condition_variable pf_cv;
+ std::vector<std::pair<const void*, size_t>> pf_q;
+ // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster
+ // on one V100 - more for a PC to try: each its staging buffers and
+ // copy streams), switched between windows
+ struct Swap {
+ int layer = 0, in = -1, out = -1;
+ int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM
+ };
+ static constexpr int kMaxSwaps = 8;
+ // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay
+ // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)
+ static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }
+ const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),
+ c_decay = env_f("DS4_CACHE_DECAY", 0.9f);
+ const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));
+ // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the
+ // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the
+ // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the
+ // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in
+ // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4
+ // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.
+ struct RamSwap {
+ int layer = 0, in = -1, out = -1, slot = -1;
+ };
+ std::vector<uint8_t*> ram_slots, ram_chunks;
+ std::vector<size_t> ram_chunk_bytes;
+ bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)
+ size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers
+ std::vector<int> ram_free;
+ std::deque<RamSwap> ram_queue;
+ std::vector<RamSwap> ram_ready;
+ std::condition_variable ram_cv;
+ std::thread ram_th;
+ int ram_inflight = 0, ram_held = 0;
+ bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;
+ // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a
+ // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route
+ // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every
+ // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the
+ // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left
+ // out still counts as used (the expert cache, the profile).
+ const float disk_skip_w = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");
+ return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;
+ }();
+ const int disk_skip_from = [] {
+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");
+ return v != nullptr ? std::max(0, std::atoi(v)) : 0;
+ }();
+ // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)
+ std::unique_ptr<ExpertStore> store;
+ uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token
+ size_t zc_stride = 0;
+ bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves
+ bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r)
+ if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;
+ return true;
+ }
+ return !file_experts || os::in_ram(file_part(l, e), l.blob);
+ }
+ // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads
+ // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between
+ // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of
+ // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'
+ // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes
+ // the RAM place of the one it takes in)
+ // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private
+ // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever
+ // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the
+ // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds
+ // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the
+ // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold
+ // them (the log's WARNING, a NOTE and INFO for the server)
+ double commit_kept = 0, pagefile_hint = 0;
+ double commit_margin() const {
+ const os::Commit c = os::commit();
+ return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);
+ }
+ // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)
+ double commit_room() const {
+ const os::Commit c = os::commit();
+ double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;
+ if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);
+ return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());
+ }
+ // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)
+ bool commit_charged(const Dev& d) const {
+ if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;
+#if defined(_WIN32)
+ int v = 0;
+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return false;
+ }
+ return v != 0;
+#else
+ (void) d;
+ return false;
+#endif
+ }
+ bool copy_mode = false;
+ std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts
+ std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes
+ std::vector<int> copy_spare; // layer -> its spare slot (-1: none)
+ std::mutex sw_mu;
+ std::condition_variable sw_cv;
+ std::deque<Swap> sw_queue;
+ std::vector<Swap> sw_ready;
+ struct SwapBufs {
+ uint8_t *a = nullptr, *b = nullptr, *h = nullptr;
+ std::vector<cudaStream_t> st; // a copy stream a device
+ };
+ std::vector<SwapBufs> sw_bufs;
+ std::vector<std::thread> cache_ths;
+ int sw_inflight = 0;
+ int64_t decay_tokens = 0;
+ bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;
+ std::vector<float> host_logits;
+ std::vector<void*> host_allocs;
+ std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;
+ // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or
+ // chunk path (tests: where the two part)
+ std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;
++ // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records
++ // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)
++ std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;
++ std::vector<int> dump_s_layers = [] {
++ std::vector<int> v;
++ const char* e = std::getenv("DS4_DUMP_S_LAYERS");
++ std::string str = e ? e : "0,1,2,3";
++ size_t a = 0;
++ while (a < str.size()) {
++ size_t b = str.find(',', a);
++ if (b == std::string::npos) b = str.size();
++ if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));
++ a = b + 1;
++ }
++ return v;
++ }();
+ // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),
+ // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the
+ // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias
+ // [n_exp] (F32)
+ std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;
+ // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,
+ // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]
+ // (F16) and y rows [rows dim] (F32)
+ int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;
+ std::FILE* dump_y = [] {
+ const char* v = std::getenv("DS4_DUMP_Y");
+ const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;
+ return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;
+ }();
+ std::mutex dump_mu;
+ // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its
+ // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU
+ // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it
+ bool prof_on = std::getenv("DS4_PROF") != nullptr;
+ // DS4_PRED_DUMP=file (predictor data, the chunked prompt path): per layer and chunk u8 layer, u16 T, T x 384 fp16 router
+ // logits, T x 256 fp16 projection of the router input. DS4_XIN_DUMP=file (expert similarity inputs): u8 layer, u16 T,
+ // T x D fp32 router inputs, the first DS4_XIN_N (1024) tokens of each layer.
+ std::FILE* pred_f = std::getenv("DS4_PRED_DUMP") != nullptr ? std::fopen(std::getenv("DS4_PRED_DUMP"), "wb") : nullptr;
+ std::FILE* xin_f = std::getenv("DS4_XIN_DUMP") != nullptr ? std::fopen(std::getenv("DS4_XIN_DUMP"), "wb") : nullptr;
+ int xin_n = std::getenv("DS4_XIN_N") ? std::atoi(std::getenv("DS4_XIN_N")) : 1024;
+ std::array<int, 64> xin_got{};
+ std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;
+ struct ProfMark {
+ const Dev* d;
+ cudaEvent_t ev;
+ int stage;
+ };
+ std::mutex prof_mu;
+ std::vector<ProfMark> prof_marks;
+ std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on
+ bool prof_window = false; // (decode windows only: T == 1)
+ // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared
+ // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run
+ bool prof_chunk = false;
+ // DS4_PROF stages of the chunked prompt (finer ones than the decode's 0..5): 0 between layers, 1 attention rest (wq_b, wo,
+ // hc_post), 2 router-side tail (shared expert), 3 experts (+ waits), 4 Engram, 5 attn mHC (mix, split, norm), 6 attention
+ // projections (wq_a, wkv, compressor), 7 compressor + indexer, 8 q_b + rope, 9 sparse attention, 10 ffn mHC + router + sort
+ double cprof_ms[16] = {};
+ // DS4_PROF: the experts' kernels alone, by type (events around the gate/up and down calls of run_set)
+ struct EkMark {
+ cudaEvent_t e0, e1, e2;
+ int tg, td, nrows, nexp, maxrows, dev, ff;
+ };
+ std::vector<EkMark> ek_marks;
+ std::map<int, std::vector<cudaEvent_t>> ek_pool; // by device
+ cudaEvent_t ek_ev() {
+ int cur = 0;
+ cudaGetDevice(&cur);
+ std::vector<cudaEvent_t>& pl = ek_pool[cur];
+ if (!pl.empty()) {
+ cudaEvent_t e = pl.back();
+ pl.pop_back();
+ return e;
+ }
+ cudaEvent_t e = nullptr;
+ cudaEventCreate(&e);
+ return e;
+ }
+ void prof_mark(const Dev& d, int stage) {
++ if (std::getenv("DS4_TLOG")) { // the host's own phase times (what the GPU meters do not see)
++ static double t_last = 0, acc[5] = {0, 0, 0, 0, 0}; static long n = 0; static bool reg = false;
++ if (!reg) {
++ reg = true;
++ std::atexit([] {
++ std::fprintf(stderr,
++ "tlog: %ld layers | host ms: attn %.0f moe_pre %.0f gate_up %.0f cpu_wait %.0f down+post %.0f\n",
++ n, acc[0], acc[1], acc[2], acc[3], acc[4]);
++ });
++ }
++ const double t = now_ms();
++ if (t_last > 0) {
++ const int b = stage == 0 ? 4 : stage - 1;
++ acc[b >= 0 && b < 5 ? b : 4] += t - t_last;
++ }
++ t_last = t;
++ if (stage == 3) ++n;
++ }
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ int cur = 0;
+ cudaGetDevice(&cur);
+ cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")
+ cudaEvent_t e = nullptr;
+ std::vector<cudaEvent_t>& pool = prof_pool[d.id];
+ if (pool.empty()) cudaEventCreate(&e);
+ else {
+ e = pool.back();
+ pool.pop_back();
+ }
+ cudaEventRecord(e, d.st);
+ cudaSetDevice(cur);
+ prof_marks.push_back({&d, e, stage});
+ }
+ void prof_collect() {
+ if (!prof_window && !prof_chunk) return;
+ std::lock_guard<std::mutex> lk(prof_mu);
+ std::vector<std::pair<const Dev*, cudaEvent_t>> last;
+ for (const ProfMark& m : prof_marks) {
+ auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });
+ if (it != last.end()) {
+ float ms = 0.0f;
+ cudaSetDevice(m.d->id);
+ if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)
+ (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;
+ it->second = m.ev;
+ } else {
+ last.push_back({m.d, m.ev});
+ }
+ }
+ for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);
+ prof_marks.clear();
+ if (!prof_chunk) ++st.prof_tokens;
+ }
+ void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {
+ std::vector<float> v((size_t) T * g.hc * g.dim);
+ cudaSetDevice(d.id);
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)
++ cudaDeviceSynchronize();
++#endif
+ cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[3] = {il, p0, T};
+ std::fwrite(hdr, 8, 3, dump_l);
+ std::fwrite(v.data(), 4, v.size(), dump_l);
+ std::fflush(dump_l);
+ }
+
++ bool dump_s_on(int il) const {
++ if (dump_s == nullptr) return false;
++ for (int x : dump_s_layers)
++ if (x == il) return true;
++ return false;
++ }
++ bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)
++ static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();
++ return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;
++ }
++ void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {
++ if (!dump_s_on(il) || !dump_s_name(name)) return;
++ std::vector<uint8_t> v(nbytes);
++ cudaSetDevice(d.id);
++#if defined(STRATA_METAL)
++ ds4_metal_flush();
++ cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)
++#endif
++ cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);
++ cudaStreamSynchronize(d.st);
++ dump_sub_host(name, il, p0, T, v.data(), nbytes);
++ }
++ void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {
++ if (!dump_s_on(il) || !dump_s_name(name)) return;
++ std::lock_guard<std::mutex> lk(dump_mu);
++ char nm[16] = {};
++ std::strncpy(nm, name, 15);
++ const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};
++ std::fwrite(nm, 1, 16, dump_s);
++ std::fwrite(hdr, 8, 4, dump_s);
++ std::fwrite(host, 1, nbytes, dump_s);
++ std::fflush(dump_s);
++ }
++
+ ~Impl() {
+ quit.store(true);
+ if (warm_bg.joinable()) warm_bg.join(); // (the warm-up uses the BLAS handles destroyed below)
+ eg_io.stop(); // (its threads write into h_eg)
+ for (auto& w : workers) w->stop();
+ for (size_t i = 0; i < chunks.size(); ++i)
+ if (chunks[i] && chunks[i]->blas) {
+ cudaSetDevice(devs[i].id);
+ cublasDestroy(chunks[i]->blas);
+ }
+ quit.store(true);
+ if (service.joinable()) service.join();
+ if (store_bg.joinable()) store_bg.join(); // (the store's background fill: it checks quit between its pieces)
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ }
+ pf_cv.notify_all();
+ if (pf_th.joinable()) pf_th.join();
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ }
+ sw_cv.notify_all();
+ ram_cv.notify_all();
+ for (std::thread& t : cache_ths)
+ if (t.joinable()) t.join();
+ if (ram_th.joinable()) ram_th.join();
+ for (size_t i = 0; i < ram_chunks.size(); ++i) {
+ if (ram_pinned) (void) cudaHostUnregister(ram_chunks[i]);
+ os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ for (void* p : d.allocs) cudaFree(p);
+ if (d.st) cudaStreamDestroy(d.st);
+ if (d.ev) cudaEventDestroy(d.ev);
+ }
+ for (void* p : host_allocs) cudaFreeHost(p);
+ for (Layer& l : L)
+ for (uint8_t* b : l.host)
+ if (b != nullptr) {
+ (void) b; // host blobs live in one mapping per layer, unmapped below
+ }
+ for (auto& m : host_maps) {
+ if (chunk > 0) cudaHostUnregister(m.first);
+ os::anon_free(m.first, m.second);
+ }
+ }
+ std::vector<std::pair<void*, size_t>> host_maps;
+
+ template <class T>
+ bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {
+ void* v = nullptr;
+ cudaSetDevice(d.id);
+ const size_t b = align256(n * sizeof(T));
+ if (cudaMalloc(&v, b) != cudaSuccess) {
+ err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st
+ // does not order against - a zeroing still queued there could wipe the weights just copied
+ if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {
+ cudaFree(v);
+ err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ if (acct) *acct += b;
+ else d.scratch += b;
+ p = (T*) v;
+ return true;
+ }
+ static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }
+ // bytes of a VRAM slot of l's experts (a member's part of each, in a group)
+ static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }
+ // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64
+ // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share
+ int head_members(int nm) const {
+ int na = nm;
+ while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;
+ return na;
+ }
+ // host-mapped memory (the same address on the host and every device), zeroed
+ template <class T>
+ bool hmap(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";
+ return false;
+ }
+ std::memset(v, 0, n * sizeof(T) + 64);
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+ template <class T>
+ bool halloc(T*& p, size_t n, std::string& err) {
+ void* v = nullptr;
+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {
+ err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";
+ return false;
+ }
+ host_allocs.push_back(v);
+ p = (T*) v;
+ return true;
+ }
+
+ // ---- weights
+ bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {
+ std::vector<uint8_t> tmp(t.bytes);
+ if (!pk.read(t, tmp.data())) {
+ err = "read failed";
+ return false;
+ }
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
+ return true;
+ }
+ // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs
+ bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);
+ std::vector<uint8_t> w((size_t) n * rb);
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";
+ return false;
+ }
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = gg::kFmtGG;
+ m.K = ggt;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = (int) gg::type_row_bytes(ggt, k);
+ return true;
+ }
+ // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
+ // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
+ // standalone [n][k] matrix
+ bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ const bool q8 = t.dtype == "Q8_0";
+ // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)
+ const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;
+ if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);
+ if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)
+ auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };
+ std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +
+ std::to_string(np) + " parts";
+ return false;
+ }
+ const size_t rb = bytes_of((size_t) k);
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = bytes_of((size_t) wd);
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r)
+ std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = q8 ? (int) bytes_of((size_t) k) : k;
+ return true;
+ }
+ // a Q8_0 tensor's values as F32 (host)
+ static std::vector<float> q8_values(const uint8_t* q, size_t n) {
+ std::vector<float> v(n);
+ for (size_t b = 0; b < n / 32; ++b) {
+ const uint8_t* p = q + b * gg::kQ8Bytes;
+ const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));
+ for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];
+ }
+ return v;
+ }
+ // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone
+ // matrix of its own (the rotations' 128-blocks stay whole)
+ // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix
+ // splits evenly, its groups only as wide as that goes)
+ bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
+ bool cols = true, int rot = 0) {
+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))
+ return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
+ const Tensor* tr = pk.get(pre + ".trellis");
+ const Tensor* su = pk.get(pre + ".suh");
+ const Tensor* sv = pk.get(pre + ".svh");
+ if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {
+ err = pre + ": not an EXL3 matrix in the pack";
+ return false;
+ }
+ if (pk.get(pre + ".mul1") == nullptr) {
+ err = pre + ": only the mul1 codebook is covered";
+ return false;
+ }
+ m.k = (int) tr->shape[0] * 16;
+ m.n = (int) tr->shape[1] * 16;
+ m.K = (int) tr->shape[2] / 16;
+ m.cb = 2;
+ m.ld = 0;
+ if (np > 1) {
+ const int whole = cols ? m.n : m.k;
+ int at = 0, w = 0;
+ if (whole % 128 != 0 || whole / 128 < np) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +
+ std::to_string(np) + " parts of 128-blocks";
+ return false;
+ }
+ split128(whole, np, part, at, w, rot);
+ std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);
+ if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));
+ std::vector<uint8_t> t1, u1, v1;
+ if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)
+ const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;
+ t1.resize((size_t) (m.k / 16) * nth * tile);
+ for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)
+ std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);
+ u1 = u0;
+ v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));
+ m.n = w;
+ } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)
+ const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;
+ t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));
+ u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));
+ v1 = v0;
+ m.k = w;
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||
+ !dalloc(d, dv, v1.size() / 2, err, acct))
+ return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);
+ }
+ uint32_t* dt = nullptr;
+ uint16_t *ds = nullptr, *dv = nullptr;
+ if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||
+ !dalloc(d, dv, sv->bytes / 2, err, acct))
+ return false;
+ if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;
+ m.trellis = dt;
+ m.suh = ds;
+ m.svh = dv;
+ if (!x3::supported(m)) {
+ err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +
+ " not covered by the EXL3 kernels";
+ return false;
+ }
+ return true;
+ }
+ // a float tensor (F32 / F16 / BF16) as F32 on the device
+ bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {
+ const Tensor* t = pk.get(name);
+ if (t == nullptr) {
+ err = name + ": not in the pack";
+ return false;
+ }
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ size_t n = 0;
+ std::vector<float> v;
+ if (t->dtype == "F32") {
+ n = t->bytes / 4;
+ v.resize(n);
+ std::memcpy(v.data(), raw.data(), t->bytes);
+ } else if (t->dtype == "F16" || t->dtype == "BF16") {
+ n = t->bytes / 2;
+ v.resize(n);
+ const uint16_t* s = (const uint16_t*) raw.data();
+ for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);
+ } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)
+ n = t->bytes / gg::kQ8Bytes * 32;
+ v = q8_values(raw.data(), n);
+ } else {
+ err = name + ": dtype " + t->dtype + " is not a float";
+ return false;
+ }
+ if (want != 0 && n != want) {
+ err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);
+ return false;
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));
+ return true;
+ }
+ bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {
+ const Tensor* t = pk.get(name);
+ if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)
+ const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();
+ std::vector<uint16_t> h(n);
+ for (size_t i = 0; i < n; ++i) {
+ float v;
+ if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);
+ else if (t->dtype == "Q8_0") v = q[i];
+ else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);
+ h[i] = strata::kernels::f16_from_f32(v);
+ }
+ if (!dalloc(d, p, n, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));
+ return true;
+ }
+ if (t == nullptr || t->dtype != "F16") {
+ err = name + ": not an F16 tensor in the pack";
+ return false;
+ }
+ if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;
+ return upload(d, *t, p, err);
+ }
+
+ // The markov bias's head as Q8_0 blocks of 32 (34 B: F16 d + 32 int8) instead of F16 values: the same bias sums with the
+ // quantisation's error, half the bytes the drafter's five markov steps read (66 MB -> 33 MB) and half the memory. The
+ // pack is untouched: the F32 / F16 / BF16 tensor is quantised as it is loaded.
+ bool mk_head_q8(Dev& d, const std::string& name, uint8_t*& p, std::string& err, size_t* acct) {
+ const Tensor* t = pk.get(name);
+ if (t == nullptr || (t->dtype != "F32" && t->dtype != "F16" && t->dtype != "BF16")) {
+ err = name + ": not an F32 / F16 / BF16 tensor";
+ return false;
+ }
+ const size_t n = t->bytes / (t->dtype == "F32" ? 4 : 2);
+ if (n == 0 || n % 32 != 0) {
+ err = name + ": not a whole number of 32-value blocks";
+ return false;
+ }
+ std::vector<uint8_t> raw(t->bytes);
+ if (!pk.read(*t, raw.data())) {
+ err = name + ": read failed";
+ return false;
+ }
+ std::vector<float> v(n);
+ for (size_t i = 0; i < n; ++i) {
+ if (t->dtype == "F32") std::memcpy(&v[i], raw.data() + i * 4, 4);
+ else v[i] = t->dtype == "F16" ? f16_to_f32(((const uint16_t*) raw.data())[i])
+ : bf16_to_f32(((const uint16_t*) raw.data())[i]);
+ }
+ const size_t nb = n / 32;
+ std::vector<uint8_t> q(nb * 34);
+ for (size_t b = 0; b < nb; ++b) {
+ const float* s = v.data() + b * 32;
+ float amax = 0.0f;
+ for (int i = 0; i < 32; ++i) amax = std::max(amax, std::fabs(s[i]));
+ const float dd = amax > 0.0f ? amax / 127.0f : 0.0f;
+ const uint16_t dh = strata::kernels::f16_from_f32(dd);
+ std::memcpy(q.data() + b * 34, &dh, 2);
+ int8_t* o = (int8_t*) (q.data() + b * 34 + 2);
+ for (int i = 0; i < 32; ++i) {
+ const float x = dd > 0.0f ? s[i] / dd : 0.0f;
+ o[i] = (int8_t) std::max(-127.0f, std::min(127.0f, std::round(x)));
+ }
+ }
+ if (!dalloc(d, p, nb * 34, err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(p, q.data(), nb * 34, cudaMemcpyHostToDevice));
+ return true;
+ }
+
+ size_t mat_bytes(const std::string& pre) const {
+ if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);
+ size_t b = 0;
+ for (const char* s : {".trellis", ".suh", ".svh"})
+ if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);
+ return b;
+ } size_t tensor_bytes(const std::string& name) const {
+ const Tensor* t = pk.get(name);
+ return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;
+ }
+
+ // ---- expert regions and blob layout
+ // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every
+ // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
+ // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
+ static int gg_type_of(const std::string& dtype) {
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})
+ if (dtype == gg::type_name(t)) return t;
+ return -1;
+ }
+ bool expert_layout_mx(Layer& l, std::string& err) {
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;
+ const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");
+ const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;
+ const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;
+ const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;
+ if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {
+ err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +
+ "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";
+ return false;
+ }
+ l.mx_file[r] = t0->file;
+ l.mx_off[r] = t0->off;
+ l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");
+ if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||
+ t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {
+ err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";
+ return false;
+ }
+ }
+ l.lay.trellis[r] = off;
+ l.lay.suh[r] = l.lay.svh[r] = off;
+ l.lay.K[r] = ty;
+ off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)
+ }
+ l.lay.fmt = gg::kFmtMXFP4;
+ if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||
+ gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||
+ gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {
+ err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +
+ " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";
+ return false;
+ }
+ l.blob = off;
+ // a type the device kernels do not take (none of the i-quants now: gg::type_on_device lists them): the layer stays on the CPU lane, where
+ // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM
+ for (int r = 0; r < 3; ++r)
+ if (!gg::type_on_device(l.lay.K[r])) {
+ l.cpu_only = true;
- std::fprintf(stderr,
- "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "
- "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",
- gg::type_name(l.lay.K[r]));
++ static bool said = false; // (a mixed pack has many such layers: one line, not one a layer)
++ if (!said) {
++ said = true;
++ std::fprintf(stderr,
++ "ds4: this pack has %s experts: no GPU kernel for that type, every such layer's "
++ "experts stay on the CPU lane (ggml-cpu's rows; VRAM tier: none)\n",
++ gg::type_name(l.lay.K[r]));
++ }
+ break;
+ }
+ return true;
+ }
+ const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
+ if (store) {
+ const int s = store->find(l.il, e);
+ if (s >= 0) return store->mat(s, r);
+ }
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)
+ return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]
+ : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];
+ return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);
+ }
+ size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)
+ return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);
+ }
+ // (copy_mode) matrix r's rows, and the rows of one NUMA part of it
+ int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }
+ size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }
+ // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them
+ const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {
+ if (copy_mode && l.ram[(size_t) e] >= 0)
+ return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ }
+ // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)
+ void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);
+ }
+ }
+ void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);
+ }
+ }
+ // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down
+ // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back
+ bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {
+ if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);
+ else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);
+ return true;
+ }
+ return read_mx_file(l, e, blob);
+ }
+ bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)
+ for (int r = 0; r < 3; ++r)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),
+ blob + l.lay.trellis[r]))
+ return false;
+ return true;
+ }
+ void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);
+ }
+ void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);
+ for (size_t row = 0; row < (size_t) g.dim; ++row)
+ std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);
+ }
+
+ bool expert_layout(Layer& l, std::string& err) {
+ if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)
+ return expert_layout_mx(l, err);
+ const std::string P = l.name + "ffn.experts.";
+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down
+ const char* pieces[3] = {".trellis", ".suh", ".svh"};
+ l.region_off.resize((size_t) l.n_exp);
+ l.region_bytes.resize((size_t) l.n_exp);
+ l.region_file.resize((size_t) l.n_exp);
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");
+ if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {
+ err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";
+ return false;
+ }
+ l.lay.K[r] = (int) tr->shape[2] / 16;
+ l.lay.trellis[r] = off;
+ off += align256(tr->bytes);
+ l.lay.suh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);
+ l.lay.svh[r] = off;
+ off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);
+ }
+ l.lay.cb = 2;
+ l.blob = off;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const std::string E = P + std::to_string(e) + ".";
+ uint64_t lo = UINT64_MAX, hi = 0;
+ int file = -1;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const Tensor* t = pk.get(E + parts[r] + pieces[q]);
+ if (t == nullptr) {
+ err = E + parts[r] + pieces[q] + ": missing";
+ return false;
+ }
+ if (file < 0) file = t->file;
+ if (t->file != file) {
+ err = E + ": split across files";
+ return false;
+ }
+ lo = std::min(lo, t->off);
+ hi = std::max(hi, t->off + t->bytes);
+ if (e == 0) l.piece_bytes[r][q] = t->bytes;
+ else if (t->bytes != l.piece_bytes[r][q]) {
+ err = E + parts[r] + ": not the shape of expert 0's";
+ return false;
+ }
+ }
+ l.region_off[(size_t) e] = lo;
+ l.region_bytes[(size_t) e] = hi - lo;
+ l.region_file[(size_t) e] = file;
+ for (int r = 0; r < 3; ++r)
+ for (int q = 0; q < 3; ++q) {
+ const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;
+ if (e == 0) l.piece_src[r][q] = src;
+ else if (src != l.piece_src[r][q]) {
+ err = E + ": the pieces lie differently from expert 0's";
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ // the region of expert e (read from the file) -> its blob
+ void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};
+ for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);
+ }
+ }
+
+ // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)
+ void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = region + l.piece_src[r][0];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // ---- loading
+ bool load(const Options& opt, std::string& err) {
+ o = opt;
+ const double t0 = now_ms();
+ if (!pk.load(o.pack, err)) return false;
+ g.from(pk);
+ if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {
+ err = o.pack + "/ds4.txt: incomplete geometry";
+ return false;
+ }
+ if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {
+ err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +
+ std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";
+ return false;
+ }
+ freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);
+ freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);
+
+ // devices
+ int ndev = 0;
+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {
+ err = "no CUDA device";
+ return false;
+ }
+ std::vector<int> ids = o.gpus;
+ if (ids.empty())
+ for (int i = 0; i < ndev; ++i) ids.push_back(i);
+ devs.resize(ids.size());
+ for (size_t i = 0; i < ids.size(); ++i) {
+ Dev& d = devs[i];
+ d.id = ids[i];
+ CK(cudaSetDevice(d.id));
+ CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));
+ CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));
++ CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));
+ if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {
+ err = "exl3 scratch on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ CK(cudaMemGetInfo(&d.free0, &d.total));
+ }
+ // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each
+ // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)
+ int tp = o.tp;
+ if (pk.gguf && tp > 1) {
+ int blk = 32;
+ for (const auto& kv : pk.t) {
+ const std::string& n = kv.first;
+ if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||
+ n.compare(n.size() - 9, 9, "w2.weight") != 0)
+ continue;
+ const int ty = gg_type_of(kv.second.dtype);
+ if (ty >= 0) blk = std::max(blk, gg::type_block(ty));
+ }
+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||
+ g.o_groups % tp != 0))
+ --tp;
+ if (tp != o.tp)
+ std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "
+ "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);
+ }
+ // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone
+ // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the
+ // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the
+ // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down
+ // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16
+ par = o.threads || tp > 1;
+ // the chunked (layer-major) prompt path: with --tp / --threads as before; --prefill-chunk N (N > 0) or
+ // kChunkedPromptDefault bring it to a single card's default run too (a host thread a GPU for the prompt only;
+ // decode stays as asked). --prefill-chunk 0 keeps the windows of 16 (DS4_WINDOWS=1 the same). Off by default
+ // until a prompt costs the decode nothing: its buffers hold VRAM the windowed path gives to experts
+ prompt_par = par || o.tp > 1 ||
+ ((o.prefill_chunk > 0 || (o.prefill_chunk < 0 && kChunkedPromptDefault)) &&
+ std::getenv("DS4_WINDOWS") == nullptr);
++#if defined(STRATA_METAL)
++ if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path
++#endif
+ const bool tp_all = tp > 1 && tp == (int) devs.size();
+ for (size_t i = 0; i < devs.size();) {
+ Group G;
+ G.mem.push_back((int) i);
+ size_t j = i + 1;
+ while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))
+ G.mem.push_back((int) j++);
+ if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own
+ for (int m : G.mem) {
+ Group one;
+ one.mem.push_back(m);
+ groups.push_back(one);
+ }
+ } else groups.push_back(G);
+ i = j;
+ }
+ for (size_t gi = 0; gi < groups.size(); ++gi)
+ for (size_t m = 0; m < groups[gi].mem.size(); ++m) {
+ devs[(size_t) groups[gi].mem[m]].grp = (int) gi;
+ devs[(size_t) groups[gi].mem[m]].mi = (int) m;
+ }
+ // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the
+ // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM
+ // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place
+ // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times
+ if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM
+ // after VRAM's are in, by --mmap-experts)
+ else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;
+ else {
+ double exp_b = 0, other_b = 0, room = 0;
+ for (const auto& kt : pk.t) {
+ const std::string& n = kt.first;
+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
+ (!o.dspark && n.rfind("mtp.", 0) == 0))
+ continue;
+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
+ }
+ for (const Dev& d : devs)
+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -
+ (prompt_par ? 1.5 : 0.4) * (1 << 30)) *
+ o.vram_experts;
+ file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
+ }
+ // The default prompt chunk, one rule (ds4_experts_rule.hpp: prefill_chunk_default). Measured: the chunked pass streams the experts the GPUs
+ // do not hold once per chunk, so where they hold less than half of them a larger chunk is fewer passes (1 x R9700 + 251 GB, 8K prompt:
+ // 1024 200, 2048 277, 4096 353, 8192 431 tok/s; RTX 5070 / 3060 12 GB in place: 8192 244 / 228 against 1024 186 / 175); where they hold
+ // most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%
+ // (H100 32 GiB: 2048 502, 4096 538, 8192 495). A GGUF pack always has file_experts set (copied only by --mmap-experts off).
+ const bool own_default = !(par || o.tp > 1); // (a run with tp 1 and no --threads: `strata launch` writes --tp 1 for 4+ AMD cards, --tp 2 for 2-3 and nothing for one card = the engine's tp 2)
+ const bool multi_tp = rdna4() && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)
+ const bool gguf_copy = pk.gguf && (o.mmap_experts == 0 || (store_gb() <= 0.0 && auto_copy_all));
+ uint64_t vmin_bytes = ~0ull;
+ for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);
+ if (devs.empty()) vmin_bytes = 0;
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+ const bool hip_build = true;
+#else
+ const bool hip_build = false;
+#endif
+ const int rule_chunk = prefill_chunk_default(own_default, gpu_expert_share(), gguf_copy, file_experts, hip_build, vmin_bytes, kVlRows, multi_tp, defaults_026());
+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : rule_chunk) : 0;
+ if (prompt_par && o.prefill_chunk < 0 && rule_chunk == 4096 && gguf_copy)
+ std::fprintf(stderr, "ds4: prompts in chunks of 4096 tokens (the experts are copied into RAM, %.0f GiB of graphics memory; --prefill-chunk N changes it)\n",
+ (double) vmin_bytes / (double) (1ull << 30));
+ const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)
+ if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
+ if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
+ if (chunk > 0) {
+ chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));
+ eg_cap = (size_t) chunk;
+ tap_cap = std::max(kWin, 128);
+ } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)
+ std::fprintf(stderr,
+ "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"
+ " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"
+ " 461 / 1010 in chunks of %d) - %s\n",
+ kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,
+ o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"
+ : "leave --tp 1 out, or pass --threads on, for chunks");
+ // the vision tower on the first GPU, before the split: the layers get what it leaves
+ if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,
+ if (o.vision_only) return false; // and the server is told pictures are off - info_lines)
+ std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "
+ "text only\n", err.c_str());
+ vision_err = err.empty() ? "unknown reason" : err;
+ err.clear();
+ vt.release();
+ img_tok = -1;
+ }
+ if (o.vision_only) return true;
+
+ // layers: roles and the sources they read
+ if (o.dspark_auto && o.dspark && g.n_mtp > 0) { // (see Options::dspark_auto)
+ double exp_b = 0, room_b = 0;
+ for (const auto& kt : pk.t)
+ if (kt.first.rfind("layers.", 0) == 0 && kt.first.find(".ffn.experts.") != std::string::npos) exp_b += (double) kt.second.bytes;
+ for (const Dev& d : devs) room_b += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20));
+ if (exp_b > 1.5 * room_b) {
+ o.dspark = false;
+ std::fprintf(stderr, "ds4: DSpark not loaded: the routed experts (%.0f GB) are far more than the GPU holds (%.0f GB), where its verify "
+ "rounds read 3.3x a token's bytes and gain nothing (--dspark on loads it anyway)\n", exp_b / 1e9, room_b / 1e9);
+ }
+ }
+ if (!o.dspark) g.n_mtp = 0;
+ if (o.dspark_block > 0 && std::getenv("DS4_DSPARK_MAX") == nullptr) max_drafts = o.dspark_block; // (--dspark-block N: at most N drafts a round)
+ L.resize((size_t) (g.nl + g.n_mtp));
+ int last_src = -1;
+ int experts_used = 1 << 20;
+ for (int il = 0; il < g.nl + g.n_mtp; ++il) {
+ Layer& l = L[(size_t) il];
+ l.il = il;
+ l.mtp = il >= g.nl;
+ l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
+ l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
+ l.k_exp = l.mtp ? g.d_k : g.k_exp;
+ if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)
+ if (il == 0) {
+ uint64_t vram_all = 0;
+ bool integrated = false; // (a GPU that shares the PC's RAM: Strix Halo)
+ for (const Dev& dv : devs) {
+ vram_all += (uint64_t) dv.total;
+ cudaDeviceProp ip{};
+ if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;
+ }
+ // two or more RDNA4 cards holding at least half of the experts (the 4 x R9700 class): Faster by default too
+ const bool multi_rdna4 = rdna4() && devs.size() >= 2 && gpu_expert_share() >= 0.5;
+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated, multi_rdna4, defaults_026()), l.k_exp);
+ experts_used = n;
+ if (n < l.k_exp)
+ std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,
+ o.experts >= 1 ? "as asked" : multi_rdna4 ? "faster mode: the cards hold most of the experts" : "faster mode", l.k_exp);
+ else if (o.experts >= 1)
+ std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);
+ }
+ l.k_exp = std::min(l.k_exp, experts_used);
+ }
+ l.ratio = g.ratios[(size_t) il];
+ if (l.mtp) { // window attention only: no compressor, indexer or Engram
+ if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
+ err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +
+ std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";
+ return false;
+ }
+ if (!expert_layout(l, err)) return false;
+ continue;
+ }
+ l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();
+ l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();
+ l.owns_k = l.kv_src;
+ l.cand_src = il == g.cand_src;
+ l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;
+ if (l.kv_src) {
+ Src s;
+ s.layer = il;
+ s.ratio = l.ratio;
+ s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;
+ srcs.push_back(s);
+ last_src = (int) srcs.size() - 1;
+ }
+ if (l.ratio > 0) {
+ if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {
+ err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +
+ " without a KV source of that ratio before it";
+ return false;
+ }
+ l.src = last_src;
+ l.ik_src = last_src;
+ }
+ auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);
+ if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());
+ if (!expert_layout(l, err)) return false;
+ }
+
++ // A layer whose experts the chunk path's tile GEMM cannot take (IQ1_M on Metal today) would be streamed to the
++ // GPU for every prompt and stop in "kernel launch with no Metal dispatch". Prompts then run in windows of 16,
++ // which is slower but correct; say so instead of failing at the first prompt.
++ {
++ bool chunk_unsupported = false;
++ const char* why = "";
++ for (const Layer& l : L) {
++ for (int r = 0; r < 3; ++r)
++ if (!gg::type_on_chunk(gg::expert_type(l.lay, r))) { chunk_unsupported = true; why = gg::type_name(gg::expert_type(l.lay, r)); break; }
++ if (chunk_unsupported) break;
++ }
++ if (chunk > 0 && chunk_unsupported) {
++ chunk = 0;
++ std::fprintf(stderr,
++ "ds4: the prompt chunk path has no GPU kernel for %s: prompts run in windows of %d instead "
++ "of chunks - slower prompts, and the only correct path on this build\n",
++ why, kWin);
++ }
++ }
++
+ // the split: each layer's dense weights + its experts, the head on the last GPU
+ for (Layer& l : L) {
+ const std::string& P = l.name;
+ size_t b = 0;
+ if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");
+ for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",
+ "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",
+ "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})
+ b += mat_bytes(P + m);
+ for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));
+ b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");
+ b += (size_t) g.win * win_rb() + (2u << 20);
+ l.dense_bytes = b;
+ }
+ const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);
+ const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;
+ const size_t kv_per_src_dev = 0; // counted after the split
+ (void) kv_per_src_dev;
+ std::vector<double> avail(groups.size(), 0.0);
+ double avail_sum = 0;
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ for (int m : groups[gi].mem)
+ avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));
+ avail_sum += avail[gi];
+ }
+ double need_sum = (double) head_b;
+ for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ {
+ // contiguous layer ranges, each group's share of the total by its free memory
+ size_t d = 0;
+ double cum = 0, target = avail[0] / avail_sum * need_sum;
+ std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers
+ for (Layer& l : L)
+ if (l.mtp) order.push_back(&l);
+ for (Layer& l : L)
+ if (!l.mtp) order.push_back(&l);
+ for (Layer* lp : order) {
+ Layer& l = *lp;
+ const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ if (cum + lb / 2 > target && d + 1 < groups.size()) {
+ ++d;
+ target += avail[d] / avail_sum * need_sum;
+ }
+ l.grp = (int) d;
+ cum += lb;
+ }
+ // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its
+ // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage
+ // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all
+ if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {
+ const int G = (int) groups.size(), NL = g.nl;
+ std::vector<double> cap(groups.size());
+ for (int gi = 0; gi < G; ++gi) {
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ double mn = 1e300;
+ for (int m : groups[(size_t) gi].mem)
+ mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -
+ 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)
+ cap[(size_t) gi] = mn * nm;
+ }
+ for (const Layer& l : L)
+ if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;
+ cap[(size_t) G - 1] -= (double) head_b;
+ std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense
+ for (int il = 0; il < NL; ++il) {
+ pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;
+ pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;
+ }
+ auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi
+ const int nm = (int) groups[(size_t) gi].mem.size();
+ const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);
+ return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;
+ };
+ // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)
+ std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));
+ std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));
+ for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);
+ for (int gi = 1; gi < G; ++gi)
+ for (int b = gi + 1; b <= NL; ++b)
+ for (int a = gi; a < b; ++a) {
+ const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));
+ if (v < best[(size_t) gi][(size_t) b]) {
+ best[(size_t) gi][(size_t) b] = v;
+ cut[(size_t) gi][(size_t) b] = a;
+ }
+ }
+ int b = NL;
+ for (int gi = G - 1; gi >= 0; --gi) {
+ const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;
+ for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;
+ b = a;
+ }
+ }
+ // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)
+ if (const char* sp = std::getenv("DS4_SPLIT")) {
+ std::vector<int> n;
+ for (const char* q = sp; *q;) {
+ n.push_back(std::atoi(q));
+ while (*q && *q != ',') ++q;
+ if (*q == ',') ++q;
+ }
+ int il = 0;
+ for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)
+ for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;
+ for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;
+ }
+ for (Layer& l : L) {
+ l.dev = groups[(size_t) l.grp].mem[0];
+ l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)
+ }
+ head_dev = L[(size_t) g.nl - 1].dev;
+ d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;
+ }
+ // the groups' members: copies of each layer's device state, a member's part of an expert
+ for (Layer& l : L) {
+ l.ff0 = 0;
+ l.ffn = g.ff;
+ l.ah0 = l.og0 = 0;
+ l.ahn = g.heads;
+ l.ogn = g.o_groups;
+ if (l.nm == 1) continue;
+ const int na = head_members(l.nm);
+ if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";
+ return false;
+ }
+ for (int m = 1; m < l.nm; ++m) {
+ Layer c = l;
+ c.mates.clear();
+ c.mi = m;
+ c.dev = groups[(size_t) l.grp].mem[(size_t) m];
+ l.mates.push_back(std::move(c));
+ }
+ // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even
+ // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its
+ // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)
+ const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const int ha = m - (l.nm - na); // (its place among the members with heads)
+ x.ahn = ha < 0 ? 0 : g.heads / na;
+ x.ah0 = ha < 0 ? 0 : ha * x.ahn;
+ x.ogn = ha < 0 ? 0 : g.o_groups / na;
+ x.og0 = ha < 0 ? 0 : ha * x.ogn;
+ if (gguf_x) {
+ x.ffn = g.ff / l.nm;
+ x.ff0 = m * x.ffn;
+ } else
+ split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;
+ x.hlay.K[r] = l.lay.K[r];
+ x.hlay.trellis[r] = off;
+ if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks
+ const int ty = gg::expert_type(l.lay, r);
+ if (k % gg::type_block(ty) != 0) {
+ err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +
+ " blocks do not split";
+ return false;
+ }
+ x.hlay.suh[r] = x.hlay.svh[r] = off;
+ off += align256((size_t) n * gg::type_row_bytes(ty, k));
+ continue;
+ }
+ off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));
+ x.hlay.suh[r] = off;
+ off += align256((size_t) k * 2);
+ x.hlay.svh[r] = off;
+ off += align256((size_t) n * 2);
+ }
+ x.hlay.cb = l.lay.cb;
+ x.hlay.fmt = l.lay.fmt;
+ x.hblob = off;
+ }
+ }
+ // which devices hold each source's compressed KV / index keys
+ for (Src& s : srcs) {
+ s.ckv.assign(devs.size(), nullptr);
+ s.ik.assign(devs.size(), nullptr);
+ }
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ const Layer& l = member(l0, m);
+ if (l.src >= 0) {
+ srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;
+ }
+ if (l.ix_src && l.ik_src >= 0) {
+ srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;
+ if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;
+ }
+ }
+
+ // per device: the scratch, the caches, the dense weights
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ const int D = g.dim, H = g.hc;
+ const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&
+ dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&
+ dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&
+ dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&
+ dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&
+ dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&
+ dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&
+ dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&
+ dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&
+ dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&
+ dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&
+ dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&
+ dalloc(d, d.rw, W * kPicks, err) &&
- dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.moe_out, W * D, err) &&
++ dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&
++ dalloc(d, d.moe_out, W * D, err) &&
+ dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&
+ dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&
+ dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&
+ dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&
+ dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&
+ dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
+ dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
+ dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
++ if (ok && std::getenv("DS4_DBG_DEVH"))
++ std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);
+ if (!ok) return false;
+ d.attn_scr.max_t = kAttnScratchT;
+ d.attn_scr.cap = (int) topk_stride();
+ if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;
+ if (!g.eg_layers.empty()) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&
+ dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);
+ if (!ok) return false;
+ }
+ if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;
+ if (!halloc(d.bounce, bounce_bytes(), err)) return false;
+ if (!dalloc(d, d.xcnt, 4, err) || !dalloc(d, d.hccnt, kWin, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||
+ !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||
+ !dalloc(d, d.n_groups, 4, err))
+ return false;
+ if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;
+ if (g.n_mtp > 0 && (int) di == head_dev &&
+ (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))
+ return false;
+ CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));
+ CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));
+ for (Src& s : srcs) {
+ if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)
+ if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))
+ return false;
+ CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));
+ CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));
+ CK(cudaSetDevice(d.id));
+ }
+ if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))
+ return false;
+ if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))
+ return false;
+ }
+ }
+ for (Src& sr : srcs) {
+ if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))
+ return false;
+ if (sr.remote && chunk > 0 &&
+ (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||
+ !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))
+ return false;
+ const int sg = L[(size_t) sr.layer].grp;
+ for (size_t di = 0; di < devs.size(); ++di)
+ if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);
+ }
+ for (Group& G : groups) {
+ G.ar.nm = (int) G.mem.size();
+ G.ar.slots = 4;
+ G.ar.stride = (size_t) kWin * g.dim;
+ if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))
+ return false;
+ // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)
+ if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||
+ !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))
+ return false;
+ if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&
+ (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||
+ !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))
+ return false;
+ G.arb = G.ar; // (a chunk's: two slots of chunk rows)
+ G.arb.slots = 2;
+ G.arb.stride = (size_t) chunk * g.dim;
+ if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))
+ return false;
+ if (chunk > 0 && G.arb.nm > 2 &&
+ (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||
+ !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))
+ return false;
+ }
+ if (g.n_mtp > 0) {
+ if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;
+ CK(cudaSetDevice(devs[(size_t) d_dev].id));
+ CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));
+ CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));
+ }
+ // the dense weights, each device's layers by a thread of its own
+ {
+ std::vector<std::string> errs(devs.size());
+ std::vector<std::thread> th;
+ for (size_t di = 0; di < devs.size(); ++di)
+ th.emplace_back([this, di, &errs] {
+ std::string e;
+ if (!load_dense((int) di, e)) errs[di] = e;
+ });
+ for (auto& t : th) t.join();
+ for (auto& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);
+ // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E
+ // host-mapped, the steps' slots
+ // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)
+ bool mk_rows = g.n_mtp > 0 && head_nm() > 1;
+ for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;
+ if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {
+ const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");
+ if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {
+ err = "the markov embed: not a BF16 [vocab][rank] table";
+ return false;
+ }
+ if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||
+ !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {
+ if (err.empty()) err = "the markov embed: read failed";
+ return false;
+ }
+ devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)
+ mk_split = true;
+ }
+
+ // the embedding table, pinned on the host
+ {
+ const Tensor* t = pk.get("embed.weight");
+ const size_t nv = (size_t) g.vocab * g.dim;
+ const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;
+ // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):
+ // each row through ggml's to_float, to the same BF16 the host table holds
+ const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;
+ const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;
+ const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;
+ if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {
+ err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";
+ return false;
+ }
+ if (!halloc(embed_pinned, nv, err)) return false;
+ if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)
+ std::vector<uint8_t> row(erb);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {
+ err = "embed.weight: ggml has no decode for " + t->dtype;
+ return false;
+ }
+ }
+ } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)
+ std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);
+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {
+ if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {
+ err = "embed.weight: read failed";
+ return false;
+ }
+ const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);
+ for (size_t c = 0; c < (size_t) g.dim; ++c) {
+ uint32_t u;
+ std::memcpy(&u, &v[c], 4);
+ u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even
+ embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);
+ }
+ }
+ } else if (!pk.read(*t, embed_pinned)) {
+ err = "embed.weight: read failed";
+ return false;
+ }
++ if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,
++ "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",
++ (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],
++ (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],
++ (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);
++
+ }
+ // the host side of the CPU lane and the Engram rows
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;
+ if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||
- !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) ||
++ !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||
+ !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||
+ !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))
+ return false;
+ if (pk.eg_layers > 0) {
+ // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);
+ // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
+ const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
+#if defined(_WIN32)
+ // (Windows: each thread's ReadFile is synchronous, so the threads are the queue depth - Project Maya: a 28K-token
+ // prompt in one chunk, 10 threads 17.1 s, 64 13.4 s on a 20-core PC. Here (6 cores, shipped profile) 64 readers
+ // gave +4.5% on a 4K prompt but no decode gain in every pair: DS4_ENGRAM_THREADS=64 turns them on)
+#endif
+ const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
+ int nth = (int) std::min(16u, std::max(4u, hw / 2));
+ if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
+ const uint64_t avail = ram_avail();
+ uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
+ if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);
+ eg_cache.init((size_t) ((mb << 20) / row), row);
+ std::vector<std::string> paths;
+ for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);
+ // Windows: the Engram tables' files that hold nothing else are unmapped and read unbuffered - an SSD's
+ // queue kept by the reader threads instead of the page cache's (whose reads of one file from many threads
+ // went ~255K rows a second here); unbuffered reads of a mapped file take a slow path, so only unmapped
+ // ones (off by default: -8% on a 4K prompt here; DS4_EG_DIRECT=1 turns it on)
+ std::vector<char> eg_direct(pk.files.size(), 0);
+#if defined(_WIN32)
+ if (pk.gguf && std::getenv("DS4_EG_DIRECT") != nullptr && std::atoi(std::getenv("DS4_EG_DIRECT")) != 0) {
+ std::vector<char> eg_file(pk.files.size(), 0), other(pk.files.size(), 0);
+ for (const auto& kt : pk.t)
+ if (kt.second.file >= 0 && (size_t) kt.second.file < pk.files.size())
+ (kt.first.find(".engram.embed.") != std::string::npos ? eg_file : other)[(size_t) kt.second.file] = 1;
+ for (size_t i = 0; i < pk.files.size(); ++i)
+ if (eg_file[i] && !other[i]) {
+ if (i < pk.maps.size() && pk.maps[i].first != nullptr) {
+ os::unmap(pk.maps[i].first, pk.maps[i].second);
+ pk.maps[i] = {nullptr, 0};
+ }
+ eg_direct[i] = 1;
+ if (o.verbose) std::fprintf(stderr, "ds4: %s unmapped, its Engram rows read unbuffered\n",
+ pk.files[i].c_str());
+ }
+ }
+#endif
+ eg_io.start(paths, nth, eg_direct);
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ eg_win.emplace_back(new EgBatch());
+ const std::string id = std::to_string(g.eg_layers[(size_t) li]);
+ const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");
+ // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read
+ // as some other format's bytes - the old code then read the scale tensor of a pack that has none and
+ // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.
+ int fmt;
+ if (tw == nullptr) {
+ std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());
+ std::exit(2);
+ } else if (tw->dtype == "MXFP8") {
+ fmt = 0;
+ } else if (tw->dtype == "Q6_K") {
+ fmt = 1;
+ } else if (tw->dtype == "IQ4_NL") {
+ fmt = 2;
+ } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {
+ fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)
+ } else {
+ std::fprintf(stderr,
+ "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "
+ "(MXFP8, Q6_K, IQ4_NL)\n",
+ id.c_str(), tw->dtype.c_str());
+ std::exit(2);
+ }
+ if (fmt == 1 && pk.eg_hd % 256 != 0) {
+ err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole superblocks of 256)";
+ return false;
+ }
+ if (fmt == 2 && pk.eg_hd % 32 != 0) {
+ err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +
+ " values (not whole blocks of 32)";
+ return false;
+ }
+ eg_fmt.push_back((uint8_t) fmt);
+ }
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "
+ "their layer; row cache %llu MB (%zu rows)\n",
+ nth, (unsigned long long) mb, eg_cache.capacity());
+ }
+ prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);
+ if (!o.profile.empty()) {
+ std::FILE* f = std::fopen(o.profile.c_str(), "rb");
+ if (f != nullptr) {
+ std::vector<uint32_t> p(prof.size());
+ const size_t n = std::fread(p.data(), 4, p.size(), f);
+ if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());
+ std::fclose(f);
+ }
+ }
+
+ // the chunked prompt path's buffers (before the experts: they take what is left)
+ if (chunk > 0) {
+ // recommend, never force: a chunk the cards cannot hold in buffers is halved (down to 256 rows) and then the
+ // prompt runs in windows of 16, said in one line - the load only fails for a reason that is not this
+ std::vector<size_t> na0, sc0;
+ for (Dev& d : devs) {
+ na0.push_back(d.allocs.size());
+ sc0.push_back(d.scratch);
+ }
+ const size_t nh = host_allocs.size();
+ std::string aerr;
+ while (chunk > 0 && !alloc_chunks(aerr)) {
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ cudaSetDevice(d.id);
+ cudaDeviceSynchronize();
+ if (di < chunks.size() && chunks[di]) {
+ if (chunks[di]->blas) cublasDestroy(chunks[di]->blas);
+ if (chunks[di]->cs) cudaStreamDestroy(chunks[di]->cs);
+ }
+ while (d.allocs.size() > na0[di]) {
+ cudaFree(d.allocs.back());
+ d.allocs.pop_back();
+ }
+ d.scratch = sc0[di];
+ }
+ while (host_allocs.size() > nh) {
+ cudaFreeHost(host_allocs.back());
+ host_allocs.pop_back();
+ }
+ chunks.clear();
+ if (slab_auto_grew_) { // the automatic slab did not fit: the 0.2.6 slab first, the chunk only if that does not fit either
+ slab_auto_grew_ = false;
+ slab_force_base_ = true;
+ std::fprintf(stderr, "ds4: the automatic attention slab did not fit (%s): trying the 256-row slab\n", aerr.c_str());
+ aerr.clear();
+ continue;
+ }
+ const int was = chunk;
+ chunk = chunk > 512 ? chunk / 2 / 16 * 16 : 0;
+ if (chunk > 0) {
+ eg_cap = (size_t) chunk;
+ std::fprintf(stderr, "ds4: the prompt path's buffers for chunks of %d rows did not fit (%s): trying %d\n",
+ was, aerr.c_str(), chunk);
+ } else {
+ prompt_par = false;
+ eg_cap = (size_t) kWin;
+ tap_cap = kWin;
+ std::fprintf(stderr, "ds4: the prompt path's buffers did not fit (%s): prompts run in windows of %d "
+ "tokens (slower on long prompts; more free GPU memory or a lower --reserve-mib "
+ "brings the chunks back)\n", aerr.c_str(), kWin);
+ }
+ aerr.clear();
+ }
+ if (chunk > 0 && aerr.empty()) err.clear();
+ }
+ // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM
+ // now leaves over
+ if (file_experts) {
+ double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;
+ }
+ for (const Layer& l : L) all += (double) l.blob * l.n_exp;
+ const double need = std::max(0.0, all - room);
+ const uint64_t avail = ram_avail();
+ warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;
+ if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;
+ if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)
+ std::fprintf(stderr,
+ "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "
+ "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",
+ need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"
+ : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",
+ avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);
+ }
+ // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in
+ // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)
+ cores = node_cores();
+ NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());
+ if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;
+ if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)
+ {
+ const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;
+ c_xh.resize((size_t) NH * S * 2 * E);
+ c_xsb.resize((size_t) NH * S * 2 * (E / 128));
+ c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));
+ c_xhd.resize(S * FF);
+ c_xsd.resize(S * (FF / 128));
+ c_accd.resize((size_t) NH * S * DQ * (E / NH));
+ c_yd.resize(S * E);
+ }
+ // the experts: VRAM slots in what each device has left, the rest on the host
+ if (!place_experts(err)) return false;
+ if (!load_experts(err)) return false;
+ if (pk.gguf) make_pools();
+ if (chunk > 0) {
+ // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots
+ const double tr = now_ms();
+ for (auto& m : host_maps)
+ if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {
+ err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";
+ return false;
+ }
+ if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);
+ if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)
+ size_t mr = 0;
+ for (const Layer& l : L)
+ if (!l.mtp) {
+ for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)
+ mr = std::max(mr, l.blob);
+ for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));
+ }
+ }
+ const char* nr = std::getenv("DS4_READERS");
+ const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);
+ const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;
+ // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the
+ // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the
+ // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)
+ const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ auto& cp = chunks[di];
+ if (cp == nullptr || cp->ring == nullptr) continue;
+ cp->stage_stride = (mr + 4095) & ~(size_t) 4095;
+ // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)
+ bool fl = flow_on;
+ int n_exp_max = 0;
+ for (const Layer& l : L)
+ if (!l.mtp)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ // (a group's members: each its share of the down's blocks in every row)
+ fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&
+ (l.nm == 1 || (flow_tp && (g.ff / l.nm) %
+ std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));
+ n_exp_max = std::max(n_exp_max, l.n_exp);
+ }
+ if (fl) // (each part read unbuffered from its offset rounded down: a page more each)
+ cp->stage_stride = std::max(cp->stage_stride, flow_stride());
+ cp->S = cp->ring_n;
+ if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;
+ cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);
+ if (!fl) continue;
+ cudaSetDevice(devs[di].id);
+ cp->flow = true;
++#if defined(STRATA_METAL)
++ cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;
++#endif
+ cp->dev = (int) di;
+ cp->io = std::make_unique<IoQueue>(n_readers);
+ cp->stage_item.assign((size_t) cp->S, -1);
+ cp->stage_ev.resize((size_t) cp->S);
+ cp->ring_fill.resize((size_t) cp->ring_n);
+ cp->ring_free.resize((size_t) cp->ring_n);
+ for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})
+ for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ // (a region is written again only bc_n batches later - more than a layer's items, so after that
+ // layer's bounds wait, which every copy from it precedes)
+ cp->bc_n = n_exp_max + 2;
+ cp->bc_w = cp->ring_n + 1;
+ if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||
+ !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))
+ return false;
+ }
+ // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one
+ // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each
+ // window, a page fault at a time) - DS4_CHUNK_MIN still overrides
+ bool all_flow = true;
+ for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);
+ if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;
+ }
+ for (Layer& l : L) {
+ if (l.mtp) continue;
+ for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)
+ Layer& x = member(l, m);
+ if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;
+ if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;
+ }
+ }
+ }
+ {
+ void* m = nullptr;
+ if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's requests";
+ return false;
+ }
+ host_allocs.push_back(m);
+ reqs = (Req*) m;
+ if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {
+ err = "host-mapped memory for the decode path's answers";
+ return false;
+ }
+ host_allocs.push_back(m);
+ resps = (Resp*) m;
+ std::memset((void*) reqs, 0, sizeof(Req) * kRing);
+ std::memset((void*) resps, 0, sizeof(Resp) * kRing);
+ for (auto& x : layer_of) x.store(-1);
+ ahead_fl.assign(L.size() * 1024, 0);
+ ahead_rank.assign(L.size() * 1024, 0);
+ ahead_for.assign(L.size(), 0);
+ const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;
+ service = std::thread([this, core] {
+ if (core >= 0) pin_self(core);
+ serve_lane();
+ });
+ if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });
+ }
+ if (o.expert_cache) {
+ size_t maxb = 0;
+ for (const Layer& l : L) maxb = std::max(maxb, l.blob);
+ const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));
+ sw_bufs.resize((size_t) nt);
+ for (SwapBufs& B : sw_bufs) {
+ if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStream_t cs = nullptr;
+ CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));
+ B.st.push_back(cs);
+ }
+ }
+ for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ }
+ if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)
+ for (size_t di = 0; di < devs.size(); ++di) {
+ workers.push_back(std::make_unique<Worker>());
+ workers.back()->start();
+ }
+ calibrate_paths();
+ init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)
+ preload_store();
+ if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());
+ if (chunk > 0) (void) pick_flow_min(); // (the startup line)
+ reset();
+ start_warm();
+ return true;
+ }
+
+
+ // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed
+ // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)
+ size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }
+ size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }
+ size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }
+
+ // The chunked prompt's first GEMM of each kind pays the BLAS library's start (its handle's heuristics, the code objects of its
+ // kernels loaded on first use): ~1 s of hipModuleLoad plus ~2.5 s of library time on the R9700 (rocprofv3), 1-2 s on CUDA, once a
+ // process - 3.6 s on the first prompt that went through the flow, measured as 4.9 s for a 25-token pass against 1.0 s for the
+ // next one. Paid here instead, at load, on a thread beside the store's read: the chunk GEMMs on tiny inputs into the chunk's own
+ // scratch buffers (the mHC mix GEMM in F32, the FP16 GEMM the projections and the indexer weights use) - no engine state is
+ // touched (heat, profile, stats, positions, KV), so the answers are what they were. DS4_WARM=0 turns it off.
+ void warm_flow() {
+ const int T = 16, D = g.dim, hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
+ const float one = 1.0f, zero = 0.0f;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ if (!chunks[di] || chunks[di]->blas == nullptr) continue;
+ Chunk& c = *chunks[di];
+ if (c.C < T) continue;
+ cudaSetDevice(devs[di].id);
+ (void) cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, c.h, hcd, c.h, hcd, &zero, c.mix, 32);
+ const int ns[2] = {256, 32};
+ for (int n : ns)
+ (void) cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, T, D, &one, c.w16, CUDA_R_16F, D, c.x16, CUDA_R_16F, D, &zero,
+ c.y, CUDA_R_32F, n, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ (void) cudaStreamSynchronize(devs[di].st);
+ (void) cudaGetLastError();
+ }
+ }
+ // The warm-up runs once the model is loaded, on a thread of its own: HIP serialises the BLAS start with the store's page-locked
+ // arena (measured: beside the store's read it finished only when the read did, 21 s, and load still grew by its 3.5 s), so
+ // it cannot hide inside the load; after it, a person typing gives it the seconds it needs. A chunked pass that starts first
+ // waits for it (warm_mu) - never later than that pass would have been without it - and a warm-up that finds the flow already
+ // used does nothing. DS4_WARM=0 turns it off.
+ std::mutex warm_mu;
+ std::atomic<bool> flow_used{false};
+ std::thread warm_bg;
+ void start_warm() {
+ if (chunk <= 0 || (std::getenv("DS4_WARM") && std::atoi(std::getenv("DS4_WARM")) == 0)) return;
+ warm_bg = std::thread([this] {
+ std::lock_guard<std::mutex> lk(warm_mu);
+ if (flow_used.load() || quit.load()) return;
+ const double tw = now_ms();
+ warm_flow();
+ if (o.verbose) std::fprintf(stderr, "ds4: prompt path warmed up in %.0f ms\n", now_ms() - tw);
+ });
+ }
+ // ---- the chunked prompt path
+ size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows
+ size_t chunk_hand_bytes() const {
+ return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+
+ // the staging ring's size (MB) for the experts a chunk streams from the CPU lane: 400 where a good part of them are
+ // streamed; 16 where the cards hold most of the experts (they are within 30% of fitting) (a 4-card box) - the ring's VRAM is then worth more
+ // as expert slots, and the decode after a prompt keeps the windowed path's speed (R9700 x4: 29.6 -> 31.7 tok/s)
+ // The share of the routed experts the GPUs can hold, from what is known at layout (their free memory less the reserve, the
+ // prompt path's buffers and the dense weights; the trunk's experts only when no drafter is loaded): 1 = all of them.
+ double gpu_expert_share(double* exp_out = nullptr, double* room_out = nullptr, double* other_out = nullptr) const {
+ double exp_b = 0, other_b = 0, room = 0;
+ for (const auto& kt : pk.t) {
+ const std::string& n = kt.first;
+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||
+ (!o.dspark && n.rfind("mtp.", 0) == 0))
+ continue;
+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;
+ }
+ for (const Dev& d : devs)
+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) - 1.5 * (1 << 30)) * o.vram_experts;
+ if (exp_out) *exp_out = exp_b;
+ if (room_out) *room_out = room;
+ if (other_out) *other_out = other_b;
+ return exp_b > 0 ? std::min(1.0, std::max(0.0, room - other_b) / exp_b) : 1.0;
+ }
+ size_t ring_mb_auto() const {
+ double exp_b = 0, other_b = 0, room = 0;
+ const double share = gpu_expert_share(&exp_b, &room, &other_b);
+ // One rule for the staging ring (ds4_experts_rule.hpp: staging_ring_mb): 16 MB where the cards hold the experts, a small
+ // ring from each device's room where they hold most of them (4 x R9700), the larger streamed ring from the summed room on a
+ // unified-memory GPU (Strix Halo: 400 -> 1200 MB) and where the cards hold under half. DS4_RING_MB overrides.
+ bool unified = false;
+ for (const Dev& d : devs) {
+ cudaDeviceProp ip{};
+ if (cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0) unified = true;
+ }
+ const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, rdna4(), defaults_026());
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,
+ room / 1e9, other_b / 1e9, (int) mb);
+ return mb;
+ }
+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)
+ bool alloc_chunks(std::string& err) {
+ slab_auto_grew_ = false;
+ chunks.resize(devs.size());
+ const int D = g.dim, H = g.hc, HD = g.hd;
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ Dev& d = devs[di];
+ cudaSetDevice(d.id);
+ chunks[di] = std::make_unique<Chunk>();
+ Chunk& c = *chunks[di];
+ const int nm = (int) groups[(size_t) d.grp].mem.size();
+ const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);
+ int FF = 0, ff0_ = 0;
+ split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)
+ const size_t C = (size_t) chunk;
+ c.C = chunk;
+ // the base slab: STRATA_DS4_A_MB megabytes of q. Every dense GEMM of a layer is issued once a slab, so the slab width divides
+ // the prompt's dense weight traffic. A unified-memory GPU (Strix Halo, main 3133160ed) takes 32 MB: measured 8 -> 32 MB took
+ // its 32K prompt 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, decode unmoved. A discrete card keeps 8 MB
+ // (the 0.2.6 slab): its VRAM is the expert tier's, and on the RTX 5070 a bigger slab bought +2% prompt for -3% decode.
+ // STRATA_DS4_A_MB overrides the base on any GPU (8 restores the old slab).
+ bool unified = false;
+ {
+ cudaDeviceProp ip{};
+ unified = cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0;
+ }
+ static const size_t a_mb_env = [] {
+ const char* v = std::getenv("STRATA_DS4_A_MB");
+ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 0);
+ }();
+ const size_t a_mb = strata::ds4::attn_slab_base_mb(unified, a_mb_env, defaults_026());
+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);
+ // DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd
+ // once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the
+ // dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa
+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers
+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.
+ {
+ int env = -1;
+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));
+ if ((slab_force_base_ || unified || a_mb_env != 0) && env < 0) env = 0; // (no automatic growth on a unified-memory GPU, or when STRATA_DS4_A_MB chose the base)
+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)
+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;
+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert
+ double share = 0.0; // the share of the routed experts the cards hold (0 with the tier off)
+ if (o.vram_experts > 0.0) {
+ double exp_b = 0, room = 0, other_b = 0;
+ share = gpu_expert_share(&exp_b, &room, &other_b);
+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());
+ }
+ const int base = c.A;
+ // one rule (ds4_experts_rule.hpp: attn_slab_rows): the chunk's whole slab (<= 1024 rows) where the cards hold half the
+ // experts or more (4 x R9700, +9% prompt), up to 2048 rows where the VRAM is spare, the base otherwise
+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, rdna4() ? share : 0.0, defaults_026());
+ if (env < 0 && c.A > base) {
+ slab_auto_grew_ = true;
+ if (di == 0 && o.verbose) {
+ if (rdna4() && share >= 0.5)
+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: the cards hold %.0f%% of the experts; DS4_ATTN_SLAB=0 keeps %d)\n",
+ c.A, share * 100.0, base);
+ else
+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)\n",
+ c.A, spare / 1e9, base);
+ }
+ }
+ }
+ // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
+ c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
+ c.w16_elems = (size_t) 8 << 20;
+ bool has_eg = false, has_first = false;
+ size_t stride = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di && !l.mtp) {
+ has_eg = has_eg || l.eg >= 0;
+ has_first = has_first || l.il == 0;
+ stride = std::max(stride, align256(unit(member(l, m))));
+ }
+ const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;
+ bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&
+ dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&
+ dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&
+ dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&
+ dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&
+ dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&
+ dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&
+ dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&
+ dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&
+ dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&
+ dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&
+ dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&
+ dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&
+ dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&
+ dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&
+ dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&
+ dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&
+ dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&
- dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
++ dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&
+ halloc(c.bounds_pin, 2048, err);
+ if (!ok) return false;
+ if (has_first && !dalloc(d, c.emb, C * D, err)) return false;
+ if (has_eg) {
+ const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);
+ if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||
+ !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))
+ return false;
+ }
+ if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches
+ const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : ring_mb_auto();
+ c.ring_stride = stride;
+ c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);
+ if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;
+ }
+ CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));
+ for (int i = 0; i < Chunk::kParts; ++i) {
+ CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));
+ }
+ if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {
+ err = "cuBLAS on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ for (int sl = 0; sl < 2; ++sl) {
+ if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;
+ if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;
+ if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;
+ CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));
+ CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));
+ }
+ if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;
+ }
+ return true;
+ }
+
+ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, then the product.
+ // The product prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu) over hipBLAS: on gfx1151 hipBLAS
+ // measures 4.97 TFLOPS at this call convention while that kernel measures 20-40 (BM 128 x BN 256, grouped tile order), and
+ // the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate. Same operands and the same values (the
+ // k order differs, so it is rounding-level: the quality gate and the acceptance gate cover it); STRATA_DENSE_PFGEMM=0 keeps
+ // hipBLAS for an A/B.
+ bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,
+ std::string& err) {
+ if (rows <= 0) return true;
+#if defined(STRATA_USE_HIP) // (wmma_gemm.cu is part of the AMD builds only: a CUDA build has no such kernel and keeps cuBLAS)
+ static const bool pf = strata::ds4::dense_pfgemm_default(env_tri("STRATA_DENSE_PFGEMM"), defaults_026());
+#endif
+ const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);
+ for (int c0 = 0; c0 < m.n; c0 += cols) {
+ const int nc = std::min(cols, m.n - c0);
+ if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)
+ else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);
+#if defined(STRATA_USE_HIP)
+ if (pf && rows >= 64 && ((uintptr_t) x16 & 15u) == 0 &&
+ strata_pf_gemm_f16_ld(x16, ldx, c.w16, m.k, y + c0, rows, nc, m.k, ldy, beta, d.st))
+ continue;
+#endif
+ const float one = 1.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,
+ m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx: status " + std::to_string((int) st);
+ return false;
+ }
+ }
+ return true;
+ }
+ // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)
+ bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,
+ float* y, int ldy, std::string& err) {
+ const float zero = 0.0f;
+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,
+ x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP);
+ if (st != CUBLAS_STATUS_SUCCESS) {
+ err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);
+ return false;
+ }
+ return true;
+ }
+ // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale
+ bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {
+ const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;
++#if defined(STRATA_METAL)
++ if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass
++ hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);
++ return true;
++ }
++#endif
+ const float one = 1.0f, zero = 0.0f;
+ if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=
+ CUBLAS_STATUS_SUCCESS) {
+ err = "cublasSgemm (mHC mixes)";
+ return false;
+ }
+ hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);
+ return true;
+ }
+
+ // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full
+ // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file
+ // region read into pinned staging)
+ bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,
+ const uint8_t* src = nullptr) {
+ if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into
+ // pinned staging as its blob) or from the file's mapping
+ auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };
+ bool ok = true;
+ if (l.nm == 1) {
+ if (src != nullptr)
+ ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ else
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ } else {
+ const int ffh = g.ff / l.nm;
+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),
+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),
+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);
+ for (int r = 0; r < 2; ++r)
+ ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,
+ (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,
+ (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+ const auto& parts = l.hh[(size_t) e];
+ const int D = g.dim, FF = g.ff;
+ auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {
+ return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {
+ return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;
+ };
+ bool ok = true;
+ if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets
+ const uint8_t* p = src != nullptr ? src : parts[0];
+ if (l.nm == 1) {
+ for (int r = 0; r < 3; ++r)
+ ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&
+ cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&
+ cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);
+ } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;
+ ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,
+ nt * tile, nth * tile, (size_t) D / 16) &&
+ cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&
+ cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];
+ ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,
+ (size_t) (fn / 16) * row) &&
+ cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&
+ cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);
+ }
+ } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,
+ (size_t) (k / 16));
+ ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ }
+ } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of
+ // both parts' columns
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)
+ for (int X = 0; X < 2; ++X) {
+ const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);
+ if (a >= b) continue;
+ ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,
+ (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&
+ cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,
+ (size_t) (b - a) * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);
+ }
+ const size_t tile = (size_t) 32 * l.lay.K[2];
+ const int nh = D / 2;
+ for (int X = 0; X < 2; ++X) {
+ ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,
+ parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,
+ (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));
+ ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);
+ }
+ ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ return ok;
+ }
+
+ // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk
+ // in layer order; the copies run at most kParts batches ahead of the compute
+ struct Batch {
+ int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range
+ };
+ struct Streamer {
+ std::vector<Batch> b;
+ size_t cp = 0, kp = 0; // the next batch to copy, to compute
+ std::atomic<size_t> done{0}; // cp, for read_ahead's thread
+ };
+ // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel
+ // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -
+ // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone
+ void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {
+ static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *
+ 1048576.0;
+ std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied
+ double ahead = 0;
+ for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {
+ const size_t done = S.done.load(std::memory_order_relaxed);
+ while (!issued.empty() && issued.front().first < done) {
+ ahead -= issued.front().second;
+ issued.pop_front();
+ }
+ if (ahead > cap) {
+ std::this_thread::sleep_for(std::chrono::microseconds(500));
+ continue;
+ }
+ const Batch& bt = S.b[q];
+ const Layer& l0 = L[(size_t) bt.il];
+ double nb = 0;
+ if (l0.dev == di)
+ for (int i = 0; i < bt.n; ++i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4) {
+ if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);
+ nb += (double) l0.blob;
+ continue;
+ }
+ os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);
+ nb += (double) l0.region_bytes[(size_t) e];
+ }
+ issued.push_back({q++, nb});
+ ahead += nb;
+ }
+ }
+ bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {
+ const int P = Chunk::kParts;
+ while (S.cp < S.b.size() && S.cp < S.kp + P) {
+ const Batch& bt = S.b[S.cp];
+ const int half = (int) (S.cp % P);
+ const Layer& l0 = L[(size_t) bt.il];
+ int mi = 0;
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di) mi = m;
+ if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));
+ uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots
+ if (c.stage != nullptr) {
+ const double t_evt = now_ms();
+ if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)
+ c.dbg_evt_ms += now_ms() - t_evt;
+ sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;
+ std::atomic<bool> bad{false};
+ const double t_st = now_ms();
+ c.readers->run(bt.n, [&](int i) {
+ const int e = l0.host_list[(size_t) (bt.j0 + i)];
+ if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)
+ : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],
+ l0.region_bytes[(size_t) e],
+ sb + (size_t) i * c.stage_stride))
+ bad.store(true);
+ });
+ c.dbg_stage_ms += now_ms() - t_st;
+ if (bad.load()) {
+ err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";
+ return false;
+ }
+ }
+ const double t_dma = now_ms();
+ for (int i = 0; i < bt.n; ++i) {
+ uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;
+ if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,
+ sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))
+ return false;
+ c.stream_bytes += (double) unit(member(l0, mi));
+ }
+ c.dbg_dma_ms += now_ms() - t_dma;
+ CK(cudaEventRecord(c.filled[half], c.cs));
+ ++S.cp;
+ S.done.store(S.cp, std::memory_order_relaxed);
+ }
+ return true;
+ }
+
+ // ---- the flow (see Chunk)
+ // Windows (WDDM: HIP's runtime and CUDA's driver alike) submits queued work only when the host waits on the GPU:
+ // the flow's copies and the compute it queued between its waits are submitted at once (win_submit); elsewhere
+ // nothing. (CUDA too since #303: two RTX 3080s hung before the first layer, one GPU spinning on the other's flow)
+ static void kick(cudaStream_t s) { win_submit(s); }
+ size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)
+ return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;
+ }
+ size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }
+ static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }
+ // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from
+ // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of
+ // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole
+ size_t mx_read_off(const Layer& l, int m, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : 0;
+ }
+ size_t mx_read_len(const Layer& l, int r) const {
+ return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)
+ : mx_part_bytes(l, r);
+ }
+ // the device's member index in L0's group (0 alone)
+ int flow_mi(const Chunk& c, const Layer& L0) const {
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == c.dev) return m;
+ return 0;
+ }
+ void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {
+ c.items.emplace_back();
+ Chunk::Item& it = c.items.back();
+ it.il = l.il;
+ it.mi = mi;
+ it.k = k;
+ it.e = e;
+ it.v = v;
+ bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)
+ if (!in_ram && store && l.nm == 1) {
+ const int ss = store->find(l.il, e);
+ if (ss >= 0) {
+ store_blob = store->blob(ss);
+ in_ram = true;
+ }
+ }
+ if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)
+ const uint8_t* p = copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0]
+ : ram_slots[(size_t) l.ram[(size_t) e]];
+ for (int r = 0; r < 3; ++r)
+ it.src[r] = p + (copy_mode ? cp_lay[(size_t) l.il][(size_t) r] : l.lay.trellis[r]) + mx_read_off(l, mi, r);
+ ++c.it_ram;
+ return;
+ }
+ if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)
+ it.ram = store_blob != nullptr ? store_blob
+ : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];
+ bool hit = false;
+ if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after
+ const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;
+ if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again
+ c.lc_il = l.il; // once that layer's items are all read - `after` below)
+ c.lc_used = 0;
+ c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);
+ c.lc_first = i;
+ std::fill(c.lc_of.begin(), c.lc_of.end(), -1);
+ }
+ const int j = c.lc_of[(size_t) e];
+ if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)
+ it.ram = c.lc + (size_t) j * stride;
+ it.after = c.lc_writer[(size_t) j];
+ hit = true;
+ } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)
+ const int jn = c.lc_used++;
+ c.lc_of[(size_t) e] = jn;
+ c.lc_writer[(size_t) jn] = (int64_t) i;
+ it.keep = c.lc + (size_t) jn * stride;
+ it.after = (int64_t) c.lc_first - 1;
+ }
+ }
++ if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)
++ it.direct = true;
++ it.stage = -1;
++ it.left.store(3, std::memory_order_relaxed);
++ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
++ return;
++ }
+ it.stage = (int) (c.n_file++ % (uint64_t) c.S);
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);
+ it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);
+ sb += flow_region(r);
+ }
+ it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);
+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);
+ }
+ // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance
+ // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the
+ // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -
+ // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is
+ // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer
+ // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the
+ // rest of its picks are streamed in)
+ // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same
+ // profile and the same routing - and each reads only its part of every expert)
+ void flow_spec(Chunk& c, int k, int il, int T) {
+ static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;
+ Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ int* vs = member(l, mi).vs_pin;
+ c.spec_k = k;
+ c.spec_il = il;
+ c.spec_begin = c.base + c.items.size();
+ const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += pr[e];
+ std::vector<std::pair<double, int>> cand;
+ const double picks = (double) l.k_exp * T;
+ // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE
+ // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole
+ // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376
+ // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and
+ // 21 of 1568, the early reads 9-12% faster there)
+ static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;
+ const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;
+ for (int e : l.host_list) {
+ if (!early) break;
+ const double p = sum > 0 ? pr[e] / sum : 0.0;
+ const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));
+ if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {
+ return a.first > b.first;
+ });
+ if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));
+ std::vector<char> is((size_t) l.n_exp, 0);
+ int j = 0;
+ for (const auto& x : cand) {
+ vs[x.second] = l.n_slots + j++;
+ is[(size_t) x.second] = 1;
+ }
+ for (int e : l.host_list)
+ if (!is[(size_t) e]) vs[e] = l.n_slots + j++;
+ for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);
+ c.n_spec = (int) cand.size();
+ c.it_spec += cand.size();
+ }
+ // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones
+ void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {
+ const Layer& l = L[(size_t) il];
+ const int mi = flow_mi(c, l);
+ const int* vs = member(l, mi).vs_pin;
+ for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {
+ const int v = fit(c, i).v;
+ if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;
+ }
+ for (int e : l.host_list) { // (index order: their virtual slots ascending)
+ const int v = vs[e];
+ if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);
+ }
+ }
+ // the next (chunk, layer) of the device after (k, il) in the run's order that streams experts (false: none) - a
+ // long prompt's skip leaves out its early chunks' layers from Run::yoco on (kv_only there: no experts)
+ bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {
+ while (flow_next1(c, k, il, k2, il2)) {
+ if (c.fl_yoco < 0 || k2 >= c.fl_cut_k || il2 < c.fl_yoco) return true;
+ k = k2;
+ il = il2;
+ }
+ return false;
+ }
+ // the rows chunk k runs layer il on (a long prompt's skip: the cut chunk's from cut_r on, from the yoco layer on)
+ static int fl_rows(const Chunk& c, int k, int il) {
+ const int T = c.fl_T[(size_t) k];
+ return c.fl_yoco >= 0 && k == c.fl_cut_k && il >= c.fl_yoco ? T - c.fl_cut_r : T;
+ }
+ bool flow_next1(const Chunk& c, int k, int il, int& k2, int& il2) const {
+ const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);
+ const size_t i = (size_t) (it - c.fl_layers.begin());
+ if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)
+ if (k + 1 < (int) c.fl_T.size()) {
+ k2 = k + 1;
+ il2 = il;
+ return true;
+ }
+ if (i + 1 >= c.fl_layers.size()) return false;
+ k2 = 0;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (i + 1 < c.fl_layers.size()) {
+ k2 = k;
+ il2 = c.fl_layers[i + 1];
+ return true;
+ }
+ if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {
+ k2 = k + 1;
+ il2 = c.fl_layers[0];
+ return true;
+ }
+ return false;
+ }
+ // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each
+ // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now
+ bool flow_pump(Chunk& c, std::string& err) {
+ const size_t end = c.base + c.items.size();
+ IoQueue::Job jobs[6];
+ while (c.n_read < end) {
+ Chunk::Item& it = fit(c, c.n_read);
+ if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot
+ // written again once the layer before's items are all read (none still copies out of it)
+ bool wait = false;
+ if (it.keep == nullptr)
+ wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;
+ else {
+ while (c.lc_chk < c.n_read &&
+ (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))
+ ++c.lc_chk;
+ wait = c.lc_chk <= (size_t) it.after;
+ }
+ if (wait) break;
+ }
- if (it.stage >= 0) {
++ if (it.direct) {
++ const size_t R = (size_t) c.ring_n, r = c.n_read % R;
++ if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it
++ if (c.n_comp <= c.n_read - R) break;
++ CK(cudaEventSynchronize(c.ring_free[r]));
++ }
++ const Layer& l = L[(size_t) it.il];
++#if defined(STRATA_METAL)
++ uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)
++#else
++ uint8_t* dst = c.ring + r * c.ring_stride;
++#endif
++ const x3::ExpertLayout& lay = l.lay;
++ int nj = 0;
++ for (int q = 0; q < 3; ++q) {
++ IoQueue::Job& j = jobs[nj++];
++ j = IoQueue::Job{};
++ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];
++ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];
++ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];
++ j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];
++ j.len = mx_part_bytes(l, q);
++ j.dst = dst + lay.trellis[q];
++ j.exact = true;
++ j.left = &it.left;
++ j.bad = &it.bad;
++ j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;
++ j.map = pk.at(l.mx_file[q], j.off);
++ j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;
++ }
++ c.io->push(jobs, nj);
++ } else if (it.stage >= 0) {
+ const int64_t prev = c.stage_item[(size_t) it.stage];
+ if (prev >= 0) {
+ if ((size_t) prev >= c.n_dma) break;
+ const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);
+ if (q == cudaErrorNotReady) break;
+ CK(q);
+ }
+ c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;
+ const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)
+ int nj = 0;
+ for (int r = 0; r < 3; ++r) {
+ if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X) {
+ IoQueue::Job& j = jobs[nj++];
+ j = IoQueue::Job{};
+ j.len = nb;
+ j.dst = sb + (size_t) X * nb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];
+ }
+ sb += flow_region(r);
+ continue;
+ }
+ IoQueue::Job& j = jobs[nj++];
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);
+ j.len = mx_read_len(l, r);
+ j.dst = sb;
+ j.left = &it.left;
+ j.bad = &it.bad;
+ j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;
+ j.map = pk.at(l.mx_file[r], j.off);
+ j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;
+ sb += flow_region(r);
+ }
+ c.io->push(jobs, nj);
+ }
+ ++c.n_read;
+ }
+ const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;
+ struct Kick { // (every way out: what was queued is submitted)
+ cudaStream_t s;
+ const size_t &a, b;
+ ~Kick() {
+ if (a != b) kick(s);
+ }
+ } kq{c.cs, c.n_dma, dma0};
+ while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {
+ Chunk::Item& it = fit(c, c.n_dma);
+ if (it.left.load(std::memory_order_acquire) != 0) break;
+ if (it.bad.load(std::memory_order_acquire)) {
+ err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +
+ " from the model's files failed";
+ return false;
+ }
+ const size_t r = c.n_dma % R;
+ if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));
+ const Layer& l = L[(size_t) it.il];
+ uint8_t* dst = c.ring + r * c.ring_stride;
+ const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;
++ if (it.direct) { // (already in the slot)
++ CK(cudaEventRecord(c.ring_fill[r], c.cs));
++ c.stream_bytes += (double) unit(l);
++ ++c.n_dma;
++ continue;
++ }
+ for (int p = 0; p < 2; ++p)
+ CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));
+ if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)
+ const int t2 = gg::expert_type(l.lay, 2);
+ const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);
+ CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,
+ cudaMemcpyHostToDevice, c.cs));
+ } else
+ CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));
+ CK(cudaEventRecord(c.ring_fill[r], c.cs));
+ if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));
+ c.stream_bytes += (double) unit(l);
+ ++c.n_dma;
+ }
+ return true;
+ }
+ // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM
+ // ones in shares between them (the device works while the reads run)
+ bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {
+ cudaStream_t s = d.st;
+ const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)
+ if (c.spec_k != k || c.spec_il != L0.il) {
+ err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";
+ return false;
+ }
+ flow_rest(c, k, L0.il, b);
+ const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;
+ int k2 = 0, il2 = 0;
+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, fl_rows(c, k2, il2));
+ const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);
+ int v_done = 0;
+ while (c.n_comp < le) {
+ // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what
+ // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so
+ // the sums come out the same on every run whatever the reads' timing
+ const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});
+ if (!flow_pump(c, err)) return false;
+ if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy
+ const double tw = now_ms();
+ kick(s);
+ if (c.n_read > c.n_dma) c.io->wait_any(500);
+ else {
+ const Chunk::Item& it = fit(c, c.n_read);
+ if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));
+ }
+ c.wait_ms += now_ms() - tw;
+ continue;
+ }
+ const int n = (int) (i1 - i0);
+ int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;
+ ++c.n_batch;
+ // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)
+ for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];
+ bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];
+ if (bc[n] > bc[0]) {
+ CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));
+ CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));
+ if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))
+ return false;
+ }
+ for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));
+ kick(s);
+ c.n_comp = i1;
+ if (!flow_pump(c, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)
+ c.items.pop_front();
+ ++c.base;
+ }
+ return true;
+ }
+
+ // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)
+ // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order
+ bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,
+ const int* b, int i0, int i1, int T, std::string& err) {
+ const int D = g.dim, FF = l.ffn;
+ while (i0 < i1) {
+ int j = i0 + 1;
+ while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;
+ const int a = b[i0], nrows = b[j] - a;
+ int max_rows = 0;
+ for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);
+ if (nrows > 0) {
+ x3::SetArgs sa;
+ sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)
+ sa.stride = stride;
+ sa.lay = l.nm > 1 ? l.hlay : l.lay;
+ sa.n_exp = j - i0;
+ sa.bounds = d_bounds + i0;
+ sa.row0 = a;
+ sa.max_rows = max_rows;
++ sa.hbounds = b + i0;
++ sa.total_rows = nrows;
+ sa.n_embd = D;
+ sa.n_ff = FF;
+ sa.limit = g.lim;
+ // gate / up read the layer's F16 copy of the activations (the same rounding the tiles make for their
+ // shared tile): with 18-36 column blocks an expert the F32 rows were read that many times
+ sa.x16 = strata::ds4::gate_up_x16_default(defaults_026()) ? c.x16 : nullptr;
+ EkMark ek{};
+ const bool ekp = prof_chunk;
+ if (ekp) {
+ std::lock_guard<std::mutex> lk(prof_mu);
+ ek.e0 = ek_ev();
+ ek.e1 = ek_ev();
+ ek.e2 = ek_ev();
+ ek.tg = gg::expert_type(sa.lay, 0);
+ ek.td = gg::expert_type(sa.lay, 2);
+ ek.nrows = nrows;
+ ek.nexp = j - i0;
+ ek.maxrows = max_rows;
+ ek.ff = FF;
+ cudaGetDevice(&ek.dev);
+ cudaEventRecord(ek.e0, d.st);
+ }
+ if (sa.lay.fmt == gg::kFmtMXFP4) {
+ gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ if (ekp) cudaEventRecord(ek.e1, d.st);
+ gg::set_down(sa, c.h16, FF, c.y, D, d.st);
+ } else {
+ x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);
+ if (ekp) cudaEventRecord(ek.e1, d.st);
+ x3::set_down(sa, c.h16, FF, c.y, D, d.st);
+ }
+ if (ekp) {
+ cudaEventRecord(ek.e2, d.st);
+ std::lock_guard<std::mutex> lk(prof_mu);
+ ek_marks.push_back(ek);
+ }
+ if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {
+ std::vector<int> bb(b + i0, b + j + 1);
+ std::vector<uint16_t> hv((size_t) nrows * FF);
+ std::vector<float> yv((size_t) nrows * D);
+ CK(cudaStreamSynchronize(d.st));
+ CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&
+ wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};
+ std::fwrite(hdr, 8, 6, dump_y);
+ std::fwrite(bb.data(), 4, bb.size(), dump_y);
+ std::fwrite(hv.data(), 2, hv.size(), dump_y);
+ std::fwrite(yv.data(), 4, yv.size(), dump_y);
+ std::fflush(dump_y);
+ }
+ moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);
+ }
+ i0 = j;
+ }
+ (void) err;
+ return true;
+ }
+
+ bool load_dense(int di, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ size_t* acct = &d.weights;
+ for (Layer& l0 : L)
+ for (int mi = 0; mi < l0.nm; ++mi) {
+ Layer& l = member(l0, mi);
+ if (l.dev != di) continue;
+ // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,
+ // of the shared expert's FFN; the rest whole)
+ const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);
+ const std::string& P = l.name;
+ const std::string A = P + "attn.";
+ bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&
+ mat(d, A + "wkv", l.wkv, err, acct) &&
+ (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&
+ f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&
+ f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&
+ f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&
+ f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&
+ f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&
+ f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&
+ f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&
+ f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&
+ f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&
+ f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&
+ (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||
+ f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&
+ mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&
+ mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);
+ if (!ok) return false;
+ l.sink += (size_t) l.ah0;
+ for (int gi = 0; gi < l.ogn; ++gi)
+ if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;
+ if (l.kv_src) {
+ const std::string C = A + "compressor.";
+ if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;
+ if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;
+ if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ if (l.ix_src) {
+ const std::string I = A + "indexer.";
+ if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))
+ return false;
+ if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))
+ return false;
+ }
+ if (l.eg >= 0) {
+ if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;
+ if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already
+ if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;
+ } else {
+ float *qw = nullptr, *kw = nullptr;
+ if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))
+ return false;
+ std::vector<float> a((size_t) g.hc * g.dim), b(a.size());
+ CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));
+ for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product
+ CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));
+ l.eg_w = qw;
+ }
+ }
+ if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;
+ if (g.n_mtp > 0 && !l.mtp) {
+ if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;
+ if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||
+ !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))
+ return false;
+ }
+ if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns
+ const int hn = head_nm();
+ if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))
+ return false;
+ if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))
+ return false;
+ if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;
+ if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head
+ const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");
+ const int R = g.d_rank;
+ head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)
+ // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same
+ // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)
+ if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {
+ std::vector<uint16_t> rows((size_t) d.mk_vn * R);
+ if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {
+ err = "the markov head: read failed";
+ return false;
+ }
+ if (th->dtype == "BF16")
+ for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));
+ if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||
+ !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))
+ return false;
+ CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));
+ }
+ }
+ }
+ if (di == head_dev) {
+ head = d.head;
+ norm_w = d.norm_w;
+ d_norm = d.d_norm;
+ if (g.n_mtp > 0) {
+ const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";
+ if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))
+ return false;
+ const Tensor* te = pk.get(Z + "markov_head.embed.weight");
+ const Tensor* th = pk.get(Z + "markov_head.head.weight");
+ if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {
+ err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";
+ return false;
+ }
+ if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||
+ (strata::ds4::markov_head_q8_default(defaults_026())
+ ? !mk_head_q8(d, Z + "markov_head.head.weight", d_mk_head_q8, err, acct)
+ : !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))) // (0.2.6: the head as F16)
+ return false;
+ }
+ }
+ if (g.n_mtp > 0 && di == d_dev) {
+ if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||
+ !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))
+ return false;
+ }
+ return true;
+ }
+
++#if defined(STRATA_METAL)
++ // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays
++ // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and
++ // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most
++ // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.
++ double metal_expert_cap() const {
++ uint64_t phys = 0;
++ size_t len = sizeof phys;
++ sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);
++ unsigned long long pages = 0;
++ size_t pl = sizeof pages;
++ double floor_b = 0;
++ if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;
++ size_t fr = 0, tot = 0;
++ cudaMemGetInfo(&fr, &tot);
++ const double gb = 1e9, P = (double) phys;
++ double keep = std::max(7.0 * gb, 0.15 * P);
++ if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;
++ double share = 0.40;
++ if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);
++ double leave = keep;
++ if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});
++ const double cap = std::min((double) tot, P - leave);
++ task_vm_info_data_t ti;
++ mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;
++ double foot = 0;
++ if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;
++ return std::max(4.0 * gb, cap - foot - 1.0 * gb);
++ }
++#endif
++#if defined(STRATA_METAL)
++ // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it
++ // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).
++ bool zc_try(Layer& l, int e) {
++ static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)
++ if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;
++ if (l.vm[(size_t) e][0].raw != nullptr) return true;
++ uint64_t a[3];
++ for (int r = 0; r < 3; ++r) {
++ a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));
++ if (a[r] == 0) return false;
++ }
++ for (int r = 0; r < 3; ++r) {
++ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
++ m = x3::Mat{};
++ m.K = gg::expert_type(l.lay, r);
++ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
++ m.raw = (const void*) a[r];
++ m.k = r < 2 ? g.dim : g.ff;
++ m.n = r < 2 ? g.ff : g.dim;
++ }
++ return true;
++ }
++#endif
+ void set_vm(Layer& l, int e, int slot) {
+ uint8_t* b = l.vram_base + (size_t) slot * l.blob;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m = x3::Mat{};
+ m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)
+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
+ m.raw = b + l.lay.trellis[r];
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ }
+ return;
+ }
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);
+ m.suh = (const uint16_t*) (b + l.lay.suh[r]);
+ m.svh = (const uint16_t*) (b + l.lay.svh[r]);
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ m.K = l.lay.K[r];
+ m.cb = 2;
+ m.ld = 0;
+ }
+ }
+ // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)
+ void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ const uint8_t* tr = blob + l.lay.trellis[r];
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ (size_t) nth * tile);
+ std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);
+ std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);
+ }
+ }
+ void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {
+ for (int r = 0; r < 3; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r];
+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;
+ uint8_t* tr = blob + l.lay.trellis[r];
+ for (int X = 0; X < NH; ++X)
+ for (int sl = 0; sl < k / 16; ++sl)
+ std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,
+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);
+ std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);
+ for (int X = 0; X < NH; ++X)
+ std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],
+ (size_t) l.hl.n[r] * 2);
+ }
+ }
+
+ // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back
+ void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,
+ blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);
+ std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);
+ std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);
+ std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);
+ }
+ void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {
+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);
+ const Layer& x = member(l, m);
+ const int f0 = x.ff0, fn = x.ffn;
+ for (int r = 0; r < 2; ++r) {
+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;
+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)
+ std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,
+ src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);
+ if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);
+ std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);
+ }
+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];
+ std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);
+ std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);
+ if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);
+ }
+
+ bool place_experts(std::string& err) {
+ // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)
+ double room = 1e300, kept = 0;
+ for (const Dev& d : devs)
+ if (commit_charged(d)) room = commit_room();
+ const double room0 = room;
+ // per group: what its tightest member has left (each member holds its part of every VRAM expert of the
+ // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round
+ // the members layer by layer, so each member's total comes to about the mean's)
+ auto ucount = [](const Layer& l) {
+ double s = 0;
+ for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));
+ return s / l.nm;
+ };
+ for (size_t gi = 0; gi < groups.size(); ++gi) {
+ double budget = 1e300;
+ bool charged = false;
+ for (int m : groups[gi].mem) {
+ cudaSetDevice(devs[(size_t) m].id);
+ size_t fr = 0, tot = 0;
+ CK(cudaMemGetInfo(&fr, &tot));
+ // (an integrated GPU, Windows: its memory is the system's. What it allocates after the experts - the long
+ // prompt's buffers, the GEMM workspaces - spills into RAM when the experts took every byte of the GPU's
+ // address space, and on a 32K prompt that ran the machine out of RAM. So the experts leave room for them,
+ // growing with the context. DS4_UMA_GPU_RESERVE_GB: that room, tests)
+ double uma_extra = 0.0;
+ if (commit_charged(devs[(size_t) m])) {
+ uma_extra = (4.0 + 2.0e-4 * std::max(0.0, (double) o.max_context - 16384.0)) * (double) (1ull << 30);
+ if (const char* v = std::getenv("DS4_UMA_GPU_RESERVE_GB")) uma_extra = std::atof(v) * (double) (1ull << 30);
+ }
+ budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20) - uma_extra)) *
+ o.vram_experts);
++#if defined(STRATA_METAL)
++ if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {
++ const double cap = metal_expert_cap();
++ if (cap < budget) {
++ std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",
++ cap / 1e9, cap / 1e9);
++ budget = cap;
++ }
++ }
++#endif
+ charged = charged || commit_charged(devs[(size_t) m]);
+ }
+ if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)
+ double all = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;
+ const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);
+ kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;
+ budget = capped;
+ room = std::max(0.0, room - std::min(budget, all) * nm);
+ }
+ double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)
+ for (Layer& l : L)
+ if (l.grp == (int) gi && l.mtp) {
+ l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));
+ budget_t -= (double) l.n_vram * ucount(l);
+ }
+ double want = 0;
+ for (const Layer& l : L)
+ if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;
+ const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
+ for (Layer& l : L) {
+ if (l.grp != (int) gi || l.mtp) continue;
+ if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)
+ l.n_vram = 0;
+ continue;
+ }
+ l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
+ // never more than fits: the floor of each layer's share can still round up the total
+ while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
+ }
+ double used = 0;
+ for (Layer& l : L)
+ if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
+ // what the floors left: whole experts, layer by layer
+ for (Layer& l : L) {
+ if (l.grp != (int) gi || l.cpu_only) continue;
+ while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
+ ++l.n_vram;
+ used += (double) ucount(l);
+ }
+ }
+ }
+ if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it
+ const os::Commit c = os::commit();
+ const double gib = (double) (1ull << 30);
+ const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;
+ commit_kept = kept;
+ pagefile_hint = std::ceil((pf + kept) / gib);
+ std::fprintf(stderr,
+ "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "
+ "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "
+ "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "
+ "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",
+ kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);
+ } else if (room0 < 1e300 && o.verbose) {
+ std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "
+ "programs)\n", room0 / 1e9, commit_margin() / 1e9);
+ }
+ for (Layer& l : L) {
+ // the hottest experts of the profile first, then by index
+ std::vector<int> order((size_t) l.n_exp);
+ std::iota(order.begin(), order.end(), 0);
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });
+ l.slot.assign((size_t) l.n_exp, -1);
+ l.host.assign((size_t) l.n_exp, nullptr);
+ // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps
+ l.n_slots = std::min(l.n_vram, l.n_exp);
+ if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)
+ l.n_vram = l.n_slots - 1;
+ l.spare_vram = l.n_slots - 1;
+ } else {
+ l.n_vram = l.n_slots;
+ l.spare_vram = -1;
+ }
+ for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;
+ for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {
+ Layer& x = member(l, m);
+ Dev& d = devs[(size_t) x.dev];
+ cudaSetDevice(d.id);
+ void* v = nullptr;
+ if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {
+ err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +
+ " experts on CUDA" + std::to_string(d.id) + " failed";
+ return false;
+ }
+ d.allocs.push_back(v);
+ d.experts += (size_t) l.n_slots * unit(x);
+ x.vram_base = (uint8_t*) v;
+ }
+ l.heat.assign((size_t) l.n_exp, 0.0f);
+ { // the profile as the starting heat: its counts scaled to ~512 hits a layer
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ const double sc = sum > 0 ? 512.0 / sum : 0.0;
+ for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);
+ }
+ const int n_host = l.n_exp - l.n_vram;
+ { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;
+ l.hl.k[r] = k;
+ l.hl.n[r] = n;
+ l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;
+ off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);
+ l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;
+ off += align256((size_t) k * 2);
+ l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;
+ off += align256((size_t) n * 2);
+ }
+ l.hl.bytes = off;
+ }
+ l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});
+ if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);
+ } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (pk.map(l.region_file[(size_t) e]) == nullptr) {
+ err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";
+ return false;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) {
+ l.hh[(size_t) e][0] = file_part(l, e);
+ l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ } else if (n_host > 0) {
+ const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)
+ const size_t bytes = (size_t) n_parts * l.hl.bytes;
+ for (int X = 0; X < NH; ++X) {
+ void* m = node_map(bytes, X, NH);
+ if (m == nullptr) {
+ err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +
+ " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";
+ return false;
+ }
+ host_maps.emplace_back(m, bytes);
+ int hi = 0;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;
+ if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;
+ }
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];
+ }
+ l.vm.assign((size_t) l.n_exp, {});
+ if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);
+ if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0)
+ tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));
+ }
+ }
+ return true;
+ }
+
+ // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the
+ // file: the mapping reads them on first use)
+ bool load_experts_mx(std::string& err) {
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ size_t maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12; ++t)
+ th.emplace_back([&] {
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);
+ for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,
+ unit(l), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!ok) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the
+ // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them
+ struct Warm {
+ int layer, e;
+ double share;
+ };
+ std::vector<Warm> warm;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0) {
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);
+ } else {
+ warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});
+ }
+ }
+ }
+ std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });
+ if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left
+ std::vector<std::pair<int, int>> hot;
+ for (const Warm& w : warm) hot.push_back({w.layer, w.e});
+ if (!copy_experts(hot, err)) return false;
+ if (copy_mode) {
+ warm.erase(std::remove_if(warm.begin(), warm.end(),
+ [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),
+ warm.end());
+ const uint64_t av = ram_avail();
+ if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;
+ }
+ }
+ if (ram_tier && warm_bytes > 0 && !warm.empty()) {
+ // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for
+ // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not
+ // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on
+ // Windows - the engine's own pinned buffers come after this
+ const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));
+ while (ram_slots.size() < want + 1) {
+ const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());
+ uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);
+ if (c == nullptr) break;
+ ram_chunks.push_back(c);
+ ram_chunk_bytes.push_back(n * maxb);
+ for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);
+ }
+ ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;
+ for (Layer& l : L)
+ if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);
+ std::atomic<size_t> ri{0};
+ std::atomic<bool> rfail{false};
+ std::vector<std::thread> rt;
+ for (int t = 0; t < 12 && ram_held > 0; ++t)
+ rt.emplace_back([&] {
+ for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {
+ const Layer& l = L[(size_t) warm[i].layer];
+ if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);
+ for (int r = 0; r < 3; ++r)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ }
+ });
+ for (auto& t : rt) t.join();
+ if (rfail.load()) {
+ err = "reading the RAM tier's experts from the model's files failed";
+ return false;
+ }
+ for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;
+ if (ram_held > 0) ram_free.push_back(ram_held);
+ if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "
+ "read from the model's files (%.1f s)\n",
+ ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ size_t nw = 0;
+ for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)
+ b += (double) L[(size_t) warm[nw].layer].blob;
+ std::atomic<size_t> wi{0};
+ std::vector<std::thread> wt;
+ for (int t = 0; t < 12 && nw > 0; ++t)
+ wt.emplace_back([&] {
+ std::vector<uint8_t> buf(maxb);
+ for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());
+ });
+ for (auto& t : wt) t.join();
+ if (o.verbose && nw > 0)
+ std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,
+ nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);
+ return true;
+ }
+
+ // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU
+ // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the
+ // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:
+ // none - read in place: the page cache follows a conversation's experts where a static set (the profile's
+ // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the
+ // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file
+ // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what
+ // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place
+ bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {
+ if (o.mmap_experts > 0 || hot.empty()) return true;
+ const double t0 = now_ms();
+ auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };
+ double all = 0;
+ {
+ std::vector<char> seen(L.size(), 0);
+ for (const auto& h : hot) {
+ const Layer& l = L[(size_t) h.first];
+ all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));
+ }
+ }
+ const uint64_t avail = ram_avail();
+ // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)
+ // DS4_COPY_HUGETLB: the arena comes from the kernel's huge-page pool, which MemAvailable does not count: add
+ // its free bytes so the full copy is taken when the pool holds it.
+ // 0.2.7: DS4_COPY_HUGETLB unset = ON when the reserved pool already holds the whole copy (HugePages_Free), else the
+ // ordinary arena exactly as before; =0 is the off switch, =1 asks for it anyway (a smaller pool is used, the rest warns).
+ const uint64_t pool_free = os::huge_free_bytes();
+ const bool huge_arena = strata::ds4::copy_hugetlb_default(env_tri("DS4_COPY_HUGETLB"), (double) pool_free, all, defaults_026());
+ if (huge_arena && std::getenv("DS4_COPY_HUGETLB") == nullptr)
+ std::fprintf(stderr, "ds4: the reserved huge-page pool (%.1f GB free) holds the whole copy (%.1f GB): the arena on 2 MB pages "
+ "(DS4_COPY_HUGETLB=0 keeps the ordinary arena)\n", (double) pool_free / 1e9, all / 1e9);
+ bool arena_all_huge = huge_arena; // (every chunk really from the pool: the pin below needs that)
+ const double pool = huge_arena ? (double) pool_free : 0.0;
+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin) + pool,
+ commit_room());
+ // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are
+ // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed
+ // on 64 GB and on 32 GB of RAM)
+ double budget = o.mmap_experts == 0 ? std::min(all, can) : auto_copy_all && all <= can ? all : 0.0; // (auto: all or none)
+ if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;
+ if (o.mmap_experts < 0 && auto_copy_all && all > can)
+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) do not all fit the RAM this process can get now (%.1f GB beside %.0f GB for "
+ "the rest): read in place from the model's files (--mmap-experts off copies what fits)\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
+ if (o.mmap_experts == 0 && all > can)
+ std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "
+ "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "
+ "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
+ if (auto_copy_all && budget > 0)
+ std::fprintf(stderr, "ds4: --mmap-experts auto: the CPU lane's experts (%.1f GB) fit the RAM this process can get (%.1f GB beside %.0f GB for "
+ "the rest): copied in at load, no demand store (--mmap-experts on keeps them in place)\n", all / 1e9, can / 1e9, kRamMargin / 1e9);
+ if (budget <= 0) {
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "
+ "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",
+ all / 1e9, can / 1e9, kRamMargin / 1e9);
+ return true;
+ }
+ // NUMA parts: whole rows of each matrix, so any row count the nodes divide
+ int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));
+ if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)
+ nh = std::atoi(v) >= 2 ? 2 : 1;
+ if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;
+ NH = nh;
+ cp_lay.assign(L.size(), {0, 0, 0, 0});
+ for (const Layer& l : L) {
+ size_t off = 0;
+ for (int r = 0; r < 3; ++r) {
+ cp_lay[(size_t) l.il][(size_t) r] = off;
+ off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));
+ }
+ cp_lay[(size_t) l.il][3] = off;
+ }
+ // the hottest that fit the budget (a layer's first one brings its spare)
+ std::vector<int> n_copy(L.size(), 0);
+ double used = 0;
+ size_t take = 0;
+ for (; take < hot.size(); ++take) {
+ const Layer& l = L[(size_t) hot[take].first];
+ const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));
+ if (used + add > budget) break;
+ used += add;
+ ++n_copy[(size_t) l.il];
+ }
+ // each layer's slots: one block per node of (copies + spare) parts
+ std::vector<int> first(L.size(), -1);
+ copy_spare.assign(L.size(), -1);
+ for (Layer& l : L) {
+ l.ram.assign((size_t) l.n_exp, -1);
+ const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);
+ if (ns == 0) continue;
+ const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;
+ std::array<uint8_t*, 2> base = {nullptr, nullptr};
+ // DS4_COPY_HUGETLB=1: the arena on explicit 2 MB pages (the pool must be reserved), so a pin of it keeps
+ // 2 MB pages and the prompt's DMA reads stay stable (a THP arena pinned by cudaHostRegister runs in 4 KB
+ // pages: measured 11.9 s one prompt, 30.4 s the next for the same 140 GB)
+ const bool huge = huge_arena;
+ bool ok = true;
+ for (int X = 0; X < NH && ok; ++X) {
+ size_t got = bytes;
+ base[(size_t) X] = (uint8_t*) (huge ? os::huge_alloc(bytes, &got) : node_map(bytes, X, NH));
+ if (huge && base[(size_t) X] == nullptr) { // no pool on this machine: the ordinary arena instead
+ static bool warned = false;
+ if (!warned) {
+ std::fprintf(stderr, "ds4: the 2 MB huge-page arena was asked for but the pool ran out (reserve more: sysctl "
+ "vm.nr_hugepages=N); using the ordinary arena\n");
+ warned = true;
+ }
+ arena_all_huge = false;
+ got = bytes;
+ base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);
+ }
+ ok = base[(size_t) X] != nullptr;
+ if (ok) {
+ ram_chunks.push_back(base[(size_t) X]);
+ ram_chunk_bytes.push_back(got);
+ }
+ }
+ if (!ok) {
+ std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,
+ ns, (double) bytes * NH / 1e9);
+ n_copy[(size_t) l.il] = 0;
+ continue;
+ }
+ first[(size_t) l.il] = (int) ram_part.size();
+ for (int i = 0; i < ns; ++i)
+ ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});
+ if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;
+ }
+ struct Job {
+ int layer, e, slot;
+ };
+ std::vector<Job> jobs;
+ {
+ std::vector<int> next = first;
+ for (size_t i = 0; i < take; ++i) {
+ const int il = hot[i].first;
+ if (n_copy[(size_t) il] == 0) continue;
+ jobs.push_back({il, hot[i].second, next[(size_t) il]++});
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];
+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];
+ });
+ std::atomic<size_t> ji{0};
+ std::atomic<bool> bad{false};
+ std::atomic<int64_t> done_b{0};
+ std::vector<std::thread> th;
+ for (int t = 0; t < 12 && !jobs.empty(); ++t)
+ th.emplace_back([&] {
+ for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {
+ const Job& j = jobs[i];
+ const Layer& l = L[(size_t) j.layer];
+ for (int r = 0; r < 3; ++r) {
+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);
+ for (int X = 0; X < NH; ++X)
+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,
+ ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))
+ bad.store(true);
+ }
+ const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;
+ if (o.verbose && i % 2048 == 0)
+ std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,
+ db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ });
+ for (auto& t : th) t.join();
+ if (bad.load()) {
+ err = "copying the CPU lane's experts from the model's files failed";
+ return false;
+ }
+ for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;
+ copy_mode = true;
+ ram_held = (int) jobs.size();
+ double cb = 0;
+ for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;
+ std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",
+ jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,
+ jobs.size() < hot.size() ? "; the others read in place" : "");
+ // DS4_COPY_PIN: pin the copies, so the prompt path DMAs each expert from where it lies instead of copying it to
+ // the pinned staging first (the pass streams 122 GB at 6.8 GB/s through the staging, where the RAM-to-staging
+ // and DMA probes are 24 and 28 GB/s). The CPU lane reads the same memory, and pinned 4 KB pages can cost its
+ // TLB (an earlier registration measured -15% decode on a 12 GB card, where the CPU lane does nearly all of the
+ // decode) - so AUTO (unset) pins when the cards hold at least half the routed experts: the CPU lane is then a
+ // minority of the decode and the prompt stream is what hurts. Measured on 4x R9700, 2.25 bpw (share 0.72),
+ // three interleaved pairs: the second 8K prompt of a process 12.0/11.9/11.9 s against 28.8/13.8/12.3 s unpinned,
+ // decode 35.42/35.50/34.9-36.0 against 35.43/35.45/34.9-36.0 tok/s (no cost). DS4_COPY_PIN=0/1 forces it.
+ // DS4_COPY_PIN_RO=1: cudaHostRegisterReadOnly (the GPU only reads it).
+ // 0.2.7: DS4_COPY_PIN unset = ON when EITHER condition measured reliable holds: (a) the whole arena is on explicit 2 MB pages (a pin of a
+ // THP arena registers 4 KB pages and the DMA rate was erratic on the 12 GB cell: 11.9 s one prompt, 30.4 s the next), or (b) the
+ // cards hold at least half the routed experts on RDNA4 (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off
+ // switch, =1 pins any arena.
+ const bool pin_copies = strata::ds4::copy_pin_default(env_tri("DS4_COPY_PIN"), arena_all_huge, rdna4(), gpu_expert_share(),
+ !ram_chunks.empty(), defaults_026());
+ if (pin_copies) {
+ const bool ro = std::getenv("DS4_COPY_PIN_RO") != nullptr && std::atoi(std::getenv("DS4_COPY_PIN_RO")) != 0;
+ const double tr = now_ms();
+ bool ok = true;
+ for (size_t i = 0; i < ram_chunks.size() && ok; ++i)
+ ok = cudaHostRegister(ram_chunks[i], ram_chunk_bytes[i],
+ ro ? cudaHostRegisterReadOnly : cudaHostRegisterPortable) == cudaSuccess;
+ if (ok) {
+ ram_pinned = true;
+ std::fprintf(stderr, "ds4: the copied experts pinned in %.1f s (%s): the prompt path DMAs them in place\n",
+ (now_ms() - tr) / 1000.0, ro ? "read-only" : "read-write");
+ } else {
+ std::fprintf(stderr, "ds4: pinning the copied experts failed: the prompt path stages them\n");
+ }
+ }
+ return true;
+ }
+
+ bool load_experts(std::string& err) {
+ if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);
+ // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place
+ // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their
+ // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are
+ // dropped from it again
+ struct Job {
+ int layer, e;
+ };
+ std::vector<Job> jobs;
+ for (const Layer& l : L)
+ for (int e = 0; e < l.n_exp; ++e)
+ if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});
+ uint64_t warm = 0;
+ if (file_experts) {
+ std::vector<std::pair<double, Job>> cand;
+ for (const Layer& l : L) {
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0;
+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ for (const auto& c : cand) {
+ const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];
+ if (warm + b > warm_bytes) break;
+ warm += b;
+ jobs.push_back(c.second);
+ }
+ }
+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {
+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];
+ if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])
+ return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];
+ return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];
+ });
+ std::atomic<size_t> next{0};
+ std::atomic<bool> failed{false};
+ std::mutex emu;
+ std::string ferr;
+ const int nth = 12;
+ size_t maxr = 0, maxb = 0;
+ for (const Layer& l : L) {
+ maxb = std::max(maxb, l.blob);
+ for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);
+ }
+ std::atomic<int64_t> done_bytes{0};
+ const double t0 = now_ms();
+ std::vector<std::thread> th;
+ for (int t = 0; t < nth; ++t)
+ th.emplace_back([&] {
+ std::vector<uint8_t> region(maxr);
+ uint8_t *stage = nullptr, *stage2 = nullptr;
+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);
+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);
+ int cur_dev = -1;
+ for (;;) {
+ const size_t i = next.fetch_add(1);
+ if (i >= jobs.size() || failed.load()) break;
+ const Layer& l = L[(size_t) jobs[i].layer];
+ const int e = jobs[i].e;
+ if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],
+ region.data())) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";
+ failed.store(true);
+ break;
+ }
+ if (l.slot[(size_t) e] >= 0) {
+ repack(l, region.data(), stage);
+ bool up = true;
+ for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)
+ const Layer& x = member(l, m);
+ const Dev& d = devs[(size_t) x.dev];
+ if (cur_dev != d.id) {
+ cudaSetDevice(d.id);
+ cur_dev = d.id;
+ }
+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);
+ up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,
+ unit(x), cudaMemcpyHostToDevice) == cudaSuccess;
+ }
+ if (!up) {
+ std::lock_guard<std::mutex> lk(emu);
+ ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);
+ failed.store(true);
+ break;
+ }
+ if (file_experts)
+ os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);
+ } else if (!file_experts) {
+ for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);
+ }
+ const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +
+ (int64_t) l.region_bytes[(size_t) e];
+ if (o.verbose && (i % 1024) == 0)
+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),
+ db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));
+ }
+ if (stage) cudaFreeHost(stage);
+ if (stage2) cudaFreeHost(stage2);
+ });
+ for (auto& t : th) t.join();
+ if (failed.load()) {
+ err = ferr;
+ return false;
+ }
+ if (file_experts)
+ std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",
+ (now_ms() - t0) / 1000.0, warm / 1e9);
+ return true;
+ }
+
+ std::string describe() const {
+ std::ostringstream s;
+ for (size_t di = 0; di < devs.size(); ++di) {
+ const Dev& d = devs[di];
+ int a = -1, b = -1, ev = 0, et = 0;
+ for (const Layer& l : L)
+ for (int m = 0; m < l.nm; ++m)
+ if (member(l, m).dev == (int) di) {
+ if (a < 0) a = l.il;
+ b = l.il;
+ ev += l.n_vram;
+ et += l.n_exp;
+ }
+ size_t fr = 0, tot = 0;
+ cudaSetDevice(d.id);
+ cudaMemGetInfo(&fr, &tot);
+ const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();
+ char tp[64] = "";
+ if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);
+ char line[384];
+ std::snprintf(line, sizeof line,
+ "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "
+ "%.2f GB, %.0f MiB free\n",
+ d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,
+ d.caches / 1e9, fr / 1048576.0);
+ s << line;
+ }
+ int nh = 0;
+ size_t hb_ = 0;
+ for (const Layer& l : L) {
+ nh += l.n_exp - l.n_vram;
+ hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;
+ }
++ if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)
++ uint64_t h = 1469598103934665603ull;
++ for (const Layer& l : L)
++ for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }
++ char hl[96];
++ std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);
++ s << hl;
++ }
+ char line[200];
+ if (copy_mode)
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "
+ "parts), the rest in place in the model's files; %d threads\n",
+ nh, hb_ / 1e9, ram_held, NH, cpu_threads());
+ else if (file_experts)
+ std::snprintf(line, sizeof line,
+ "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",
+ nh, hb_ / 1e9, cpu_threads());
+ else
+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,
+ hb_ / 1e9, NH, cpu_threads());
+ s << line;
+ return s.str();
+ }
+
+ // The most the automatic store may take: all of the trunk's routed experts (the CPU lane's are a part of them; a store larger
+ // than they are is never filled), never less than the old 48 GB. A PC whose free RAM holds them all (the 251 GB R9700 box:
+ // 115 GB of CPU-lane experts, 1 card, 2.00 set: 12.6 -> 21.4 tokens/s with the whole set resident, misses 15.4 -> 1.5 a token)
+ // gets every one; the RAM that the room below leaves (free - 12 GB) is what bounds it elsewhere.
+ // Whether the experts the GPUs will not hold (the CPU lane's: the pack's less the GPUs' share) all fit the RAM this process can get beside
+ // kRamMargin, so that --mmap-experts auto copies them in (a copied expert is the CPU lane's own memory: no page cache to lose to the
+ // Engram tables and the files' churn, no store indirection or miss path). Measured, 1 x R9700 + 251 GB, 2.00 pack, warm: decode +10-15%.
+ // --mmap-experts on/off and --ssd-cache N stand as given.
+ bool cpu_lane_fits_ram() const {
+ if (!pk.gguf || o.mmap_experts >= 0 || o.ssd_cache_gb >= 0.0 || std::getenv("DS4_STORE_GB") || std::getenv("DS4_COPY_GB")) return false;
+ if (const char* v = std::getenv("DS4_COPY_ALL")) return std::atoi(v) != 0 && ram_avail() != ~uint64_t{0};
+ // An integrated GPU's experts are already in this same RAM (and, above the RAM, in the page file it commits): copying
+ // the CPU lane's in beside them takes the pages the GPU's experts live on. Measured on Strix Halo (64 GB, 2.00 set):
+ // 82 GB of experts on the GPU, then 53 GB copied in, and an 8K prompt did not finish in 11 minutes (the in-place path is
+ // what the 8-hour soak ran). DS4_COPY_ALL=1 still forces it; DS4_COPY_ALL=0 leaves it off everywhere.
+ for (const Dev& d : devs)
+ if (commit_charged(d)) return false;
+ const uint64_t avail = ram_avail();
+ if (avail == ~uint64_t{0}) return false;
+ double exp_b = 0, room = 0, other_b = 0;
+ const double share = gpu_expert_share(&exp_b, &room, &other_b);
+ if (exp_b <= 0) return false;
+ const double lane = exp_b * (1.0 - std::min(1.0, share));
+ return copy_all_fits(lane, avail, (double) kRamMargin, commit_room()); // (Windows: the commit room too, not only the free RAM)
+ }
+ mutable bool auto_copy_all = false; // (set by store_gb(): the auto size is the CPU lane's experts copied in, no demand store)
+ double store_cap_gb() const {
+ double b = 0;
+ for (const auto& kt : pk.t)
+ if (kt.first.find(".ffn.experts.") != std::string::npos && kt.first.rfind("mtp.", 0) != 0) b += (double) kt.second.bytes;
+ return std::max(48.0, b / 1e9);
+ }
+ double store_gb() const {
+ static double cached = -2.0; // (decided once: the layout of the blobs depends on it)
+ if (cached > -1.5) return cached;
+ double gb = o.ssd_cache_gb;
+ if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);
+ if (!pk.gguf) gb = 0.0;
+ else if (gb < 0.0 && cpu_lane_fits_ram()) { // auto, and the RAM holds every CPU-lane expert beside kRamMargin: they are copied in at load
+ auto_copy_all = true; // (copy_experts), the demand store would only be a second, smaller copy of them
+ gb = 0.0;
+ } else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, all the experts at most
+ const uint64_t avail = ram_avail();
+ avail0_ = avail;
+ const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;
+ gb = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;
+ }
+ return cached = gb;
+ }
+ // The auto size is decided again here, after the weights are on the GPU: on a unified-memory machine (Strix Halo) the GPU's
+ // allocations come out of the same RAM the store would use, and the first estimate (made at the layout, before any upload)
+ // saw all of it free. The explicit sizes (--ssd-cache, DS4_STORE_GB) stand.
+ mutable uint64_t avail0_ = ~uint64_t{0}; // (free RAM when the first estimate was made, before the weights went to the GPU)
+ double store_gb_now() const {
+ const double gb = store_gb();
+ if (gb <= 0.0 || o.ssd_cache_gb >= 0.0 || std::getenv("DS4_STORE_GB")) return gb;
+ const uint64_t avail = ram_avail();
+ if (avail == ~uint64_t{0}) return gb;
+ // (the prompt buffers and the context's caches are touched after this point and, on such a machine, take RAM too: measured on
+ // Strix Halo about 0.11 MB a token of context beyond 16384, a 40960 context ending 2.5 GB lower than a 16384 one)
+ const bool uma = avail0_ != ~uint64_t{0} && avail0_ > avail && (double) (avail0_ - avail) / 1e9 >= 4.0;
+ const double ctx_gb = uma ? 1.1e-4 * std::max(0.0, (double) o.max_context - 16384.0) : 0.0;
+ const double room = (double) avail / 1e9 - 12.0 - ctx_gb;
+ const double now = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;
+ if (now < gb)
+ std::fprintf(stderr, "ds4: RAM store: %.1f GB of RAM free after the GPU's weights: store sized %.1f GB, not %.1f\n",
+ (double) avail / 1e9, now, gb);
+ return std::min(gb, now);
+ }
+ void init_store() {
+ const double gb = store_gb_now();
+ if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;
+ if (o.ssd_cache_gb < 0.0 && !std::getenv("DS4_STORE_GB")) { // (the auto size: say when it holds less than the CPU lane's experts)
+ double exp_b = 0;
+ const double share = gpu_expert_share(&exp_b), lane_gb = exp_b * (1.0 - std::min(1.0, share)) / 1e9;
+ if (lane_gb > gb + 1.0 && o.verbose)
+ std::fprintf(stderr, "ds4: RAM store: the auto size (%.1f GB) holds less than the CPU lane's experts (%.1f GB): the hottest are kept, the "
+ "rest is read from the SSD as the prompt picks it (--ssd-cache N sets the size)\n", gb, lane_gb);
+ }
+ std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ ExpertStore::LayerGeo& gl = geo[(size_t) l.il];
+ gl.n_exp = l.n_exp;
+ for (int r = 0; r < 3; ++r) {
+ gl.file[r] = l.mx_file[r];
+ gl.off[r] = l.mx_off[r];
+ gl.stride[r] = l.mx_stride[r];
+ gl.bytes[r] = mx_part_bytes(l, r);
+ gl.tr[r] = l.lay.trellis[r];
+ }
+ gl.blob = l.blob;
+ }
+ ExpertStore::Cfg cfg;
+ cfg.budget_bytes = (size_t) (gb * 1e9);
+ cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;
+ cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;
+ cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);
+ cfg.ram_avail = [] { return ram_avail(); };
+ cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);
+ cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;
+ // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a
+ // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane
+ // took which expert, the CPU lane's are the mmap path's byte for byte)
+ const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&
+ L[0].nm == 1;
+ if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)
+ const int dev0 = devs[0].id;
+ { // (a unified-memory GPU whose budget the weights already fill: the registered store is the same RAM, past the budget the GPU faults)
+ cudaDeviceProp zp{};
+ size_t zfree = 0, ztotal = 0;
+ cudaSetDevice(dev0);
+ if (cudaGetDeviceProperties(&zp, dev0) == cudaSuccess && zp.integrated && cudaMemGetInfo(&zfree, &ztotal) == cudaSuccess &&
+ (double) zfree < gb * 1e9)
+ std::fprintf(stderr,
+ "ds4: DS4_STORE_ZC=1 on a GPU that shares the PC's RAM: the weights already fill its memory budget (%.1f GB left, the store would "
+ "register up to %.1f GB more), so this will likely fail with a GPU fault. The store is the same RAM either way, zero-copy gains nothing here; "
+ "leave DS4_STORE_ZC off.\n", (double) zfree / 1e9, gb);
+ }
+ cfg.map_chunk = [dev0](void* h, size_t n) -> void* {
+ cudaSetDevice(dev0);
+ unsigned reg_flags = cudaHostRegisterMapped | cudaHostRegisterPortable;
+#if defined(STRATA_USE_HIP)
+ // (test arm: DS4_ZC_COARSE=1, hipExtHostRegisterCoarseGrained, which ROCm Core SDK 10.1 honors on Windows; the store is read-only to the GPU)
+ if (std::getenv("DS4_ZC_COARSE") && std::atoi(std::getenv("DS4_ZC_COARSE")) != 0) reg_flags |= 0x8u;
+#endif
+ if (cudaHostRegister(h, n, reg_flags) != cudaSuccess) {
+ (void) cudaGetLastError();
+ return nullptr;
+ }
+ void* d = nullptr;
+ if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {
+ (void) cudaGetLastError();
+ cudaHostUnregister(h);
+ return nullptr;
+ }
+ if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);
+ return d;
+ };
+ cfg.unmap_chunk = [dev0](void* h) {
+ cudaSetDevice(dev0);
+ cudaHostUnregister(h);
+ };
+ }
+ bool direct = true;
+ for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;
+ auto st = std::make_unique<ExpertStore>();
+ if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {
+ std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");
+ return;
+ }
+ store = std::move(st);
+ // (the router lookahead's guesses are not read ahead with the store: measured 4.5 against 4.7 tok/s, 62% of the guesses are right and
+ // the SSD is the narrow part; DS4_AHEAD=1 turns them on)
+ if (std::getenv("DS4_AHEAD") == nullptr) ahead_mode = 0;
+ for (Layer& l : L)
+ if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);
+ std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",
+ store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,
+ (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");
+ if (want_zc) {
+ if (!store->gpu_mapped()) {
+ std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;
+ cudaSetDevice(devs[0].id);
+ if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {
+ (void) cudaGetLastError();
+ zc_stage_buf = nullptr;
+ for (Layer& l : L) l.zc.clear();
+ } else {
+ for (Layer& l : L)
+ if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);
+ else if (!l.zc.empty()) {
+ (void) cudaGetLastError();
+ l.zc.clear();
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",
+ zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);
+ }
+ }
+ // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)
+ void preload_store() {
+ if (!store) return;
+ if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;
+ std::vector<std::pair<double, std::pair<int, int>>> cand;
+ double tot = 0;
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) tot += p[e];
+ }
+ if (tot <= 0) return; // (no profile yet: the first tokens fill it)
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
+ }
+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ std::vector<std::pair<int, int>> list;
+ list.reserve(cand.size());
+ for (const auto& c : cand) list.push_back(c.second);
+ // The slots the CPU lane's experts leave over: the experts the VRAM cache holds that the profile counts least (the cache gives
+ // up its coldest to the CPU lane, and a store without them reads each from the SSD while the CPU lane waits - 1.8 misses a
+ // token on the 251 GB R9700 box, 2.9 ms of a 49 ms token), then the lane's experts the profile never saw. Last in the list: they
+ // take only the slots the likely ones left. DS4_STORE_SPARE=0: off.
+ if (strata::ds4::store_feature_default(env_tri("DS4_STORE_SPARE"), defaults_026())) {
+ std::vector<std::pair<double, std::pair<int, int>>> vr, zero;
+ for (const Layer& l : L) {
+ if (l.il >= g.nl) continue;
+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0) vr.push_back({(double) p[e] / (double) l.blob, {l.il, e}});
+ else if (p[e] == 0) zero.push_back({0.0, {l.il, e}});
+ }
+ }
+ std::stable_sort(vr.begin(), vr.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
+ for (const auto& c : vr) list.push_back(c.second);
+ for (const auto& c : zero) list.push_back(c.second);
+ }
+ const double t0 = now_ms();
+ // (with the spare experts in the list the reserve is theirs: last in the list, so the oldest ticks, so the first a demand read replaces)
+ const bool spare_on = strata::ds4::store_feature_default(env_tri("DS4_STORE_SPARE"), defaults_026());
+ const size_t leave = spare_on ? 0 : (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;
+ // The first 48 GB (what a store held before it could take every expert) are read before the first token; the rest of the
+ // hottest-first list is read after it, in pieces of 128 experts on a thread of its own, behind the model's own reads (a
+ // guess's priority) - the same bytes. Opt-in (DS4_STORE_BG=1): "ready" comes 9 s sooner on the 251 GB R9700 box (24 s against 34), but a prompt sent at
+ // once then waits for the same SSD (200 tokens: 18.7 s against 9.0), so the first answer starts no sooner and decode after
+ // it measured 3.7% lower; it only pays when the first prompt comes more than ~10 s after ready.
+ const size_t first = !(std::getenv("DS4_STORE_BG") && std::atoi(std::getenv("DS4_STORE_BG")) == 1)
+ ? list.size()
+ : std::min(list.size(), (size_t) (48e9 / (double) std::max<size_t>(1, L[0].blob)));
+ const std::vector<std::pair<int, int>> head(list.begin(), list.begin() + (long) first), tail(list.begin() + (long) first, list.end());
+ const size_t n = store->preload(head, leave);
+ std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s%s\n", n,
+ (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0,
+ tail.empty() ? "" : "; the rest is read in the background");
+ if (tail.empty()) return;
+ store_bg = std::thread([this, tail, leave] {
+ const double tb = now_ms();
+ size_t got = 0;
+ std::vector<std::pair<int, int>> piece;
+ for (size_t i = 0; i < tail.size() && !quit.load(std::memory_order_relaxed); i += 128) {
+ piece.assign(tail.begin() + (long) i, tail.begin() + (long) std::min(tail.size(), i + 128));
+ got += store->preload(piece, leave);
+ }
+ std::fprintf(stderr, "ds4: RAM store: %zu more experts (%.1f GB) read in the background in %.1f s\n", got,
+ (double) got * (double) L[0].blob / 1e9, (now_ms() - tb) / 1000.0);
+ });
+ }
+ // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;
+ // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window
+ void store_sync() {
+ if (!store || !store->gpu_mapped()) return;
+ std::vector<ExpertStore::Vis> ev;
+ store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });
+ if (ev.empty()) return;
+ std::vector<char> dirty(L.size(), 0);
+ for (const auto& v : ev) {
+ Layer& l = L[(size_t) v.layer];
+ if (l.zc.empty()) continue;
+ l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;
+ dirty[(size_t) v.layer] = 1;
+ }
+ for (Layer& l : L) {
+ if (!dirty[(size_t) l.il]) continue;
+ if (l.zc_tab == nullptr) continue;
+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.zc[(size_t) e]) {
+ const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;
+ tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;
+ if (s < 0) l.zc[(size_t) e] = 0;
+ }
+ cudaSetDevice(devs[0].id);
+ cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);
+ }
+ }
+
+ int cpu_threads() const {
+ int n = 0;
+ for (const auto& pl : pools) n += pl->size() - 1;
+ return n + 1;
+ }
+ // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over
+ // every node's cores (the pages lie where the cache put them)
+ void make_pools() {
+ std::vector<std::vector<int>> cs = cores;
+ if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)
+ std::vector<int> all = cs[0];
+ cs.assign((size_t) NH, {});
+ for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);
+ }
+ static const int lane_free = [] {
+ const char* v = std::getenv("STRATA_DS4_LANE_FREE");
+ return v != nullptr ? std::max(0, std::atoi(v)) : 2;
+ }();
+ size_t lane_cpus = 0;
+ for (const auto& v : cs) lane_cpus += v.size();
+ const bool lane_takes_all = lane_cpus >= (size_t) std::max(1u, std::thread::hardware_concurrency());
+ for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {
+ std::vector<int> c = cs[(size_t) X];
+ if (file_experts && NH == 1)
+ for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());
+ if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)
+ // cores of the first pool left free beside the host thread's, for the thread that launches the decode's kernels
+ // (and waits on the device) and whatever else the PC runs: with a worker spinning on every other logical CPU
+ // that thread waits for a core (Project Maya #359, a 20-core Windows 11 PC: decode 28.3 -> 55.5 tok/s). Only
+ // when the lane would take every logical CPU and --cpu-threads is not given.
+ // STRATA_DS4_LANE_FREE: the cores left free (2 by default, 0: as before)
+ if (X == 0 && o.cpu_threads <= 0 && lane_free > 0 && (int) c.size() > lane_free + 1 && lane_takes_all)
+ c.resize(c.size() - (size_t) lane_free);
+ int nw = (int) c.size();
+ if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));
+ pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)
+ }
+ (void) gg::mx_tier(); // (measured here, not on the first token)
+ if (o.verbose) {
+ int nthr = 0;
+ for (auto& p : pools) nthr += p->size();
+ std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),
+ gg::mx_tier_report());
+ }
+ }
+
+ void reset() {
+ pos = 0;
+ toks.clear();
+ ctoks.clear();
+ slot_held = false;
+ for (Check& c0 : ck) c0.pos = -1;
+ for (Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m) {
+ Layer& l = member(l0, m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);
+ if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the
+ // state is indexed by position, and what lies past a restored position is written again)
+ void checkpoint(int slot) {
+ Check& c = ck[slot & (kChecks - 1)];
+ c.pos = pos;
+ c.win.resize(L.size());
+ c.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ c.win[i].resize((size_t) g.win * win_rb());
+ cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);
+ if (l.c_kv) {
+ c.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);
+ cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyDeviceToHost, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ }
+
+ // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1
+ int best_check(int64_t limit) const {
+ int b = -1;
+ for (int i = 0; i < kChecks; ++i)
+ if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;
+ return b;
+ }
+ int64_t restorable(int64_t limit) const {
+ const int b = best_check(limit);
+ return b < 0 ? -1 : ck[b].pos;
+ }
+
+ // checkpoint 1 (the position before the last prompt's end) becomes the anchor too
+ void pin() {
+ if (ck[1].pos > 0) ck[2] = ck[1];
+ }
+ // the checkpoints the state can go back to (what a slot of it keeps), latest first
+ std::vector<int64_t> check_positions() const {
+ std::vector<int64_t> v;
+ for (const Check& c : ck)
+ if (c.pos > 0 && c.pos <= pos && c.win.size() == L.size() &&
+ std::find(v.begin(), v.end(), c.pos) == v.end())
+ v.push_back(c.pos);
+ std::sort(v.rbegin(), v.rend());
+ return v;
+ }
+ // the checkpoints a slot keeps (each with its role: 0, 1 or the anchor)
+ std::vector<int> slot_checks() const {
+ std::vector<int> v;
+ for (int i = 0; i < kChecks; ++i)
+ if (ck[i].pos > 0 && ck[i].pos <= pos && ck[i].win.size() == L.size()) v.push_back(i);
+ return v;
+ }
+ size_t check_bytes() const {
+ size_t n = 16;
+ for (const Layer& l : L) n += (size_t) g.win * win_rb() + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);
+ return n;
+ }
+
+ bool restore(int64_t limit) {
+ const int b = best_check(limit);
+ if (b < 0) return false;
+ const Check& c = ck[b];
+ for (size_t i = 0; i < L.size(); ++i)
+ for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)
+ Layer& l = member(L[i], m);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);
+ if (l.c_kv) {
+ cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,
+ cudaMemcpyHostToDevice, d.st);
+ }
+ }
+ for (Dev& d : devs) {
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ pos = c.pos;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ slot_held = false;
+ for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has
+ if (o2.pos > pos) o2.pos = -1;
+ return true;
+ }
+
+ // ---- the kept conversations (a slot): the state out of the model and back, through a file or a host buffer.
+ // A prompt that continues the state is reused in place (checkpoint / restore, the compressed KV left where it is);
+ // a conversation that leaves the model for another one is a slot - the whole state: the position, its tokens,
+ // every window ring (the DSpark blocks' included), the compressors' partial groups and each KV source's compressed
+ // KV and index keys up to it (rows 0 .. pos / ratio, the groups its position completed; one the prompt has not
+ // completed yet lives in the compressor's partial group). A tensor parallel group's members hold the same state,
+ // so one member's ring is saved and written back into all of them. Text only: a conversation that holds pictures
+ // is not a slot (the Engram history's rows are the tokens' again, no -1 among them).
+ static constexpr uint64_t kSlotMagic = 0x31544f4c53345344ull; // "DS4SLOT1" (little endian)
+
+ int64_t src_rows(const Src& s) const { return pos / std::max(1, s.ratio); }
+
+ // the slot's bytes: the header, the tokens, every layer's ring and partial group, every source's rows, the end
+ size_t slot_bytes() const {
+ size_t n = 8 + 8 + slot_identity().size() + 8 + (size_t) pos * 4;
+ for (const Layer& l : L) n += 8 + (size_t) g.win * win_rb() + 8 + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);
+ for (const Src& s : srcs) n += 8 + (size_t) src_rows(s) * (ckv_rb() + ik_rb());
+ n += 8 + slot_checks().size() * check_bytes(); // (the checkpoints: what the slot can be taken back at)
+ return n + 8;
+ }
+
+ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err) {
+ if (slot_held) {
+ err = "it is already set aside";
+ return 0;
+ }
+ if (pos <= 0) {
+ err = "there is no conversation";
+ return 0;
+ }
+ if ((int64_t) toks.size() != pos) {
+ err = "the state's tokens do not match its position";
+ return 0;
+ }
+ if (!imgs.empty()) {
+ err = "it holds pictures";
+ return 0;
+ }
+ const std::string id = slot_identity();
+ out.assign(slot_bytes(), 0);
+ uint8_t* p = out.data();
+ size_t off = 0;
+ const auto put = [&](const void* s, size_t b) {
+ std::memcpy(p + off, s, b);
+ off += b;
+ };
+ const uint64_t il = id.size(), n = (uint64_t) pos;
+ put(&kSlotMagic, 8);
+ put(&il, 8);
+ put(id.data(), (size_t) il);
+ put(&n, 8);
+ put(toks.data(), (size_t) n * 4);
+ for (Dev& d : devs) { // (the devices' work ends before their state is read)
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ for (size_t i = 0; i < L.size(); ++i) {
+ const Layer& l = L[i];
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ const uint64_t wb = (uint64_t) g.win * win_rb();
+ put(&wb, 8);
+ if (cudaMemcpy(p + off, l.win, (size_t) wb, cudaMemcpyDeviceToHost) != cudaSuccess) {
+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += (size_t) wb;
+ const uint64_t cn = l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0;
+ put(&cn, 8);
+ if (cn > 0) {
+ const size_t cb = (size_t) cn * 4;
+ if (cudaMemcpy(p + off, l.c_kv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||
+ cudaMemcpy(p + off + cb, l.c_score, cb, cudaMemcpyDeviceToHost) != cudaSuccess) {
+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += 2 * cb;
+ }
+ }
+ for (const Src& s : srcs) {
+ const uint64_t rows = (uint64_t) src_rows(s);
+ put(&rows, 8);
+ if (rows == 0) continue;
+ const int di = L[(size_t) s.layer].dev;
+ const uint8_t* ckv = s.ckv[(size_t) di];
+ const uint8_t* ik = s.ik[(size_t) di]; // (a pack whose sources are not index sources: none, zeros)
+ if (ckv == nullptr) {
+ err = "source layer " + std::to_string(s.layer) + "'s rows are not on its device";
+ return 0;
+ }
+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();
+ cudaSetDevice(devs[(size_t) di].id);
+ if (cudaMemcpy(p + off, ckv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||
+ (ik != nullptr && cudaMemcpy(p + off + cb, ik, ib, cudaMemcpyDeviceToHost) != cudaSuccess)) {
+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());
+ return 0;
+ }
+ off += cb + ib;
+ }
+ {
+ const std::vector<int> cs = slot_checks();
+ const uint64_t nc = cs.size();
+ put(&nc, 8);
+ for (int k : cs) {
+ const Check& c = ck[k];
+ const uint64_t role = (uint64_t) k, cp = (uint64_t) c.pos;
+ put(&role, 8);
+ put(&cp, 8);
+ for (size_t i = 0; i < L.size(); ++i) {
+ put(c.win[i].data(), (size_t) g.win * win_rb());
+ if (L[i].c_kv) {
+ put(c.comp[2 * i].data(), (size_t) L[i].ratio * g.hd * 4);
+ put(c.comp[2 * i + 1].data(), (size_t) L[i].ratio * g.hd * 4);
+ }
+ }
+ }
+ }
+ if (off + 8 != out.size()) {
+ err = "the state's size does not add up";
+ return 0;
+ }
+ put(&kSlotMagic, 8);
+ slot_held = true;
+ return (uint64_t) off;
+ }
+
+ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {
+ size_t off = 0;
+ bool wrote = false, ok = false;
+ // a load that stops after some of another conversation's state went into the layers leaves a state no
+ // checkpoint can repair: it is reset (the prompt is read from token 0), never served from the mix
+ struct FailClosed {
+ Impl& p;
+ const bool& wrote;
+ const bool& ok;
+ ~FailClosed() {
+ if (!ok && wrote) p.reset();
+ }
+ } guard{*this, wrote, ok};
+ const auto room = [&](size_t b) { return b <= bytes && off <= bytes - b; };
+ uint64_t magic = 0, il = 0, n = 0;
+ if (!room(16)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&magic, data + off, 8);
+ std::memcpy(&il, data + off + 8, 8);
+ off += 16;
+ if (magic != kSlotMagic || il == 0 || il > (1u << 20)) {
+ err = "not a saved conversation";
+ return false;
+ }
+ if (!room((size_t) il + 8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ {
+ std::string id((size_t) il, '\0');
+ std::memcpy(id.data(), data + off, (size_t) il);
+ off += (size_t) il;
+ std::memcpy(&n, data + off, 8);
+ off += 8;
+ if (id != slot_identity()) {
+ err = "another model's, or another engine version's, conversation";
+ return false;
+ }
+ }
+ if (n != (uint64_t) n_pos || n_pos <= 0 || n_pos > o.max_context || !room((size_t) n * 4)) {
+ err = "the conversation's position is not this prompt's";
+ return false;
+ }
+ std::vector<int32_t> toks_in((size_t) n);
+ std::memcpy(toks_in.data(), data + off, (size_t) n * 4);
+ off += (size_t) n * 4;
+ for (Dev& d : devs) { // (their work ends before the state is written over)
+ cudaSetDevice(d.id);
+ cudaStreamSynchronize(d.st);
+ }
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ uint64_t wb = 0, cn = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&wb, data + off, 8);
+ off += 8;
+ if (wb != (uint64_t) g.win * win_rb() || !room((size_t) wb + 8)) {
+ err = l.name + "window ring of another size";
+ return false;
+ }
+ wrote = true;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& lm = member(l, m);
+ if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += (size_t) wb;
+ std::memcpy(&cn, data + off, 8);
+ off += 8;
+ if (cn != (l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0)) {
+ err = l.name + "compressor's group of another size";
+ return false;
+ }
+ if (cn > 0) {
+ const size_t cb = (size_t) cn * 4;
+ if (!room(2 * cb)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& lm = member(l, m);
+ if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||
+ cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += 2 * cb;
+ }
+ }
+ for (const Src& s : srcs) {
+ uint64_t rows = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&rows, data + off, 8);
+ off += 8;
+ if (rows != (uint64_t) (n_pos / std::max(1, s.ratio))) {
+ err = "source layer " + std::to_string(s.layer) + "'s rows are not this conversation's";
+ return false;
+ }
+ if (rows == 0) continue;
+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();
+ if (!room(cb + ib)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ for (size_t di = 0; di < devs.size(); ++di) {
+ if (s.ckv[di] == nullptr && s.ik[di] == nullptr) continue;
+ cudaSetDevice(devs[di].id);
+ if (s.ckv[di] != nullptr && cudaMemcpy(s.ckv[di], data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ if (s.ik[di] != nullptr &&
+ cudaMemcpy(s.ik[di], data + off + cb, ib, cudaMemcpyHostToDevice) != cudaSuccess) {
+ err = "source layer " + std::to_string(s.layer) + "'s index keys: " + cudaGetErrorString(cudaGetLastError());
+ return false;
+ }
+ }
+ off += cb + ib;
+ }
+ Check got[kChecks]; // the slot's checkpoints: the positions it can be taken back at
+ {
+ uint64_t nc = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&nc, data + off, 8);
+ off += 8;
+ if (nc > (uint64_t) kChecks) {
+ err = "the slot's checkpoints are damaged";
+ return false;
+ }
+ for (uint64_t k = 0; k < nc; ++k) {
+ uint64_t role = 0, cp = 0;
+ if (!room(16)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&role, data + off, 8);
+ std::memcpy(&cp, data + off + 8, 8);
+ off += 16;
+ if (role >= (uint64_t) kChecks || got[role].pos >= 0 || cp == 0 || cp > (uint64_t) n_pos) {
+ err = "the slot's checkpoints are damaged";
+ return false;
+ }
+ Check& c = got[role];
+ c.pos = (int64_t) cp;
+ c.win.resize(L.size());
+ c.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ const size_t wb = (size_t) g.win * win_rb(), cb = L[i].c_kv ? (size_t) L[i].ratio * g.hd * 4 : 0;
+ if (!room(wb + 2 * cb)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ c.win[i].assign(data + off, data + off + wb);
+ off += wb;
+ if (cb > 0) {
+ c.comp[2 * i].resize(cb / 4);
+ c.comp[2 * i + 1].resize(cb / 4);
+ std::memcpy(c.comp[2 * i].data(), data + off, cb);
+ std::memcpy(c.comp[2 * i + 1].data(), data + off + cb, cb);
+ off += 2 * cb;
+ }
+ }
+ }
+ }
+ uint64_t end = 0;
+ if (!room(8)) {
+ err = "the slot is cut short";
+ return false;
+ }
+ std::memcpy(&end, data + off, 8);
+ off += 8;
+ if (end != kSlotMagic) {
+ err = "the slot does not end where it should";
+ return false;
+ }
+ pos = n_pos;
+ toks.swap(toks_in);
+ ctoks.resize((size_t) pos);
+ for (size_t i = 0; i < (size_t) pos; ++i)
+ ctoks[i] = toks[i] < pk.eg_vocab ? pk.eg_map[(size_t) toks[i]] : 0;
+ for (int k = 0; k < kChecks; ++k) ck[k] = std::move(got[k]); // (the slot's own; the old ones held the state
+ // this one replaced)
+ slot_held = false;
+ ok = true;
+ return true;
+ }
+
+ std::string slot_identity() const {
+ const auto size_of = [](const std::string& p) {
+ std::error_code ec;
+ const auto s = std::filesystem::file_size(p, ec);
+ return ec ? 0ull : (unsigned long long) s;
+ };
+ std::ostringstream id;
+ id << "strata-ds4-session-3 engine=" << STRATA_VERSION << " mtp=" << g.n_mtp << " dim=" << g.dim << " hd=" << g.hd
+ << " win=" << g.win << " ratios=";
+ for (int r : g.ratios) id << r << ",";
+ id << " kv=";
+ for (int v : g.kv_src) id << v << ",";
+ id << " ix=";
+ for (int v : g.ix_src) id << v << ",";
+ id << " ds4.txt:" << size_of(pk.dir + "/ds4.txt") << " engram.bin:" << size_of(pk.dir + "/engram.bin")
+ << " tensors.txt:" << size_of(pk.dir + "/tensors.txt");
+ for (size_t i = 0; i < pk.files.size(); ++i)
+ id << " " << pk.files[i] << ":" << os::file_size(pk.fds[i]);
+ return id.str();
+ }
+
+ size_t bounce_bytes() const {
+ return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);
+ }
+ size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }
+ size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }
+ size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }
+ static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need
+
+ // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the
+ // host; a prompt's pictures as their spans' rows (set_images)
+ VisionTower vt;
+ VisionGeo vg;
+ int img_tok = -1;
+ std::string vision_err; // why the tower did not load (pictures off), or empty
+ std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]
+ std::vector<Image> imgs; // the pictures of the prompt being fed, by start
+ // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst
+ bool read_bf16(const Tensor& t, size_t n, void* dst) {
+ if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);
+ if (t.dtype != "F32" || t.bytes != n * 4) return false;
+ std::vector<float> f(n);
+ if (!pk.read(t, f.data())) return false;
+ uint16_t* o = (uint16_t*) dst;
+ for (size_t i = 0; i < n; ++i) {
+ uint32_t u;
+ std::memcpy(&u, &f[i], 4);
+ o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);
+ }
+ return true;
+ }
+ bool load_vision(std::string& err) {
+ vg.layers = (int) pk.num("vision.num_hidden_layers", 0);
+ const int tok = (int) pk.num("image_token_id", -1);
+ if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none
+ vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);
+ vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);
+ vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);
+ vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;
+ const char* mark[3] = {"image_start", "image_newline", "image_end"};
+ for (int i = 0; i < 3; ++i) {
+ const Tensor* t = pk.get(mark[i]);
+ img_mark[i].resize((size_t) g.dim);
+ if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {
+ err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";
+ return false;
+ }
+ }
+ const double t0 = now_ms();
+ Dev& d = devs[0];
+ // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which
+ // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)
+ auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {
+ const Tensor* t = pk.get(n);
+ if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {
+ const std::string pre = n.substr(0, n.size() - 13);
+ const Tensor* tg = pk.get(pre + "mlp.gate.weight");
+ const Tensor* tu = pk.get(pre + "mlp.up.weight");
+ if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;
+ shape = {tg->shape[0] * 2, tg->shape[1]};
+ const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];
+ return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));
+ }
+ if (t == nullptr) return false;
+ shape = t->shape;
+ size_t n_el = 1;
+ for (int64_t x : t->shape) n_el *= (size_t) x;
+ return dst == nullptr || read_bf16(*t, n_el, dst);
+ };
+ if (!vt.load(vg, d.id, read, err)) return false;
+ img_tok = tok;
+ CK(cudaSetDevice(d.id));
+ CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)
+ if (o.verbose)
+ std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",
+ vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),
+ (now_ms() - t0) / 1000.0);
+ return true;
+ }
+ bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {
+ if (img_tok < 0) {
+ err = "this model has no vision tower (or it is off)";
+ return false;
+ }
+ std::vector<uint16_t> rows;
+ int nh = 0, nw = 0;
+ if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;
+ const size_t D = (size_t) g.dim;
+ span.resize(((size_t) nh * (nw + 1) + 2) * D);
+ uint16_t* o = span.data();
+ auto put = [&](const uint16_t* r) {
+ std::memcpy(o, r, D * 2);
+ o += D;
+ };
+ put(img_mark[0].data());
+ for (int i = 0; i < nh; ++i) {
+ for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);
+ put(img_mark[1].data());
+ }
+ put(img_mark[2].data());
+ return true;
+ }
+ // the row of a picture's span at position p (null: p is in none)
+ const uint16_t* image_row(int64_t p) const {
+ if (imgs.empty()) return nullptr;
+ auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });
+ if (it == imgs.begin()) return nullptr;
+ const Image& im = *(it - 1);
+ const int64_t r = p - im.start;
+ if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;
+ return im.rows->data() + (size_t) r * g.dim;
+ }
+ // token tk's embedding row at position p: inside a picture's span, the picture's
+ const uint16_t* emb_row(int64_t p, int tk) const {
+ const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;
+ return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;
+ }
+ bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }
+ // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)
+ int32_t ctok(int64_t p, int tk) const {
+ if (tk == img_tok && image_row(p) != nullptr) return -1;
+ return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;
+ }
+ // a chunk's image rows (route()'s bits); false: none
+ bool vl_bits(int64_t p0, int T, VlBits& b) const {
+ if (imgs.empty()) return false;
+ std::memset(&b, 0, sizeof b);
+ bool any = false;
+ for (int t = 0; t < T && t < kVlRows; ++t)
+ if (in_image(p0 + t)) {
+ b.w[t >> 5] |= 1u << (t & 31);
+ any = true;
+ }
+ return any;
+ }
+
+ // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]
+ // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a
+ // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,
+ // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.
+ void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;
+ const size_t per = (size_t) cols * (hd + hd / 32);
+ b.clear();
+ b.reads.reserve((size_t) T * cols);
+ const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";
+ const Tensor* tw = pk.get(P + "weight");
+ const Tensor* ts = pk.get(P + "scale");
+ const int fmt = eg_fmt[(size_t) li];
+ const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)
+ const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)
+ const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)
+ std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read
+ int64_t hits = 0;
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (int t = 0; t < T; ++t) {
+ const int64_t p = p0 + t;
+ uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;
+ uint8_t* scales = rows + (size_t) cols * hd;
+ if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)
+ std::memset(rows, 0, per);
+ continue;
+ }
+ int64_t prod[8];
+ bool blocked = false;
+ for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)
+ blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;
+ const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];
+ prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];
+ }
+ int64_t rolling = prod[0];
+ for (int n = 1; n < pk.eg_ngram; ++n) {
+ rolling ^= prod[n];
+ for (int h = 0; h < pk.eg_heads; ++h) {
+ const int c = (n - 1) * pk.eg_heads + h;
+ const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];
+ uint8_t* rd = rows + (size_t) c * hd;
+ uint8_t* sd = scales + (size_t) c * (hd / 32);
+ const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;
+ if (const uint8_t* hit = eg_cache.find(key)) {
+ std::memcpy(rd, hit, (size_t) hd);
+ std::memcpy(sd, hit + hd, (size_t) hd / 32);
+ ++hits;
+ continue;
+ }
+ const auto fi = first.find(key);
+ if (fi != first.end()) {
+ b.dups.push_back({fi->second, {rd, sd}});
+ continue;
+ }
+ first.emplace(key, (int) b.reads.size());
+ EgRead r;
+ r.file = tw->file;
+ r.dst = rd;
+ r.scales = sd;
+ r.key = key;
+ if (mx8) {
+ r.mx8 = true;
+ r.n = (uint32_t) (hd / 32 * 33);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)
+ r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);
+ r.off = tw->off + (uint64_t) id * r.n;
+ } else {
+ r.n = (uint32_t) hd;
+ r.off = tw->off + (uint64_t) id * hd;
+ r.file2 = ts->file;
+ r.n2 = (uint32_t) (hd / 32);
+ r.off2 = ts->off + (uint64_t) id * (hd / 32);
+ }
+ b.reads.push_back(r);
+ }
+ }
+ }
+ }
+ eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();
+ eg_hits_n += hits + (int64_t) b.dups.size();
+ eg_reads_n += (int64_t) b.reads.size();
+ eg_io.submit(b);
+ }
+ // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)
+ void eg_finish(EgBatch& b) {
+ if (b.post.load(std::memory_order_acquire) == 2) return;
+ const int64_t t0 = now_ns();
+ eg_io.wait(b);
+ int want = 0;
+ if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {
+ const int hd = pk.eg_hd;
+ for (const auto& d : b.dups) {
+ const EgRead& r = b.reads[(size_t) d.first];
+ std::memcpy(d.second.first, r.dst, (size_t) hd);
+ std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
+ }
++ if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite
++ for (size_t i = 0; i < b.reads.size(); ++i) {
++ const EgRead& r = b.reads[i];
++ for (int k = 0; k < hd / 32; ++k) {
++ uint16_t hb;
++ std::memcpy(&hb, r.dst + (size_t) k * 18, 2);
++ if ((hb & 0x7c00) == 0x7c00)
++ std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),
++ r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);
++ }
++ }
++ }
+ {
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ for (size_t i = 0; i < b.reads.size(); ++i) {
+ const EgRead& r = b.reads[i];
+ if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)
+ std::memset(r.dst, 0, (size_t) hd);
+ std::memset(r.scales, 0, (size_t) hd / 32);
+ continue;
+ }
+ if (uint8_t* c = eg_cache.put(r.key)) {
+ std::memcpy(c, r.dst, (size_t) hd);
+ std::memcpy(c + hd, r.scales, (size_t) hd / 32);
+ }
+ }
+ }
+ eg_io_ns += b.t1.load() - b.t0;
+ b.live = false;
+ b.post.store(2, std::memory_order_release);
+ } else {
+ while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();
+ }
+ eg_wait_ns += now_ns() - t0;
+ }
+ void eg_stats() {
+ st.eg_rows = eg_rows_n.load();
+ st.eg_hits = eg_hits_n.load();
+ st.eg_reads = eg_reads_n.load();
+ st.eg_io_ms = (double) eg_io_ns.load() / 1e6;
+ std::lock_guard<std::mutex> lk(eg_cache_mu);
+ st.eg_cached = (int64_t) eg_cache.size();
+ st.eg_cache_cap = (int64_t) eg_cache.capacity();
+ }
+ // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)
+ void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {
+ EgBatch b;
+ eg_issue(p0, T, only, dst, b);
+ eg_finish(b);
+ }
+
+ // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its
+ // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,
+ // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's
+ // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down
+ // pieces; the down bands). yd[slot] gets each row's expert output.
+ static constexpr int kEntMax = kWin > kc::kExl3CpuMaxN ? kWin : kc::kExl3CpuMaxN;
+ struct Ent {
+ int e = 0, n = 0, slot0 = 0;
+ int tok[kEntMax] = {};
+ };
+ // tokens an entry holds: the i-quant GGUF experts take a whole window's tokens of an expert in one call (the CPU
+ // lane's exact multi-token rows, 7+ tokens an expert: decoded once for all), the others (MXFP4, EXL3) up to 8
+ static int ent_cap(const Layer& l) {
+ return l.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(l.lay) ? kEntMax : kc::kExl3CpuMaxN;
+ }
+ void run_nodes(int n, const std::function<void(int)>* f) {
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);
+ pools[0]->help();
+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();
+ }
+
+ // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each
+ // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),
+ // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits
+ // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds
+ // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever
+ // number of entries take it, and each job its band of h straight into the down's int8 input.
+ // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)
+ void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);
+ constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);
+ const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);
+ const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)
+ const float lim = g.lim;
+ static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");
+ { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at
+ // once, so the SSD reads them with large requests side by side while the first entries compute
+ // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost
+ // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ os::prefetch_cold(want);
+ }
+ // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token
+ // index past the window (none on the decode or chunk paths) gets a row of its own after them
+ uint8_t* xq = (uint8_t*) c_xh.data();
+ uint8_t* hq = xq + (size_t) kWin * xa;
+ uint8_t* xq_extra = hq + (size_t) kSlots * ha;
+ auto x_act = [&](const Ent& en, int i) -> const void* {
+ const int t = en.tok[i];
+ return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));
+ }
+ }
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];
+ float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ gp[i] = yg[i];
+ up[i] = yu[i];
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's)
+ gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);
+ gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);
+ for (int i = 0; i < en.n; ++i) {
+ float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;
+ for (int r = r0; r < r1; ++r) {
+ float gv = yg[i][r - r0], uv = yu[i][r - r0];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);
+ }
+ };
+ run_nodes(ne * GB, f);
+ if (!band_q)
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* ap[gg::kMxCpuMaxN];
+ float* yp[gg::kMxCpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = hq + (size_t) (en.slot0 + i) * ha;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;
+ }
+ gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);
+ };
+ run_nodes(ne * DB, f);
+ }
+ // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input
+ // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same
++#if defined(STRATA_METAL)
++ // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD
++ // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN
++ // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the
++ // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per
++ // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).
++ static bool la_enabled() { return la_depth() > 0; }
++ static int la_depth() {
++ static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)
++ return std::min(d, 2);
++ }
++ struct LaPool {
++ std::mutex m;
++ std::condition_variable cv;
++ std::deque<std::pair<const void*, size_t>> q;
++ LaPool() {
++ const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;
++ for (int i = 0; i < nth; ++i)
++ std::thread([this] {
++ for (;;) {
++ std::pair<const void*, size_t> r;
++ {
++ std::unique_lock<std::mutex> lk(m);
++ cv.wait(lk, [&] { return !q.empty(); });
++ r = q.front();
++ q.pop_front();
++ while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)
++ }
++ const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;
++ const size_t len = r.second + ((uintptr_t) r.first - a);
++ std::vector<unsigned char> vec((len + 16383) / 16384);
++ bool cold = true;
++ if (mincore((void*) a, len, (char*) vec.data()) == 0) {
++ cold = false;
++ for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }
++ }
++ if (cold) os::prefetch({r});
++ }
++ }).detach();
++ }
++ void push(const void* p, size_t n) {
++ { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }
++ cv.notify_one();
++ }
++ };
++ std::mutex la_mu;
++ std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed
++ long la_hit = 0, la_all = 0, la_issued = 0;
++ void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {
++ if (l.nm != 1) return;
++ static LaPool* pool = nullptr;
++ static bool reg = false;
++ if (pool == nullptr) {
++ la_guess.assign(L.size(), {});
++ pool = new LaPool();
++ }
++ if (std::getenv("DS4_CPU_LOG") && !reg) {
++ reg = true;
++ static Impl* self = this;
++ std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });
++ }
++ for (const Ent& en : ents) {
++ ++la_all;
++ const auto& gs = la_guess[(size_t) l.il];
++ if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;
++ }
++ if (la_n <= 0) return;
++ cudaEventSynchronize(ev2);
++ static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;
++ for (int a = 1; a <= la_n; ++a) {
++ Layer& nx = L[(size_t) (l.il + a)];
++ std::vector<int> cpu;
++ for (int r = 0; r < nx.k_exp; ++r) {
++ const int e = h_idx_a[(a - 1) * kPicks + r];
++ if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);
++ }
++ la_guess[nx.il] = cpu;
++ for (int e : cpu)
++ for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }
++ }
++ }
++#endif
+ void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {
+ constexpr int RB = 64; // rows a job
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;
+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();
+ gg::IqCpu fm;
+ std::string ferr;
+ if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||
+ fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {
+ static std::atomic<bool> said{false};
+ if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());
+ for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);
+ return;
+ }
++ static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;
++ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }
++ ++n_calls;
++ double tp0 = now_ms();
+ const float lim = g.lim;
+ const double tp0 = now_ms();
+ { // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a
+ // range, ~0.13 ms an IQ2_XXS expert, on the lane's critical path before the first entry computes.
+ // off by default (the mmap path, owner PC: not faster in every pair); DS4_IQ_PF_COLD=1 turns it on)
+ std::vector<std::pair<const void*, size_t>> want;
+ for (const Ent& en : ents)
+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
+ static const bool pf_all = std::getenv("DS4_IQ_PF_COLD") == nullptr || std::atoi(std::getenv("DS4_IQ_PF_COLD")) == 0;
++#if defined(__APPLE__) // (macOS: the cold reads of a call's matrices in parallel, and a page-cache census under DS4_CPU_LOG)
++ if (pf_all && std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)
++ static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;
++ if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }
++ for (const auto& w : want) {
++ const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;
++ unsigned char vec[64];
++ const size_t len = std::min<size_t>(w.second, 64 * 16384);
++ bool cold = false;
++ if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;
++ const double t0 = now_ms();
++ os::prefetch({w});
++ (cold ? cold_ms : warm_ms) += now_ms() - t0;
++ (cold ? cold_n : warm_n)++;
++ }
++ } else if (pf_all && want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {
++ // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of
++ // all the call's experts at once keep the SSD busy)
++ std::function<void(int)> pf[4];
++ for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };
++ run_nodes((int) want.size(), pf);
++ } else
+ if (pf_all) os::prefetch(want);
+ else os::prefetch_cold(want);
++#else
++ if (pf_all) os::prefetch(want);
++ else os::prefetch_cold(want);
++#endif
+ }
++ double tp1 = now_ms(); t_pre += tp1 - tp0;
+ // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of
+ // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or
+ // chunk paths), then (x no longer needed) the slot's h
+ uint8_t* aq = (uint8_t*) c_xh.data();
+ auto x_act = [&](const Ent& en, int i) -> uint8_t* {
+ const int t = en.tok[i];
+ return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;
+ };
+ {
+ bool done[kWin] = {};
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i) {
+ const int t = en.tok[i];
+ if (t < kWin && done[t]) continue;
+ if (t < kWin) done[t] = true;
+ gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
+ }
+ }
- const double tp1 = now_ms();
++ double tq1 = now_ms(); t_q1 += tq1 - tp1;
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];
+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);
+ const void* ap[kEntMax];
+ float* hp[kEntMax];
+ for (int i = 0; i < en.n; ++i) {
+ ap[i] = x_act(en, i);
+ hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)
+ }
+ // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any
+ // block size of the type)
+ gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,
+ r0 - X * FFh, r1 - X * FFh, lim);
+ };
+ run_nodes(ne * GB, f);
- const double tp2 = now_ms();
++ double tr1 = now_ms(); t_r1 += tr1 - tq1;
+ for (const Ent& en : ents)
+ for (int i = 0; i < en.n; ++i)
+ gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
+ aq + (size_t) (kWin + en.slot0 + i) * E * 4);
- const double tp3 = now_ms();
++ double tq2 = now_ms(); t_q2 += tq2 - tr1;
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);
+ const void* hp[kEntMax];
+ float* yp[kEntMax];
+ for (int i = 0; i < en.n; ++i) {
+ hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;
+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)
+ }
+ gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
+ };
+ run_nodes(ne * DB, f);
++ t_r2 += now_ms() - tq2;
+ { // DS4_PROF: where the CPU lane's time goes (act prep + prefetch, gate/up, h quant, down), summed
+ static const bool on = std::getenv("DS4_PROF") != nullptr;
+ if (on) {
+ static double acc[6] = {};
+ const double te = now_ms();
- acc[0] += tp1 - tp0; acc[1] += tp2 - tp1; acc[2] += tp3 - tp2; acc[3] += te - tp3; acc[4] += 1; acc[5] += ne;
++ acc[0] += tq1 - tp0; acc[1] += tr1 - tq1; acc[2] += tq2 - tr1; acc[3] += te - tq2; acc[4] += 1; acc[5] += ne;
+ if (((long long) acc[4]) % 1000 == 0)
+ std::fprintf(stderr, "ds4 prof CPU lane (IQ rows) per call: n %.0f, entries/call %.2f, prep %.3f ms, gate+up %.3f ms, h-quant %.3f ms, down %.3f ms\n", acc[4], acc[5] / acc[4], acc[0] / acc[4], acc[1] / acc[4], acc[2] / acc[4], acc[3] / acc[4]);
+ }
+ }
+ }
+
+ // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
+ struct StoreHold {
+ Impl* m = nullptr;
+ int layer = 0;
+ std::vector<int> es, sl;
+ std::vector<char> ok;
+ // lazy: the reads are started here and waited for by wait_all() / wait_expert(); the destructor releases
+ StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {
+ if (!m_->store) return;
+ m = m_;
+ layer = l.il;
+ for (const Ent& en : ents)
+ if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);
+ m->store->begin(layer, es.data(), (int) es.size(), ok, sl);
+ }
+ int idx(int e) const {
+ for (size_t i = 0; i < es.size(); ++i)
+ if (es[i] == e) return (int) i;
+ return -1;
+ }
+ // expert e needs no waiting: not a store expert (RAM copy) or its reads are done
+ bool ready(int e) const {
+ if (!m) return true;
+ const int i = idx(e);
+ return i < 0 || m->store->loaded(sl[(size_t) i]);
+ }
+ void wait_expert(int e) {
+ if (!m) return;
+ const int i = idx(e);
+ if (i >= 0) m->store->wait_slots(sl, ok, i, i + 1);
+ }
+ void wait_all() {
+ if (m) m->store->wait_slots(sl, ok, 0, (int) es.size());
+ }
+ ~StoreHold() {
+ if (m) m->store->release(layer, es.data(), (int) es.size(), ok);
+ }
+ };
+ // DS4_MISS_SPLIT=0: the old order (wait for every read, then compute); default: the entries whose reads are done
+ // computed first, the others each as its read lands (the same bits: every entry's rows are independent)
+ void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,
+ const int* ord = nullptr) {
+ if (ldx == 0) ldx = (size_t) g.dim;
+ if (l.lay.fmt == gg::kFmtMXFP4) {
+ StoreHold hold(this, l, ents);
+ static const bool split = std::getenv("DS4_MISS_SPLIT") == nullptr || std::atoi(std::getenv("DS4_MISS_SPLIT")) != 0;
+ if (!split || ord != nullptr || !hold.m || ents.size() < 2) {
+ hold.wait_all();
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
+ std::vector<Ent> hot, cold;
+ for (const Ent& en : ents) (hold.ready(en.e) ? hot : cold).push_back(en);
+ if (cold.empty()) {
+ hold.wait_all();
+ return cpu_experts_mx(l, x, ents, ldx, ord);
+ }
+ for (const Ent& en : hot) hold.wait_expert(en.e); // (done reading: only the bookkeeping)
+ if (!hot.empty()) cpu_experts_mx(l, x, hot, ldx, nullptr);
+ // the ones that came in meanwhile first, then the rest in the order they land
+ std::vector<char> done(cold.size(), 0);
+ for (size_t left = cold.size(); left > 0;) {
+ size_t pick = cold.size();
+ for (size_t i = 0; i < cold.size(); ++i)
+ if (!done[i] && hold.ready(cold[i].e)) {
+ pick = i;
+ break;
+ }
+ if (pick == cold.size())
+ for (size_t i = 0; i < cold.size(); ++i)
+ if (!done[i]) {
+ pick = i;
+ break;
+ }
+ hold.wait_expert(cold[pick].e);
+ std::vector<Ent> one(1, cold[pick]);
+ cpu_experts_mx(l, x, one, ldx, nullptr);
+ done[pick] = 1;
+ --left;
+ }
+ return;
+ }
+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;
+ const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;
+ const int ne = (int) ents.size();
+ const float lim = g.lim;
+ auto part = [&](int e, int X, int r) {
+ const uint8_t* b = l.hh[(size_t) e][(size_t) X];
+ kc::Exl3Part p;
+ p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);
+ p.suh = (const uint16_t*) (b + l.hl.suh[r]);
+ p.svh = (const uint16_t*) (b + l.hl.svh[r]);
+ p.k = l.hl.k[r];
+ p.n = l.hl.n[r];
+ p.K = l.lay.K[r];
+ return p;
+ };
+ auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };
+ auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };
+ auto acc = [&](int X, int sl, int r, int q) {
+ return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;
+ };
+ auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };
+ auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };
+ auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };
+ std::function<void(int)> f[4];
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / (2 * GQ))];
+ const int r = (j / GQ) % 2, q = j % GQ;
+ const kc::Exl3Part p = part(en.e, X, r);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ const float* xt = x + (size_t) en.tok[i] * ldx;
+ for (int b = 2 * q; b < 2 * q + 2; ++b)
+ xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);
+ xp[i] = xh(X, sl, r);
+ ap[i] = acc(X, sl, r, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);
+ };
+ run_nodes(ne * 2 * GQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / GB)];
+ const int b = j % GB;
+ const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float y[2][128], a[128], hb[128];
+ for (int r = 0; r < 2; ++r) {
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < GQ; ++q) {
+ const float* aq = acc(X, sl, r, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];
+ kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);
+ }
+ for (int c = 0; c < 128; ++c) {
+ float gv = y[0][c], uv = y[1][c];
+ if (lim > 0.0f) {
+ gv = std::min(gv, lim);
+ uv = std::min(std::max(uv, -lim), lim);
+ }
+ hb[c] = gv / (1.0f + std::exp(-gv)) * uv;
+ }
+ const int gb = X * GB + b;
+ xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);
+ }
+ };
+ run_nodes(ne * GB, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DQ)];
+ const int q = j % DQ;
+ const kc::Exl3Part p = part(en.e, X, 2);
+ const float* xp[kc::kExl3CpuMaxN];
+ float* ap[kc::kExl3CpuMaxN];
+ for (int i = 0; i < en.n; ++i) {
+ xp[i] = xhd(en.slot0 + i);
+ ap[i] = accd(X, en.slot0 + i, q);
+ }
+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);
+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);
+ };
+ run_nodes(ne * DQ, f);
+ for (int X = 0; X < NH; ++X)
+ f[X] = [&, X](int j) {
+ const Ent& en = ents[(size_t) (j / DB)];
+ const int b = j % DB;
+ const kc::Exl3Part pd = part(en.e, X, 2);
+ for (int i = 0; i < en.n; ++i) {
+ const int sl = en.slot0 + i;
+ float a[128];
+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;
+ for (int q = 0; q < DQ; ++q) {
+ const float* aq = accd(X, sl, q) + b * 128;
+ for (int c = 0; c < 128; ++c) a[c] += aq[c];
+ }
+ double xs = 0.0;
+ for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];
+ kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);
+ }
+ };
+ run_nodes(ne * DB, f);
+ }
+
+ // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it
+ // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs
+ // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's
+ // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:
+ // the CPU lane reads it from the file again.
+ void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot
+ for (;;) {
+ RamSwap rs;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });
+ if (quit.load()) return;
+ rs = ram_queue.front();
+ ram_queue.pop_front();
+ }
+ const Layer& l = L[(size_t) rs.layer];
+ if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {
+ for (int r = 0; r < 3; ++r) // (one copy of it in RAM)
+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],
+ mx_part_bytes(l, r), false);
+ } else {
+ rs.in = -1; // (failed: the slot goes back)
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_ready.push_back(rs);
+ }
+ }
+
+ void cache_loop(SwapBufs& B) {
+ for (;;) {
+ Swap sw;
+ {
+ std::unique_lock<std::mutex> lk(sw_mu);
+ sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });
+ if (quit.load()) return;
+ sw = sw_queue.front();
+ sw_queue.pop_front();
+ }
+ Layer& l = L[(size_t) sw.layer];
+ if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)
+ swap_in_mx(l, sw, B);
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ continue;
+ }
+ parts_to_blob(l, l.hh[(size_t) sw.in], B.a);
+ if (l.nm == 1) {
+ const Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t cs = B.st[(size_t) l.dev];
+ cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);
+ if (!file_experts)
+ cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ } else { // a group: each member's part in, each member's part out
+ for (int m = 0; m < l.nm; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = B.st[(size_t) x.dev];
+ blob_to_half(l, B.a, m, B.h);
+ cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);
+ cudaStreamSynchronize(cs);
+ if (file_experts) continue;
+ cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,
+ cudaMemcpyDeviceToHost, cs);
+ cudaStreamSynchronize(cs);
+ half_to_blob(l, B.h, m, B.b);
+ }
+ }
+ if (!file_experts)
+ for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);
+ if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte
+ std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);
+ bool ok = true;
+ for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)
+ const int e = pass == 0 ? sw.in : sw.out;
+ reg.resize(l.region_bytes[(size_t) e]);
+ pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());
+ if (pass == 0) {
+ repack(l, reg.data(), blob.data());
+ std::vector<uint8_t> dev(l.blob);
+ if (l.nm == 1)
+ cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);
+ else
+ for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)
+ std::vector<uint8_t> h(member(l, m).hblob);
+ cudaSetDevice(devs[(size_t) member(l, m).dev].id);
+ cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,
+ cudaMemcpyDeviceToHost);
+ half_to_blob(l, h.data(), m, dev.data());
+ }
+ for (int r = 0; r < 3; ++r)
+ ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&
+ std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&
+ std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;
+ } else {
+ for (int X = 0; X < NH; ++X) {
+ repack_part(l, reg.data(), X, part.data());
+ for (int r = 0; r < 3; ++r) {
+ const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];
+ ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&
+ std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&
+ std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;
+ }
+ }
+ }
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,
+ ok ? "both match the model file" : "MISMATCH");
+ }
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_ready.push_back(sw);
+ }
+ }
+
+ // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);
+ // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)
+ // from the layer's spare RAM slot, which it is copied into from VRAM here
+ void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {
+ uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)
+ const std::vector<cudaStream_t>& copy_st = B.st;
+ bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till
+ // this swap's switch)
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);
+ ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),
+ cudaMemcpyHostToDevice, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ }
+ if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot
+ bool got = ok;
+ for (int m = 0; m < l.nm && got; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),
+ unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);
+ }
+ if (got) blob_to_ram(l, sw_b, sw.ram);
+ else sw.ram = -1; // (back to the file's pages then)
+ } else if (ok && store && l.nm == 1 && l.lay.fmt == gg::kFmtMXFP4 && strata::ds4::store_feature_default(env_tri("DS4_STORE_PUT"), defaults_026())) {
+ // `out` goes back to the CPU lane: its blob is in VRAM, so it is copied into a store slot now - the lane would otherwise read it from
+ // the SSD and wait for it (1-2 misses a token on the 251 GB R9700 box). DS4_STORE_PUT=0: off.
+ uint8_t* dst = store->put_begin(l.il, sw.out);
+ if (dst != nullptr) {
+ const Layer& x = member(l, 0);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaStream_t cs = copy_st[(size_t) x.dev];
+ const bool got = cudaMemcpyAsync(dst, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l), l.blob, cudaMemcpyDeviceToHost, cs) ==
+ cudaSuccess &&
+ cudaStreamSynchronize(cs) == cudaSuccess;
+ store->put_done(l.il, sw.out, got);
+ }
+ }
+ if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte
+ std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);
+ for (int m = 0; m < l.nm && ok; ++m) {
+ const Layer& x = member(l, m);
+ cudaSetDevice(devs[(size_t) x.dev].id);
+ cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),
+ cudaMemcpyDeviceToHost);
+ if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());
+ }
+ ok = ok && read_mx_file(l, sw.in, file.data());
+ for (int r = 0; r < 3 && ok; ++r)
+ ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ bool ok_out = true;
+ if (sw.ram >= 0) {
+ ok_out = read_mx_file(l, sw.out, file.data());
+ ram_to_blob(l, sw.ram, dev.data());
+ for (int r = 0; r < 3 && ok_out; ++r)
+ ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;
+ }
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,
+ sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");
+ } else if (!ok) {
+ std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);
+ }
+ }
+
+ // after a chunked prompt: the expert cache catches up with what the prompt picked at once - a window prompt gets
+ // its swaps one tick at a time over its hundreds of windows, a chunked one has a single tick - so the decode that
+ // follows finds the hot experts in VRAM. Rounds of the ordinary swap rule (the hottest CPU-lane pick of each layer
+ // in, its coldest VRAM expert out) until nothing is worth moving or DS4_SETTLE_MS (6000) is spent
+ void cache_settle() {
+ if (!o.expert_cache || cache_ths.empty()) return;
+ static const double budget = env_f("DS4_SETTLE_MS", 6000.0f);
+ if (budget <= 0.0) return;
+ const double t0 = now_ms();
+ const int64_t s0 = st.swaps;
+ while (now_ms() - t0 < budget) {
+ for (Layer& l : L) {
+ if (l.mtp || l.spare_vram < 0 || l.swap_pending) continue;
+ l.cpu_hits.clear();
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0 && l.heat[(size_t) e] >= 2.0f) l.cpu_hits.push_back(e);
+ }
+ cache_tick_swaps(0);
+ if (sw_inflight == 0) break; // (nothing queued: every layer's best CPU-lane expert is not hotter than its coldest VRAM one)
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+ // The RAM store's preload after a chunked prompt is off by default: measured on Strix Halo (code fixture, prompt +
+ // 256 tokens, 3 runs a side) it costs the prompt 4.7 s and the decode that follows gains nothing (13.9/14.4 s with it
+ // against 14.2 s without). The swaps above are what the decode actually uses; DS4_SETTLE_STORE=1 restores it.
+ if (store && env_f("DS4_SETTLE_STORE", strata::ds4::settle_store_default(defaults_026())) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU
+ // lane's picks, a chunked one streams past it - the hottest CPU-lane experts of the prompt into its free slots
+ // (copy_mode: an expert already in the CPU lane's RAM copy gains nothing from a second copy of the same bytes -
+ // measured on the R9700 12 GB cell: 4152 such experts, 40 GB read at 6.4 GB/s, 6.2 s of an 8K prompt, 0.0 store
+ // hits in the decode after it; the VRAM swaps above already read them from the copy)
+ std::vector<std::pair<float, std::pair<int, int>>> hot;
+ for (const Layer& l : L) {
+ if (l.mtp) continue;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0 && !(strata::ds4::settle_skips_copy_experts(defaults_026()) && copy_mode && !l.ram.empty() && l.ram[(size_t) e] >= 0) &&
+ l.heat[(size_t) e] >= 4.0f) hot.push_back({l.heat[(size_t) e], {l.il, e}});
+ }
+ std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ std::vector<std::pair<int, int>> list;
+ for (const auto& h : hot) {
+ if (list.size() >= store->slots_max()) break;
+ list.push_back(h.second);
+ }
+ const double ts = now_ms();
+ const size_t n = store->preload(list, 0);
+ if (o.verbose) std::fprintf(stderr, "ds4: RAM store: %zu hot experts read after the prompt (%.0f ms)\n", n, now_ms() - ts);
+ }
+ if (o.verbose && st.swaps > s0)
+ std::fprintf(stderr, "ds4: expert cache: %lld swaps after the prompt (%.0f ms)\n", (long long) (st.swaps - s0),
+ now_ms() - t0);
+ }
+ void cache_tick(int T) {
+ cache_tick_swaps(T);
+ store_sync();
+ store_watch();
+ }
+ // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time
+ // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free
+ void store_watch() {
+ if (!store) return;
+ static int tick = 0;
+ if (++tick % 16 != 0) return;
+ const uint64_t avail = ram_avail();
+ if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;
+ const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));
+ const size_t n = store->shrink_chunks(chunks);
+ if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());
+ }
+ void cache_tick_swaps(int T) {
+ if (!o.expert_cache || cache_ths.empty()) return;
+ std::vector<Swap> ready;
+ std::vector<RamSwap> rready;
+ {
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ready.swap(sw_ready);
+ rready.swap(ram_ready);
+ }
+ for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)
+ --ram_inflight;
+ Layer& l = L[(size_t) rs.layer];
+ if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {
+ ram_free.push_back(rs.slot);
+ continue;
+ }
+ l.ram[(size_t) rs.in] = rs.slot;
+ if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {
+ ram_free.push_back(l.ram[(size_t) rs.out]);
+ l.ram[(size_t) rs.out] = -1;
+ }
+ ++st.ram_moves;
+ }
+ for (const Swap& sw : ready) {
+ Layer& l = L[(size_t) sw.layer];
+ const int old_slot = l.slot[(size_t) sw.out];
+ const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];
+ if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)
+ const int s_in = l.ram[(size_t) sw.in];
+ l.ram[(size_t) sw.in] = -1;
+ if (sw.ram >= 0) {
+ l.ram[(size_t) sw.out] = sw.ram;
+ copy_spare[(size_t) l.il] = s_in;
+ } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)
+ --ram_held;
+ if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;
+ }
+ } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)
+ ram_free.push_back(l.ram[(size_t) sw.in]);
+ l.ram[(size_t) sw.in] = -1;
+ }
+ l.slot[(size_t) sw.in] = l.spare_vram;
+ if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)
+ l.zc[(size_t) sw.in] = 0;
+ if (l.zc_tab != nullptr) { // (the device's table too: its entry pointed into the slot the store now reuses - when `in` left VRAM later,
+ // zc_stage copied whatever lay there, another expert's blob, into the plan)
+ cudaSetDevice(devs[0].id);
+ cudaMemsetAsync(l.zc_tab + sw.in, 0, 8, devs[0].st);
+ }
+ store->unmark(l.il, sw.in);
+ }
+ if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);
+ l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.in] = nullptr;
+ l.slot[(size_t) sw.out] = -1;
+ if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)
+ l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);
+ } else if (file_experts) { // (back to its pages in the file)
+ l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};
+ l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];
+ } else {
+ l.hh[(size_t) sw.out] = l.spare_host;
+ l.host[(size_t) sw.out] = l.spare_host[0];
+ l.spare_host = old_parts;
+ }
+ l.spare_vram = old_slot;
+ for (int m = 0; m < l.nm; ++m) {
+ Layer& x = member(l, m);
+ const Dev& dm = devs[(size_t) x.dev];
+ l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));
+ l.tab_pin[2 * m + 1] = 0ull;
+ cudaSetDevice(dm.id); // (stream order: before the next window)
+ cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);
+ cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);
+ }
+ l.swap_pending = false;
+ --sw_inflight;
+ ++st.swaps;
+ }
+ decay_tokens += T;
+ if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens
+ decay_tokens = 0;
+ for (Layer& l : L)
+ for (float& h : l.heat) h *= c_decay;
+ }
+ bool queued = false, rqueued = false;
+ for (Layer& l : L) {
+ if (l.cpu_hits.empty()) continue;
+ std::vector<int> hits;
+ hits.swap(l.cpu_hits);
+ int best = -1, vin = -1;
+ for (int e : hits)
+ if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;
+ if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in
+ int cold = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;
+ if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {
+ l.swap_pending = true;
+ ++sw_inflight;
+ vin = best;
+ Swap sw{l.il, best, cold};
+ if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)
+ std::lock_guard<std::mutex> lk(sw_mu);
+ sw_queue.push_back(sw);
+ queued = true;
+ }
+ }
+ if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;
+ int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)
+ for (int e : hits)
+ if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&
+ (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))
+ rb = e;
+ if (rb < 0) continue;
+ int rc = -1;
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;
+ const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)
+ if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;
+ RamSwap rs;
+ rs.layer = l.il;
+ rs.in = rb;
+ rs.out = room ? -1 : rc;
+ rs.slot = ram_free.back();
+ ram_free.pop_back();
+ ++ram_inflight;
+ std::lock_guard<std::mutex> lk(sw_mu);
+ ram_queue.push_back(rs);
+ rqueued = true;
+ }
+ if (queued) sw_cv.notify_all();
+ if (rqueued) ram_cv.notify_one();
+ }
+
+ // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes
+ void prefetch_loop() {
+ std::vector<std::pair<const void*, size_t>> b;
+ for (;;) {
+ {
+ std::unique_lock<std::mutex> lk(pf_mu);
+ pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });
+ if (quit.load(std::memory_order_relaxed)) return;
+ b.swap(pf_q);
+ }
+ os::prefetch_cold(b); // (the resident ones would only cost the page walk)
+ b.clear();
+ }
+ }
+ // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)
+ static bool file_read(const Layer& l, int e) {
+ return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);
+ }
+ // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4
+ // experts read in place, some of them from the files now (none: nothing to read early - null)
+ const Layer* ahead_layer(const Layer& l0, int a) const {
+ const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;
+ if (j >= end || j >= (int) L.size()) return nullptr;
+ const Layer& n = L[(size_t) j];
+ if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||
+ (int) n.slot.size() < n.n_exp)
+ return nullptr;
+ for (int e = 0; e < n.n_exp; ++e)
+ if (file_read(n, e)) return &n;
+ return nullptr;
+ }
+ // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the
+ // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts
+ std::vector<std::pair<const void*, size_t>> pf_want;
+ void ahead_ask(int li, const Req& r, unsigned seq, int T) {
+ pf_want.clear();
+ for (int a = 1; a <= 2; ++a) {
+ const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;
+ if (m <= 0 || j >= (int) L.size()) continue;
+ const Layer& n = L[(size_t) j];
+ uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;
+ uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;
+ if (ahead_for[(size_t) j] != seq + (unsigned) a) {
+ std::memset(fl, 0, 1024);
+ std::memset(rk, 0, 1024);
+ ahead_for[(size_t) j] = seq + (unsigned) a;
+ }
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < m; ++i) {
+ const int e = r.ahead[t][(a - 1) * kAhead1 + i];
+ if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;
+ fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);
+ if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);
+ if ((fl[e] & 8) || !file_read(n, e)) continue;
+ fl[e] |= 8;
+ if (store) store->prefetch(j, e);
+ else
+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});
+ ++st.ahead_pf;
+ }
+ }
+ if (pf_want.empty()) return;
+ {
+ std::lock_guard<std::mutex> lk(pf_mu);
+ pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());
+ }
+ pf_cv.notify_one();
+ }
+
+ // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's
+ // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.
+ void serve_lane() {
+ unsigned next = 1;
+ auto last = std::chrono::steady_clock::now();
+ std::vector<Ent> ents;
+ while (!quit.load(std::memory_order_relaxed)) {
+ Req& r = reqs[next % kRing];
+ if (*(volatile unsigned*) &r.seq != next) {
+ cpu_relax();
+ if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))
+ std::this_thread::sleep_for(std::chrono::microseconds(20));
+ continue;
+ }
+ std::atomic_thread_fence(std::memory_order_acquire);
+ const int li = layer_of[next % kRing].load(std::memory_order_acquire);
+ Resp& rp = resps[next % kRing];
+ const int T = std::max(1, std::min(kWin, r.T));
+ for (int t = 0; t < T; ++t) rp.cpu[t] = 0;
+ if (li >= 0 && li < (int) L.size()) {
+ Layer& l = L[(size_t) li];
+ uint32_t* pr = prof.data() + (size_t) li * g.n_exp;
+ ents.clear();
+ if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)
+ // lookahead: the guesses made for this layer (when they are for this request), counted on decode
+ const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next
+ ? ahead_fl.data() + (size_t) li * 1024 : nullptr;
+ // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row
+ float wv[kWin * kPicks];
+ int tok_of[kWin * kPicks];
+ int nslots = 0;
+ std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only
+ if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids
+ // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the
+ // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's
+ // predecessors (the request carries the guesses for the layers after it)
+ static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;
+ uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};
+ std::fwrite(hd, 1, 2, trace_f);
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int16_t v = (int16_t) r.ids[t][i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ if (with_ahead) {
+ const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};
+ std::fwrite(na, 1, 2, trace_f);
+ for (int t = 0; t < T; ++t) {
+ for (int i = 0; i < na[0]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ for (int i = 0; i < na[1]; ++i) {
+ const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
+ }
+ }
+ const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;
+ for (int t = 0; t < T; ++t) {
+ float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)
+ if (dsk)
+ for (int i = 0; i < l.k_exp; ++i)
+ if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = r.ids[t][i];
+ if (e < 0 || e >= l.n_exp) continue;
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (fl != nullptr) {
+ ++st.ahead_picks;
+ st.ahead_hit_k += fl[e] & 1;
+ st.ahead_hit += (fl[e] & 6) != 0;
+ st.ahead_hit2 += (fl[e] >> 2) & 1;
+ }
+ if (l.slot[(size_t) e] < 0) {
+ l.cpu_hits.push_back(e);
+ if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot
+ store->touch(li, e);
+ ++st.store_zc;
+ continue;
+ }
+ if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {
+ ++st.skipped;
+ continue;
+ }
+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;
+ else if (fl != nullptr) {
+ ++st.ahead_cold;
+ st.ahead_cold_hit += (fl[e] >> 3) & 1;
+ }
+ picks.push_back({e, t * kPicks + i});
+ }
+ }
+ }
+ if (fl != nullptr) { // the guesses by rank: how many were picked
+ const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;
+ for (int e = 0; e < l.n_exp && e < 1024; ++e) {
+ if (rk[e] == 0 || rk[e] > 16) continue;
+ bool hit = false;
+ for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;
+ ++st.ahead_rank_n[rk[e] - 1];
+ st.ahead_rank_hit[rk[e] - 1] += hit;
+ }
+ }
+ if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);
+ std::sort(picks.begin(), picks.end());
+ for (size_t a = 0; a < picks.size();) {
+ Ent en;
+ en.e = picks[a].first;
+ en.slot0 = nslots;
+ en.n = 0;
+ while (a < picks.size() && picks[a].first == en.e && en.n < ent_cap(l)) {
+ const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;
+ en.tok[en.n++] = t;
+ tok_of[nslots] = t;
+ wv[nslots] = r.w[t][i];
+ ++nslots;
+ ++a;
+ }
+ ents.push_back(en);
+ }
+ // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for
+ // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)
+ int ord[kWin * kPicks];
+ const int* po = nullptr;
+ if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&
+ ents.size() <= (size_t) (kWin * kPicks)) {
+ const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;
+ int n = 0;
+ for (int pass = 0; pass < 2; ++pass)
+ for (size_t q = 0; q < ents.size(); ++q) {
+ const int e = ents[q].e;
+ const bool early = !file_read(l, e) || (fa[e] & 8) != 0;
+ if (early == (pass == 0)) ord[n++] = (int) q;
+ }
+ po = ord;
+ }
+ if (!ents.empty()) {
+ const double tc = now_ms();
+ cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) {
+ bool any = false;
+ for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;
+ if (!any) continue;
+ float* o = rp.part[t];
+ for (int e = 0; e < D; ++e) o[e] = 0.0f;
+ for (int sl = 0; sl < nslots; ++sl) {
+ if (tok_of[sl] != t) continue;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];
+ }
+ rp.cpu[t] = 1;
+ }
+ st.cpu_ms += now_ms() - tc;
+ st.cpu_experts += nslots;
+ }
+ }
+ std::atomic_thread_fence(std::memory_order_release);
+ *(volatile unsigned*) &rp.seq = next;
+ ++next;
+ last = std::chrono::steady_clock::now();
+ }
+ }
+
+ // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's
+ static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {
+ bool g = false;
+ for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;
+ return g ? gg::mv(j, n, s) : x3::mv(j, n, s);
+ }
+ bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {
+ x3::MvJob j[x3::kMaxMvJobs];
+ int n = 0;
+ for (const x3::MvJob& x : jobs) j[n++] = x;
+ if (!mv_any(j, n, d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {
+ for (int a = 0; a < n; a += x3::kMaxMvJobs)
+ if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {
+ err = "exl3 mv failed";
+ return false;
+ }
+ return true;
+ }
+ // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs
+ static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,
+ float alpha = 1.0f) {
+ x3::MvJob j;
+ j.m = &m;
+ j.xf = x;
+ j.y = y;
+ j.alpha = alpha;
+ j.ncol = T;
+ j.xf_cs = x_cs;
+ j.y_cs = y_cs;
+ return j;
+ }
+
+ // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:
+ // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm
+ // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x
+ // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),
+ // the trunk's head, the markov bias token by token (greedy) and the confidence head.
+ bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {
+ if (g.n_mtp == 0 || n <= 0) return true;
+ if (n > kWin * kPicks) {
+ err = "seeding the drafter with " + std::to_string(n) + " rows";
+ return false;
+ }
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;
+ if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps
+ for (Dev& e : devs) {
+ cudaSetDevice(e.id);
+ CK(cudaStreamSynchronize(e.st));
+ }
+ cudaSetDevice(d.id);
+ const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);
+ std::fflush(dump_d);
+ }
+ for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)
+ const int na = std::min(kWin, n - a);
+ CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,
+ cudaMemcpyHostToDevice, d.st));
+ if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;
+ }
+ CK(cudaEventRecord(ev_seed, d.st));
+ Finish fm;
+ fm.w = d_main_norm;
+ fm.eps = g.eps;
+ finish(d.ed, n, D, D, fm, d.st);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (l.dev != d_dev) {
+ err = "the DSpark blocks are split across devices";
+ return false;
+ }
+ for (int a = 0; a < n; a += kWin) {
+ const int na = std::min(kWin, n - a);
+ if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) (p0 + a);
+ fk.freqs = d.freqs_win;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, na, HD, HD, fk, d.st);
+ ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);
+ }
+ }
+ return true;
+ }
+
+ bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,
+ int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ const double t0 = now_ms();
+ drafts_q = temperature > 0.0f && sample_drafts;
+ dq_temp = temperature;
+ dq_top_p = top_p;
+ dq_top_k = top_k;
+ dq_min_p = min_p;
+ dq_seed = seed;
+ dq_counter = counter;
+ const bool ok = draft_(y, out, conf, err);
+ if (!ok) drafts_q = false;
+ if (max_drafts > 0 && (int) out.size() > max_drafts) {
+ out.resize((size_t) max_drafts);
+ conf.resize((size_t) max_drafts);
+ }
+ st.draft_ms += now_ms() - t0;
+ ++st.drafts_rounds;
+ return ok;
+ }
+ bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {
+ out.clear();
+ conf.clear();
+ const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;
+ const int64_t p0 = pos;
+ if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;
+ int tk[kWin];
+ tk[0] = y;
+ for (int t = 1; t < T; ++t) tk[t] = g.d_noise;
+ Dev& d = devs[(size_t) d_dev];
+ cudaSetDevice(d.id);
+ for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float *hp = d.h;
++ float hh[4] = {0,0,0,0};
++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
++ }
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float hh[4] = {0, 0, 0, 0};
++ cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);
++ }
+ init_pre(d.pre, T, d.st);
+ WinPlan dp; // (window attention alone: the mailbox seqs)
+ dp.async = !sync_decode && T <= async_max_t;
+ for (int i = 0; i < g.n_mtp && dp.async; ++i) {
+ const int il = g.nl + i;
+ dp.seq[il] = ++seq_issued;
+ layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;
+ if (dump_d != nullptr && h_dump == nullptr &&
+ (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))
+ return false;
+ for (int i = 0; i < g.n_mtp; ++i) {
+ Layer& l = L[(size_t) (g.nl + i)];
+ if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)
+ ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);
+ CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));
+ }
+ if (!layer(l, 0, p0, T, dp, err)) return false;
+ if (dump_d != nullptr) {
+ cudaSetDevice(d.id);
+ CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));
+ }
+ }
+ cudaSetDevice(d.id);
+ hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)
+ Dev& hd = devs[(size_t) head_dev];
+ // (through the host to the head's device, and to the head group's other members - the drafter's device can be
+ // the head group's lead itself, a group of every GPU)
+ if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {
+ CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(d.ev_out, d.st));
+ }
+ if (d_dev != head_dev) {
+ cudaSetDevice(hd.id);
+ CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));
+ }
+ // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)
+ const std::vector<int>& hg = groups[(size_t) hd.grp].mem;
+ const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)
+ for (int m = (int) hg.size() - 1; m >= 0; --m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ if (m > 0) {
+ CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));
+ CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));
+ } else
+ CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));
+ Finish fn;
+ fn.w = e.d_norm;
+ fn.eps = g.eps;
+ finish(e.ao, T, D, D, fn, e.st);
+ if (!head_share(e, m, e.ao, T, err, !mk)) return false;
+ }
+ h_idx[0] = y;
+ if (mk) {
+ // the markov steps over the group: each member its rows of the bias and its share's argmax, every member
+ // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,
+ // each step ~1/nm of their 66 MB and no gather of the shares
+ const int nm = (int) hg.size();
+ const unsigned seq = ++mk_seq;
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));
+ CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));
+ }
+ for (int i = 0; i < T; ++i)
+ for (int m = 0; m < nm; ++m) {
+ Dev& e = devs[(size_t) hg[(size_t) m]];
+ cudaSetDevice(e.id);
+ float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;
+ markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);
+ argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);
+ markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,
+ e.st);
+ }
+ cudaSetDevice(hd.id);
+ } else {
+ CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)
+ sp.greedy = false;
+ sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);
+ sp.top_p = dq_top_p;
+ sp.top_k = dq_top_k;
+ sp.min_p = dq_min_p;
+ sp.seed = dq_seed;
+ }
+ for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft
+ if (d_mk_head_q8 != nullptr) markov_add_q8(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head_q8, d_ints + i, V, R, hd.st);
+ else markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);
+ if (drafts_q) {
+ sp.counter = dq_counter + (uint64_t) i;
+ strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);
+ } else
+ strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);
+ }
+ }
+ confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);
+ int tok[kWin + 1];
+ float cf[kWin];
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];
+ for (int i = 0; i < T; ++i) cf[i] = h_w[i];
+ if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden
+ std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream
+ CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));
+ cudaSetDevice(d.id);
+ CK(cudaStreamSynchronize(d.st));
+ const int64_t hdr[4] = {'D', p0, y, T};
+ std::fwrite(hdr, 8, 4, dump_d);
+ std::fwrite(tok + 1, 4, (size_t) T, dump_d);
+ std::fwrite(cf, 4, (size_t) T, dump_d);
+ std::fwrite(xs.data(), 4, xs.size(), dump_d);
+ for (int i = 0; i < g.n_mtp; ++i) {
+ std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);
+ std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);
+ }
+ std::fflush(dump_d);
+ cudaSetDevice(hd.id);
+ }
+ for (int i = 0; i < T; ++i) {
+ out.push_back(tok[i + 1]);
+ conf.push_back(cf[i]);
+ }
+ return true;
+ }
+
+ int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,
+ const float* bias) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::dsv41k::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ sp.logit_bias = bias;
+ strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);
+ int t = -1;
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.
+ // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own
+ // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution
+ // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the
+ // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.
+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,
+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {
+ const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);
+ const int T = n + 1, V = g.vocab;
+ const int64_t p0 = pos;
+ int tk[kWin];
+ tk[0] = y;
+ for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];
+ spec_win = true;
+ const double tv = now_ms();
+ const bool ok = forward(tk, T, T, err, 0);
+ st.verify_ms += now_ms() - tv;
+ spec_win = false;
+ if (!ok) return false;
+ Dev& hd = devs[(size_t) head_dev];
+ cudaSetDevice(hd.id);
+ int m = 0;
+ if (temperature <= 0.0f) {
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = true;
+ sp.temperature = 0.0f;
+ strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = h_idx[m];
+ } else {
+ // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)
+ // (never d itself): each row a token distributed as the model's own draw at that position, p through the
+ // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,
+ // never kept); past the first rejection the rows do not count
+ const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&
+ dq_min_p == min_p && dq_seed == seed && dq_counter == counter;
+ if (!own) { // the drafter's argmax: q a point at each draft
+ for (int i = 0; i < n; ++i) {
+ h_specq[(size_t) i].n = 1;
+ h_specq[(size_t) i].ids[0] = drafts[(size_t) i];
+ h_specq[(size_t) i].p[0] = 1.0;
+ }
+ CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),
+ cudaMemcpyHostToDevice, hd.st));
+ }
+ for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];
+ CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));
+ strata::dsv41k::SamplerParams sp;
+ sp.greedy = false;
+ sp.temperature = temperature;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));
+ CK(cudaStreamSynchronize(hd.st));
+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;
+ next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);
+ }
+ drafts_q = false;
+ rollback(p0, T, m + 1);
+ if (!seed_drafter(p0, m + 1, 0, err)) return false;
+ accepted = m;
+ st.drafts_offered += n;
+ st.drafts_accepted += m;
+ for (int i = 0; i < n && i <= m && i < 8; ++i) {
+ ++st.draft_reached[i];
+ if (i < m) ++st.draft_kept[i];
+ }
+ return true;
+ }
+
+ // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone
+ // (the compressed KV and the index keys past them are written again when those positions come)
+ void rollback(int64_t p0, int T, int keep) {
+ if (keep >= T) return;
+ // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor
+ // parallel, a millisecond of the host's before the next draft)
+ std::vector<RingBatch> rb(devs.size());
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ RingBatch& b = rb[(size_t) l.dev];
+ if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)
+ cudaSetDevice(devs[(size_t) l.dev].id);
+ ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);
+ continue;
+ }
+ b.ring[b.n] = l.win;
+ b.bak[b.n++] = l.ring_bak;
+ }
+ for (size_t di = 0; di < devs.size(); ++di)
+ if (rb[di].n > 0) {
+ cudaSetDevice(devs[di].id);
+ ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);
+ }
+ for (int il = 0; il < g.nl; ++il)
+ for (int m = 0; m < L[(size_t) il].nm; ++m) {
+ Layer& l = member(L[(size_t) il], m);
+ Dev& d = devs[(size_t) l.dev];
+ if (l.c_snap != nullptr) {
+ cudaSetDevice(d.id);
+ const size_t ns = (size_t) l.ratio * g.hd * 4;
+ cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);
+ cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);
+ compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,
+ g.hd, l.c_norm, g.eps, d.latent, d.st);
+ }
+ }
+ pos = p0 + keep;
+ toks.resize((size_t) pos);
+ ctoks.resize((size_t) pos);
+ slot_held = false;
+ }
+
+ // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows
+ bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }
+
+
+ // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the
+ // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in
+ // place, the CPU lane's streamed in batches). kv_only (a long prompt's first pass, Run::stop): only what the layers
+ // after it read of these rows - its compressed KV and index keys - and nothing of its own attention or experts
+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,
+ bool kv_only = false) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ Chunk& c = *chunks[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ prof_mark(d, 0);
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ab = groups[(size_t) L0.grp].arb;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ // Engram, in slabs of 256 tokens
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)
+ uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;
+ CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.eg_ev[slot], s));
+ for (int s0 = 0; s0 < T; s0 += 256) {
+ const int ns = std::min(256, T - s0);
+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
+ x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);
+ if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;
+ engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);
+ }
+ }
+ prof_mark(d, 4);
+ // attention
- if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;
- hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);
++ // the mix and the split in one dispatch (hc_mix + hc_split were two, back to back on the same token)
++ hc_mix_split(c.h, l.hca_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hca_scale, l.hca_base, H,
++ g.sinkhorn, g.hc_eps, c.a_pre, c.a_post, c.a_comb, s);
++ dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
+ hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);
++ dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);
++ dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);
++ dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);
++ dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);
++ dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ prof_mark(d, 5);
+ if (!kv_only && (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||
+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err)))
+ return false;
+ if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;
+ if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;
+ prof_mark(d, 6);
+ const bool snap = !kv_only && wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;
+ if (!kv_only) {
++ dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);
++ dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);
+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);
+ // the window rows: the ring's (positions p0 - win ..) then the chunk's
+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = c.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(c.kvr, T, HD, HD, fk, s);
+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it
+ if (snap) {
+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));
+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);
+ }
+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);
+ }
+ const uint8_t* ckv = nullptr;
+ const long long ss = (long long) o.max_context + 64;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ const int64_t g_first = p0 / R, g_end = (p0 + T) / R;
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ const float* latent = c.latent;
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(c.ckr, T, HD, HD, fc, s);
+ latent = c.ckr;
+ } else {
+ if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)
+ const int64_t ps = p0 + wp.snap_t;
+ for (int sl = 0; sl < R; ++sl) {
+ const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;
+ const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;
+ const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;
+ CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));
+ }
+ }
+ compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);
+ }
+ }
+ const int clen_max = (int) ((p0 + T) / R);
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);
+ if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ }
+ if (clen_max > 0 && !kv_only) {
+ if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))
+ return false;
+ for (int a0 = 0; a0 < T; a0 += c.AI) {
+ const int na = std::min(c.AI, T - a0);
+ const int64_t pa = p0 + a0;
+ if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,
+ g.ix_heads * g.ix_hd, 0.0f, err))
+ return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) pa;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ Rows lim;
+ lim.gen = 1;
+ lim.p0 = pa;
+ lim.R = R;
+ lim.kcap = g.ix_topk;
+ lim.ccap = g.cand_topk;
+ lim.blk = std::max(1, g.cand_blk);
+ const int nlast = (int) ((pa + na) / R); // the slab's widest query
+ int* tk = c.topk + (size_t) a0 * topk_stride();
+ int* cd = c.cand + (size_t) a0 * cand_stride();
+ const float* iw = c.iw + (size_t) a0 * 64;
+ if (nlast <= 0) continue;
+ if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {
+ Rows cl = lim;
+ cl.gen = 2;
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,
+ cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);
+ Rows tr = lim;
+ tr.gen = 3;
+ topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);
+ } else {
+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,
+ lim, c.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr = lim;
+ cr.kcap = g.cand_topk;
+ cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,
+ (int) cand_stride(), s);
+ }
+ topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);
+ }
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(const_cast<float*>(latent), ng, HD, HD, fc, s);
+ if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once
+ const int slot = wp.k % 2; // they took the chunk before last's)
+ if (wp.k >= 2)
+ for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.stage(slot);
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);
+ }
+ }
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ if (kv_only) return true;
+ prof_mark(d, 7);
+ // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc
+ if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)
+ for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {
+ const int na = std::min(c.A, T - a0);
+ if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) (p0 + a0);
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(c.q, na * NHL, HD, HD, fr, s);
+ prof_mark(d, 8);
++ if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);
+ AttnRows ar;
+ ar.gen = 1;
+ ar.p0 = p0;
+ ar.t0 = a0;
+ ar.win = g.win;
+ ar.R = l.ratio;
+ ar.kcap = g.ix_topk;
+ sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,
+ na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)
+ prof_mark(d, 9);
++ if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) (p0 + a0);
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(c.o, na * NHL, HD, HD, fo, s);
++ if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);
+ x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,
+ OG * g.o_lora, 0.0f, err))
+ return false;
+ x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);
+ if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;
++ if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);
+ }
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
++ dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);
+ hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);
++ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)
+ prof_mark(d, 1);
+ // the MoE
- if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;
- hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);
++ // the FFN half of the same fusion: mix and split in one dispatch
++ hc_mix_split(c.h, l.hcf_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hcf_scale, l.hcf_base, H,
++ g.sinkhorn, g.hc_eps, c.f_pre, c.f_post, c.f_comb, s);
+ hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);
+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);
+ gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)
+ for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)
+ const int nr = std::min(kVlRows, T - r0);
+ VlBits vb;
+ const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);
+ route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,
+ c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);
+ }
+ const int n_vs = L0.n_slots + (int) L0.host_list.size();
+ CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)
+ moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);
+ CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));
+ prof_mark(d, 10);
+ // the shared expert into acc while the host waits for the bounds
+ if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))
+ return false;
+ swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);
+ x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);
+ if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;
+ prof_mark(d, 2);
+ // the expert stream's copies keep going while the device runs this layer's attention, routing and shared expert:
+ // the next items (this layer's likely experts, queued when the layer before routed) go out as they are ready and
+ // ring slots free, not only once the bounds are back (Project Maya #360; DS4_FLOW_PUMP_WAIT=0: wait as before)
+ static const bool pump_wait = std::getenv("DS4_FLOW_PUMP_WAIT") == nullptr ||
+ std::atoi(std::getenv("DS4_FLOW_PUMP_WAIT")) != 0;
+ if (c.flow && pump_wait)
+ while (cudaStreamQuery(s) == cudaErrorNotReady) {
+ if (!flow_pump(c, err)) return false;
+ std::this_thread::yield();
+ }
+ CK(cudaStreamSynchronize(s));
+ std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);
+ if (dump_r != nullptr && mi == 0) {
+ const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;
+ std::vector<int> ri(np), rt(nk);
+ std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);
+ CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
+ cudaMemcpyDeviceToHost));
+ CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};
+ std::fwrite(hdr, 8, 8, dump_r);
+ std::fwrite(ri.data(), 4, np, dump_r);
+ std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);
+ std::fwrite(b.data(), 4, b.size(), dump_r);
+ std::fwrite(rt.data(), 4, nk, dump_r);
+ std::fwrite(gl.data(), 4, gl.size(), dump_r);
+ std::fwrite(gb.data(), 4, gb.size(), dump_r);
+ std::fflush(dump_r);
+ }
+ if (mi == 0 && !l.mtp && (pred_f != nullptr || xin_f != nullptr)) {
+ std::lock_guard<std::mutex> lk(dump_mu);
+ if (pred_f != nullptr) {
+ float* dp = nullptr;
+ CK(cudaMalloc(&dp, (size_t) T * 256 * 4));
- pred_proj_k<<<T, 256, 0, s>>>(c.x, (int) D, dp);
++ strata_ds4_launch(pred_proj_k, T, 256, 0, s, c.x, (int) D, dp);
+ std::vector<float> hp((size_t) T * 256), hl((size_t) T * l.n_exp);
+ CK(cudaMemcpyAsync(hp.data(), dp, hp.size() * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpy2DAsync(hl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,
+ cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ CK(cudaFree(dp));
+ std::vector<uint16_t> o(hp.size() + hl.size());
+ for (size_t i = 0; i < hl.size(); ++i) o[i] = strata::kernels::f16_from_f32(hl[i]);
+ for (size_t i = 0; i < hp.size(); ++i) o[hl.size() + i] = strata::kernels::f16_from_f32(hp[i]);
+ const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (T & 255), (uint8_t) (T >> 8)};
+ std::fwrite(hd, 1, 3, pred_f);
+ std::fwrite(o.data(), 2, o.size(), pred_f);
+ }
+ if (xin_f != nullptr && xin_got[(size_t) l.il] < xin_n) { // up to 32 evenly spaced tokens of the chunk
+ const int tk = std::min(std::min(T, 32), xin_n - xin_got[(size_t) l.il]);
+ const int stride = std::max(1, T / tk);
+ std::vector<float> hx((size_t) T * D);
+ CK(cudaMemcpyAsync(hx.data(), c.x, hx.size() * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (tk & 255), (uint8_t) (tk >> 8)};
+ std::fwrite(hd, 1, 3, xin_f);
+ for (int i = 0; i < tk; ++i) std::fwrite(hx.data() + (size_t) std::min(T - 1, i * stride) * D, 4, (size_t) D, xin_f);
+ xin_got[(size_t) l.il] += tk;
+ }
+ }
+ if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids
+ std::vector<int> ri((size_t) T * kPicks);
+ CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));
+ std::lock_guard<std::mutex> lk(dump_mu);
+ const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};
+ std::fwrite(hd, 1, 3, trace_f);
+ for (int t = 0; t < T; ++t)
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];
+ std::fwrite(&v, 2, 1, trace_f);
+ }
+ }
+ if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const int v = L0.vs_pin[e];
+ const int cnt = b[(size_t) v + 1] - b[(size_t) v];
+ pr[e] += (uint32_t) cnt;
+ if (cnt > 0) { // (the expert cache learns from a chunked prompt as from windows: heat, and the CPU lane's hits)
+ l.heat[(size_t) e] += (float) cnt;
+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
+ }
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ }
+ if (c.flow) {
+ if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+ // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones
+ // in shares between them (the device works while the copies run)
+ const int di = l.dev, P = Chunk::kParts;
+ int nb = 0;
+ for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;
+ int v_done = 0;
+ for (int bi = 0; bi < nb; ++bi) {
+ const Batch& bt = S.b[S.kp];
+ const int half = (int) (S.kp % P);
+ CK(cudaStreamWaitEvent(s, c.filled[half], 0));
+ const int v0 = L0.n_slots + bt.j0;
+ if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,
+ b.data(), v0, v0 + bt.n, T, err))
+ return false;
+ CK(cudaEventRecord(c.freed[half], s));
+ ++S.kp;
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;
+ v_done = v_to;
+ }
+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;
+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);
+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);
+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 3);
+ return true;
+ }
+
+ // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)
+ void plan_streams() {
+ for (Layer& l : L) {
+ if (l.mtp || l.vs_pin == nullptr) continue;
+ l.host_list.clear();
+ for (int e = 0; e < l.n_exp; ++e)
+ if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);
+ for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)
+ Layer& x = member(l, m);
+ int j = 0;
+ for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;
+ }
+ }
+ }
+
+ // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows
+ Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {c.h, c.pre, c.topk, c.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the
+ // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows
+ // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across
+ // layers and chunks
+ struct Run {
+ std::vector<int64_t> p0;
+ std::vector<int> T;
+ std::vector<unsigned> seq;
+ int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)
+ // a long prompt's skip (forward_chunks): the cut at row cut_r of chunk cut_k - the chunks before it go through
+ // the layers before `yoco`, that layer as layer_chunk's kv_only and none after it; chunk cut_k's rows before
+ // cut_r the same, its rows from cut_r on and the chunks after it through every layer (yoco -1: no skip)
+ int yoco = -1, cut_k = 0, cut_r = 0;
+ bool runs(int k, int il) const { return yoco < 0 || k >= cut_k || il <= yoco; }
+ bool kv_only(int k, int il) const { return yoco >= 0 && k < cut_k && il == yoco; }
+ int r0(int k, int il) const { return yoco >= 0 && k == cut_k && il >= yoco ? cut_r : 0; } // the first row
+ };
+ // layer L0 over chunk k per a long prompt's skip (Run): its kv_only, or its rows from Run::r0 on - the chunk's
+ // buffers seen from that row (the yoco layer: its kv_only over the rows before it first)
+ bool run_layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,
+ const Run& run, int k, int il) {
+ if (run.kv_only(k, il)) return layer_chunk(L0, mi, p0, T, wp, S, err, true);
+ const int r0 = run.r0(k, il);
+ if (r0 <= 0) return layer_chunk(L0, mi, p0, T, wp, S, err);
+ if (il == run.yoco) {
+ WinPlan w0 = wp;
+ w0.snap_t = -1;
+ if (!layer_chunk(L0, mi, p0, r0, w0, S, err, true)) return false;
+ }
+ Chunk& c = *chunks[(size_t) member(L0, mi).dev];
+ float* const h0 = c.h;
+ float* const pre0 = c.pre;
+ c.h += (size_t) r0 * g.hc * g.dim;
+ c.pre += (size_t) r0 * 8;
+ WinPlan w1 = wp;
+ w1.snap_t = wp.snap_t >= r0 ? wp.snap_t - r0 : -1;
+ const bool ok = layer_chunk(L0, mi, p0 + r0, T - r0, w1, S, err);
+ c.h = h0;
+ c.pre = pre0;
+ return ok;
+ }
+
+ // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):
+ // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its
+ // layers and back before the next - so a layer's experts are read from the files once for the whole run (the
+ // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk
+ bool run_dev_layers(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S; // (none: the flow streams)
+ const size_t hb = chunk_hand_bytes();
+ // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while
+ // the one before runs (slot u % 2, once unit u - 2's copy of it is done)
+ std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)
+ for (int il : c.fl_layers)
+ if (L[(size_t) il].eg >= 0)
+ for (int k = 0; k < nk; ++k)
+ if (run.runs(k, il)) egu.push_back({L[(size_t) il].eg, k});
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {
+ if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);
+ const int li = egu[u].first, k = egu[u].second;
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);
+ };
+ size_t eg_u = 0;
+ std::future<void> eg_next;
+ if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);
+ for (size_t li = 0; li < c.fl_layers.size(); ++li) {
+ const int il = c.fl_layers[li];
+ Layer& L0 = L[(size_t) il];
+ const bool last_l = li + 1 == c.fl_layers.size();
+ for (int k = 0; k < nk; ++k) {
+ if (!run.runs(k, il)) continue; // (a skipped early chunk: nothing after the yoco layer)
+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way
+ const double te = now_ms();
+ eg_next.get();
+ st.engram_ms += now_ms() - te;
+ wp.eg_slot = (int) (eg_u % 2);
+ if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);
+ }
+ if (li == 0) { // the chunk's input: its embedding rows
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float *hp = c.h;
++ float hh[4] = {0,0,0,0};
++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
++ }
+ init_pre(c.pre, T, s);
+ } else { // or its state after the layer before
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));
+ }
+ if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ const bool kvo = run.kv_only(k, il);
+ if (!run_layer(L0, 0, p0, T, wp, S, err, run, k, il)) return false;
+ if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);
+ if (kvo) continue; // (no layer after it reads this chunk's state)
+ if (!last_l) { // the chunk's state kept for the next layer
+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);
+ for (int i = 0; i < x.n; ++i)
+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));
+ continue;
+ }
+ // through every layer: the chunk's end, and the logits
+ if ((size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ lm_layers.store((int) li + 1, std::memory_order_release);
+ }
+ return true;
+ }
+ bool run_dev_chunks(int di, const Run& run, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ Chunk& c = *chunks[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();
+ Streamer S;
+ const int B = std::max(1, c.ring_n / Chunk::kParts);
+ if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)
+ CK(cudaStreamSynchronize(s));
+ CK(cudaStreamSynchronize(c.cs));
+ c.items.clear();
+ c.base = c.n_read = c.n_dma = c.n_comp = 0;
+ c.n_file = 0;
+ std::fill(c.stage_item.begin(), c.stage_item.end(), -1);
+ c.fl_layers.clear();
+ for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)
+ for (int m = 0; m < L[(size_t) il].nm; ++m)
+ if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);
+ c.fl_T = run.T;
+ c.fl_yoco = run.yoco;
+ c.fl_cut_k = run.cut_k;
+ c.fl_cut_r = run.cut_r;
+ c.spec_k = c.spec_il = -1;
+ c.wait_ms = 0;
+ c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;
+ c.lc_il = -1;
+ c.lc_used = 0;
+ c.lc_first = c.lc_chk = 0;
+ c.io_d0 = c.io->direct_bytes();
+ c.io_b0 = c.io->buffered_bytes();
+ c.io_busy0 = c.io->busy_ms();
+ if (!c.fl_layers.empty()) {
+ flow_spec(c, 0, c.fl_layers[0], run.T[0]);
+ if (!flow_pump(c, err)) return false;
+ }
+ if (c.lm) return run_dev_layers(di, run, err);
+ }
+ for (int k = 0; k < nk && !c.flow; ++k)
+ for (int il = 0; il < g.nl; ++il) {
+ if (!run.runs(k, il) || run.kv_only(k, il)) continue; // (a long prompt's skip: no experts there)
+ const Layer& L0 = L[(size_t) il];
+ bool here = false;
+ for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;
+ if (!here) continue;
+ for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)
+ S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});
+ }
+ if (!S.b.empty() && c.ring == nullptr) {
+ err = "no slots to stream experts into on CUDA" + std::to_string(d.id);
+ return false;
+ }
+ std::atomic<bool> ra_stop{false};
+ std::thread ra;
+ if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });
+ struct Join { // (every way out of the run: the read-ahead ends before S does)
+ std::atomic<bool>& stop;
+ std::thread& th;
+ ~Join() {
+ stop.store(true);
+ if (th.joinable()) th.join();
+ }
+ } ra_join{ra_stop, ra};
+ if (!copy_ahead(d, c, S, di, err)) return false;
+ // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once
+ // chunk k - 2's copy of it is done)
+ std::vector<int> egs;
+ for (const Layer& l0 : L)
+ for (int m = 0; m < l0.nm; ++m)
+ if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);
+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);
+ auto read_eg = [this, &c, &run, &egs, eg_per](int k) {
+ if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);
+ for (int li : egs)
+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);
+ };
+ std::future<void> eg_next;
+ if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);
+ for (int k = 0; k < nk; ++k) {
+ if (!chunk_go(k)) return halt_end(di, err); // (a Stop: the chunks every device reaches are done)
+ const int64_t p0 = run.p0[(size_t) k];
+ const int T = run.T[(size_t) k], slot = k % 2;
+ const bool last_k = k + 1 == nk;
+ if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way
+ const double te = now_ms();
+ eg_next.get();
+ if (d.mi == 0) st.engram_ms += now_ms() - te;
+ if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);
+ }
+ WinPlan wp;
+ wp.par = true;
+ wp.async = true;
+ wp.win = run.seq[(size_t) k];
+ wp.k = k;
+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl && run.runs(k, il); ++il) {
+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),
+ (size_t) D * 2);
+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
+ CK(cudaEventRecord(c.emb_ev[slot], s));
+ embed(c.emb, T, c.h, D, H, s);
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float *hp = c.h;
++ float hh[4] = {0,0,0,0};
++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
++ }
+ init_pre(c.pre, T, s);
+ } else {
+ Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);
+ set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);
+ }
+ }
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ Xfer x;
+ if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot);
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (ng > 0 && sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);
+ set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)
+ }
+ if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (it != g.d_targets.end()) {
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int nr = std::min(T, tap_cap);
+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);
+ CK(cudaStreamWaitEvent(s, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,
+ (size_t) nr, cudaMemcpyDeviceToHost, s));
+ }
+ }
+ if (!run_layer(L0, mi, p0, T, wp, S, err, run, k, il)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last
+ const int nnext = L[(size_t) il + 1].nm;
+ if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);
+ xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);
+ }
+ }
+ if (di == head_dev && (size_t) k < chunk_done.size()) {
+ CK(cudaEventRecord(chunk_done[(size_t) k], s));
+ chunk_rec[(size_t) k].store(1, std::memory_order_release);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits
+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time
+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;
+ for (int r0 = 0; r0 < T; r0 += kWin) {
+ const int nr = std::min(kWin, T - r0);
+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, nr, err)) return false;
+ if (di != head_dev) continue;
+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,
+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaStreamSynchronize(s));
+ }
+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)
+ const int t0 = T - n_logits;
+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);
+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ // ---- a prompt's path: the flow or windows of 16
+ // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a
+ // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in
+ // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the
+ // flow, as before)
+ void calibrate_paths() {
+ if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;
+ if (const char* v = std::getenv("DS4_PATH_PICK"))
+ if (std::atoi(v) == 0) return;
+ int di = -1;
+ for (size_t i = 0; i < chunks.size() && di < 0; ++i)
+ if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;
+ if (di < 0) return;
+ Chunk& c = *chunks[(size_t) di];
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const double t0 = now_ms();
+ // the DMA: staging -> the ring (once to warm up, once timed)
+ const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});
+ double ms = 0;
+ for (int rep = 0; rep < 2; ++rep) {
+ const double a = now_ms();
+ if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||
+ cudaStreamSynchronize(c.cs) != cudaSuccess) {
+ cudaGetLastError();
+ return;
+ }
+ ms = now_ms() - a;
+ }
+ pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);
+ // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are
+ auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {
+ const double a = now_ms();
+ left.store((int) jobs.size());
+ c.io->push(jobs.data(), (int) jobs.size());
+ while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ return now_ms() - a;
+ };
+ {
+ const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;
+ std::vector<uint8_t> src(part * 8, 1);
+ std::vector<IoQueue::Job> jobs(8);
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < 8; ++i) {
+ jobs[(size_t) i].mem = src.data() + (size_t) i * part;
+ jobs[(size_t) i].dst = c.stage + (size_t) i * part;
+ jobs[(size_t) i].len = part;
+ jobs[(size_t) i].left = &left;
+ jobs[(size_t) i].bad = &bad;
+ }
+ run_jobs(jobs, left);
+ const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);
+ pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);
+ }
+ // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:
+ // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane
+ // three times (from the disk, then in RAM: the faster of two)
+ int best = -1, most = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int n = 0;
+ for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;
+ if (n > most) most = n, best = il;
+ }
+ if (best < 0) return;
+ const Layer& l = L[(size_t) best];
+ std::vector<int> cold;
+ const int n_read = std::min(8, c.S), n_cpu = 8;
+ for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {
+ if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;
+ bool res = false;
+ for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);
+ if (!res) cold.push_back(e);
+ }
+ if ((int) cold.size() >= n_read + n_cpu) {
+ std::vector<IoQueue::Job> jobs;
+ std::atomic<int> left{0};
+ std::atomic<bool> bad{false};
+ for (int i = 0; i < n_read; ++i) {
+ uint8_t* sb = c.stage + (size_t) i * c.stage_stride;
+ for (int r = 0; r < 3; ++r) {
+ IoQueue::Job j;
+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];
+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];
+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];
+ j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];
+ j.len = mx_part_bytes(l, r);
+ j.dst = sb;
+ j.left = &left;
+ j.bad = &bad;
+ j.map = pk.at(l.mx_file[r], j.off);
+ jobs.push_back(j);
+ sb += flow_region(r);
+ }
+ }
+ const double rd = run_jobs(jobs, left);
+ if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);
+ }
+ { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM
+ const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;
+ std::vector<Ent> ents;
+ for (int i = 0; i < n_cpu; ++i) {
+ Ent en;
+ en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;
+ if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)
+ for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)
+ if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;
+ if (en.e < 0) break;
+ en.n = 1;
+ en.slot0 = i;
+ en.tok[0] = 0;
+ ents.push_back(en);
+ }
+ if (ents.empty()) return;
+ std::vector<float> x((size_t) g.dim);
+ for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);
+ const double bytes = (double) ents.size() * (double) l.blob;
+ double a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ const double t_first = now_ms() - a;
+ double t_warm = 1e30;
+ for (int rep = 0; rep < 2; ++rep) {
+ a = now_ms();
+ cpu_experts(l, x.data(), ents, (size_t) g.dim);
+ t_warm = std::min(t_warm, now_ms() - a);
+ }
+ pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);
+ if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);
+ }
+ pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;
+ if (o.verbose) {
+ char dk[32] = "not measured", cc[32] = "not measured";
+ if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);
+ if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);
+ std::fprintf(stderr,
+ "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "
+ "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",
+ pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);
+ }
+ }
+ // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's
+ // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)
+ double pm_rate(const Layer& l, int& n_cold) const {
+ const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ double sum = 0, cold = 0;
+ n_cold = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ sum += pr[e];
+ if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;
+ }
+ if (n_cold == 0) return 0.0;
+ const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;
+ return pm.m * (double) l.k_exp * share / n_cold;
+ }
+ // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))
+ double pm_picks(int n) const {
+ double d = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ int nc = 0;
+ const double a = pm_rate(L[(size_t) il], nc);
+ d += nc * (1.0 - std::exp(-a * n));
+ }
+ return d;
+ }
+ // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation
+ // that (bisection over its log; a running mean of the log)
+ void pm_learn(int n, double picked) {
+ if (!pm.on || n < 2 || picked < 1) return;
+ const double m0 = pm.m;
+ double lo = std::log(1e-3), hi = std::log(1e3);
+ for (int it = 0; it < 40; ++it) {
+ pm.m = std::exp(0.5 * (lo + hi));
+ (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);
+ }
+ const double obs = 0.5 * (lo + hi);
+ pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);
+ ++pm.n_m;
+ }
+ // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's
+ // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -
+ // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:
+ // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at
+ // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size
+ // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted
+ // faster (by half, with less than kPickRes in RAM); said with --verbose
+ // The choice is made from the prompt's length alone (pick_flow_min): the same prompt takes the same path - and so
+ // gives the same bits, the two paths round differently - in every session and on every restart. The timing prediction below
+ // (DS4_PICK=adaptive) used measured speeds and sampled page residency, so it flipped between runs: r9700, one 16-token feed
+ // through the flow in one run, another feed in the next, three different logits checksums in six runs. Measured on 4 and 1 R9700
+ // (first prompt of a run, windows against the flow, ms): 28 tokens 594 / 4966 (4 cards), 1094 / 5588 (1 card); 100: 1164 / 5375,
+ // 3279 / 7299; 300: 2831 / 5874, 9018 / 9803; 600: 5304 / 6808, 16969 / 11327; 1500: 13325 / 8012, 41547 / 18187 - the flow pays a
+ // fixed ~5 s the first time (its buffers and GEMM libraries), then wins from about 400 tokens up.
+ // The length from which the flow is taken: one number for every box (ds4_pick_rule.hpp says why 96), DS4_PICK_MIN=N overrides.
+ int pick_flow_min() {
+ if (pick_min_ > 0) return pick_min_;
+ pick_min_ = prompt_flow_min(std::getenv("DS4_PICK_MIN"));
+ std::fprintf(stderr, "ds4: prompt path: the chunked pass from %d tokens, windows of 16 below\n", pick_min_);
+ return pick_min_;
+ }
+ int pick_min_ = 0;
+ bool pick_path(int n) {
+ static const bool adaptive = [] {
+ const char* e = std::getenv("DS4_PICK");
+ return e != nullptr && std::strcmp(e, "adaptive") == 0;
+ }();
+ pm.last = -1;
+ if (!adaptive) return n >= pick_flow_min();
+ if (!pm.on || n >= kPickMax) return true;
+ double f_res = 0, f_cold = 0, w = 0;
+ const int nw = n / kWin, rem = n % kWin;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ int nc = 0;
+ const double a = pm_rate(l, nc);
+ if (nc == 0) continue;
+ int sampled = 0, res = 0, seen = 0;
+ const int step = std::max(1, nc / 32);
+ for (int e = 0; e < l.n_exp; ++e) {
+ if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;
+ bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;
+ if (!r) {
+ r = true;
+ for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);
+ }
+ ++sampled;
+ res += r;
+ }
+ const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;
+ auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };
+ f_res += share * u(n) * b;
+ f_cold += (1.0 - share) * u(n) * b;
+ w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;
+ }
+ const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;
+ const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);
+ const double t_flow = f_res / r_fr + f_cold / r_fc;
+ const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;
+ const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;
+ // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the
+ // fixed costs of every layer on every GPU again, which a pass of the flow pays once)
+ const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;
+ // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane
+ // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would
+ // cost the most; windows win where they are all in RAM)
+ const double res_share = f_res / std::max(1.0, f_res + f_cold);
+ const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));
+ pm.last = flow ? 1 : 0;
+ pm.last_cls = cls;
+ pm.last_nw = n_win;
+ pm.last_pred = flow ? t_flow : t_win;
+ if (o.verbose)
+ std::fprintf(stderr,
+ "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "
+ "experts' %.1f GB in RAM)\n",
+ n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,
+ 100.0 * res_share, (f_res + f_cold) / 1e9);
+ return flow;
+ }
+ // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more
+ static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }
+ // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs
+ // is mostly the fixed costs a window pays again and again)
+ double pm_k(int c) const {
+ for (int dc = 0; dc < 4; ++dc)
+ for (int cc : {c - dc, c + dc})
+ if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];
+ return 1.0;
+ }
+ // n tokens through windows of 16, the last one's logits_rows rows of logits
+ bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {
+ logits_all = false;
+ for (int a = 0; a < n; a += kWin) {
+ const int T = std::min(kWin, n - a);
+ if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;
+ }
+ return true;
+ }
+ // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken
+ // / predicted), windows' time a window - running means
+ void path_done(double ms) {
+ if (pm.last < 0 || pm.last_pred <= 0) return;
+ if (pm.last == 0) {
+ const double w = ms / 1000.0 / std::max(1, pm.last_nw);
+ pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;
+ ++pm.win_n;
+ } else {
+ const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));
+ double& k = pm.k[0][pm.last_cls];
+ int64_t& n = pm.kn[0][pm.last_cls];
+ k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;
+ ++n;
+ }
+ pm.last = -1;
+ }
+ // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the
+ // first would read from the files again more than the chunks' states moved to host memory and back at every layer
+ // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,
+ // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB
+ // sets it) - at least one expert, or not layer-major
+ bool lm_begin(const Run& run) {
+ static const char* env = std::getenv("DS4_LAYER_MAJOR");
+ if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;
+ Chunk& c = *chunks[0];
+ if (!c.flow) return false;
+ size_t need = 0, n_exp = 0, blob_max = 0;
+ double file_b = 0;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ size_t ne = 0;
+ for (int e : l.host_list)
+ if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;
+ const size_t stride = (l.blob + 63) & ~(size_t) 63;
+ need = std::max(need, ne * stride);
+ n_exp = std::max(n_exp, (size_t) l.n_exp);
+ blob_max = std::max(blob_max, stride);
+ file_b += (double) ne * (double) l.blob;
+ }
+ int64_t rows = 0;
+ for (int T : run.T) rows += T;
+ const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;
+ const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;
+ if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;
+ size_t lc_bytes = need;
+ if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);
+ else {
+ const uint64_t avail = ram_avail();
+ if (avail != ~uint64_t{0})
+ lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);
+ }
+ if (lc_bytes < blob_max) return false;
+ void* st_p = nullptr;
+ c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;
+ if (!c.lm_pinned) {
+ cudaGetLastError();
+ st_p = os::anon_alloc(state_b, -1, 1);
+ if (st_p == nullptr) return false;
+ }
+ c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);
+ if (c.lc == nullptr) {
+ if (c.lm_pinned) cudaFreeHost(st_p);
+ else os::anon_free(st_p, state_b);
+ return false;
+ }
+ c.lm_state = (uint8_t*) st_p;
+ c.lm_state_bytes = state_b;
+ c.lc_bytes = lc_bytes;
+ c.lc_of.assign(n_exp, -1);
+ c.lc_writer.assign(n_exp, -1);
+ c.lm = true;
+ lm_layers.store(0);
+ return true;
+ }
+ void lm_end() {
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ Chunk& c = *chunks[di];
+ if (!c.lm) continue;
+ cudaSetDevice(devs[di].id);
+ cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's
+ cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)
+ for (size_t i = c.base; i < c.n_read; ++i)
+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);
+ if (c.lm_pinned) cudaFreeHost(c.lm_state);
+ else os::anon_free(c.lm_state, c.lm_state_bytes);
+ os::anon_free(c.lc, c.lc_bytes);
+ c.lm_state = c.lc = nullptr;
+ c.lm_state_bytes = c.lc_bytes = 0;
+ c.lm = c.lm_pinned = false;
+ }
+ }
+
+ // the first layer of a long prompt's second half (forward_chunks' two passes), or -1: the first with compress ratio 1
+ // (V4.1: 20), every one after it ratio 1 too and owning no compressed KV or index keys of its own (they read its),
+ // every trunk layer on one GPU. The settings' MAYA_DS4_PROMPT_SKIP (STRATA_DS4_PROMPT_SKIP; DS4_YOCO for tests):
+ // 1 (the default) the skip, 2 DeepSeek's own serving's (the last 128 rows only: faster, the answers shift a
+ // little), 0 off
+ static int yoco_mode() {
+ static const int m = [] {
+ const char* v = std::getenv("DS4_YOCO");
+ if (v == nullptr) v = std::getenv("STRATA_DS4_PROMPT_SKIP");
+ return v != nullptr ? std::atoi(v) : 1;
+ }();
+ return m;
+ }
+ int yoco_layer() const {
+ if (yoco_mode() == 0 || chunk <= 0 || g.nl < 2 || (int) g.ratios.size() < g.nl) return -1;
+ int yl = -1;
+ for (int il = 0; il < g.nl && yl < 0; ++il)
+ if (g.ratios[(size_t) il] == 1) yl = il;
+ if (yl <= 0 || !L[(size_t) yl].kv_src) return -1;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ if (l.nm != 1 || l.dev != L[0].dev) return -1;
+ if (il > yl && (g.ratios[(size_t) il] != 1 || l.kv_src)) return -1;
+ }
+ return yl;
+ }
+
+ // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of
+ // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)
+ bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {
+ if (n < 1) return true;
+ flow_used.store(true);
+ std::lock_guard<std::mutex> warm_lk(warm_mu); // (the warm-up, if it is running, finishes first)
+ if (pos + n > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < n; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ // a long prompt's skip: the layers from yoco_layer() on (V4.1: 20-39) read the earlier positions only through
+ // that layer's compressed KV and index keys, and each looks back one window (win - 1 positions) more than the
+ // layer before - so they need their own work only on the last keep = (nl - yoco) (win - 1) + win rows for the
+ // run's last rows and the rings the decode reads to be what the full work gives (up to the experts' summing
+ // order). The chunks stay as they are: the ones before the cut go through the layers before yoco and its
+ // kv_only, the cut's chunk through those over all its rows and through the rest from the cut on (Run::r0, a
+ // view of its buffers), the chunks after it through every layer - each layer's experts read as without the
+ // skip. One GPU (DS4_YOCO=1; 0: off); not with every row's logits (tests)
+ const int yl = logits_rows > kWin ? -1 : yoco_layer();
+ // (DS4_YOCO_KEEP=<rows>: fewer - DeepSeek's own serving replays the last 128 tokens through those layers
+ // (DeepInfer's "SWA bounded replay"), which shifts the answers a little; at least one window)
+ static const int keep_env = std::getenv("DS4_YOCO_KEEP") ? std::atoi(std::getenv("DS4_YOCO_KEEP")) : 0;
+ const int keep = yl <= 0 ? 0
+ : keep_env > 0 ? std::max(keep_env, g.win)
+ : yoco_mode() == 2 ? g.win : (g.nl - yl) * (g.win - 1) + g.win;
+ const int a = yl > 0 && n - keep >= kYocoMin ? n - keep : 0;
+ const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;
++ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);
+ Run run;
+ for (int x = 0; x < n; x += per) {
+ run.p0.push_back(pos + x);
+ run.T.push_back(std::min(per, n - x));
+ run.seq.push_back(++win_seq);
+ }
+ if (a > 0) {
+ run.yoco = yl;
+ run.cut_k = a / per;
+ run.cut_r = a - run.cut_k * per;
+ }
+ logits_all = logits_rows > kWin;
+ if (logits_all) chunk_logits.resize((size_t) n * g.vocab);
+ for (int t = 0; t < n; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ plan_streams();
+ if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way
+ run.snap_pos = pos + n - 1;
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ Dev& d = devs[(size_t) l.dev];
+ if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;
+ if (l.c_kv != nullptr && l.snap_kv == nullptr &&
+ (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))
+ return false;
+ }
+ }
+ const int T_last = run.T.back();
+ n_logits = logits_all ? 0 : std::min(logits_rows, T_last);
+ const double tl = now_ms();
+ { // each chunk's end on the head device, for the progress reports
+ cudaSetDevice(devs[(size_t) head_dev].id);
+ while (chunk_done.size() < run.T.size()) {
+ cudaEvent_t e = nullptr;
+ CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));
+ chunk_done.push_back(e);
+ }
+ chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);
+ for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);
+ }
+ prof_chunk = prof_on;
+ const bool lm = lm_begin(run);
+ struct LmEnd { // (every way out: its host memory freed once the device is done with it)
+ Impl* p;
+ bool on;
+ ~LmEnd() {
+ if (on) p->lm_end();
+ }
+ } lm_end_{this, lm};
+ std::vector<std::string> errs(devs.size());
+ halted = false;
+ halt.store(false);
+ {
+ std::lock_guard<std::mutex> lk(halt_mu);
+ halt_k = INT_MAX;
+ halt_seen = -1;
+ }
+ bool halting = false;
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, &run, &errs] {
+ std::string e;
+ if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ size_t reported = 0;
+ for (;;) {
+ bool all = true;
+ for (auto& w : workers) all = all && w->idle();
+ if (!all && !halting && cancel && cancel()) { // a Stop: the devices end the run early
+ halting = true;
+ std::lock_guard<std::mutex> lk(halt_mu);
+ halt_k = halt_seen + 1;
+ halt.store(true, std::memory_order_release);
+ }
+ if (progress && lm) { // (layer-major: the share of the layers every chunk went through)
+ const int nd = lm_layers.load(std::memory_order_acquire);
+ if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);
+ } else if (progress) { // chunks through every layer (the last layer's device past their event)
+ while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&
+ cudaEventQuery(chunk_done[reported]) == cudaSuccess)
+ ++reported;
+ if (reported > 0 && reported < run.T.size())
+ progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);
+ }
+ if (all) break;
+ std::this_thread::sleep_for(std::chrono::milliseconds(50));
+ }
+ for (const std::string& e : errs) // (a device's failure first: a Stop's early end is not one)
+ if (!e.empty() && e != kHalted) {
+ err = e;
+ return false;
+ }
+ for (const std::string& e : errs)
+ if (e == kHalted) {
+ halted = true;
+ err = "stopped while reading the prompt";
+ return false;
+ }
+ st.layer_ms += now_ms() - tl;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ cudaSetDevice(d.id);
+ prof_collect();
+ std::fprintf(stderr,
+ "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "
+ "(with the waits for streamed ones), between layers %.0f ms\n",
+ n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);
+ { // the finer table (ms, % of the sum)
+ static const char* nm_[16] = {"between layers", "attn rest (q_b, wo_a/b, hc_post)", "shared expert", "experts (incl waits)",
+ "engram", "attn mHC (mix/split/norm)", "attn projections", "compressor + indexer",
+ "q_b + rope", "sparse attention", "ffn mHC + router + sort", "", "", "", "", ""};
+ double sum = 0;
+ for (double v : cprof_ms) sum += v;
+ for (int i = 0; i < 11; ++i)
+ std::fprintf(stderr, "ds4 prof %-34s %9.1f ms %5.1f%%\n", nm_[i], cprof_ms[i], sum > 0 ? 100.0 * cprof_ms[i] / sum : 0.0);
+ struct Agg {
+ double ms = 0, flops = 0;
+ long calls = 0, rows = 0, experts = 0;
+ };
+ std::map<std::pair<int, int>, Agg> agg; // (type, role 0 gate+up, 1 down)
+ {
+ std::lock_guard<std::mutex> lk(prof_mu);
+ for (const EkMark& m : ek_marks) {
+ float a = 0, b = 0;
+ cudaSetDevice(m.dev);
+ cudaEventElapsedTime(&a, m.e0, m.e1);
+ cudaEventElapsedTime(&b, m.e1, m.e2);
+ Agg& x = agg[{m.tg, 0}];
+ x.ms += a;
+ x.flops += 2.0 * 2.0 * m.nrows * (double) g.dim * (double) m.ff;
+ x.calls++, x.rows += m.nrows, x.experts += m.nexp;
+ Agg& y = agg[{m.td, 1}];
+ y.ms += b;
+ y.flops += 2.0 * m.nrows * (double) g.dim * (double) m.ff;
+ y.calls++, y.rows += m.nrows, y.experts += m.nexp;
+ ek_pool[m.dev].push_back(m.e0);
+ ek_pool[m.dev].push_back(m.e1);
+ ek_pool[m.dev].push_back(m.e2);
+ }
+ ek_marks.clear();
+ }
+ double tot = 0;
+ for (auto& kv : agg) tot += kv.second.ms;
+ for (auto& kv : agg)
+ std::fprintf(stderr,
+ "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld experts %6ld (%.1f us each, %.1f a call) rows/expert %6.1f ~%5.1f TFLOPS\n",
+ gg::iq::name(kv.first.first), kv.first.second ? "down" : "gate+up", kv.second.ms, kv.second.calls,
+ kv.second.experts, kv.second.experts > 0 ? 1e3 * kv.second.ms / (double) kv.second.experts : 0.0,
+ kv.second.calls > 0 ? (double) kv.second.experts / (double) kv.second.calls : 0.0,
+ kv.second.experts > 0 ? (double) kv.second.rows / (double) kv.second.experts : 0.0,
+ kv.second.ms > 0 ? kv.second.flops / kv.second.ms / 1e9 : 0.0);
+ std::fprintf(stderr, "ds4 prof expert kernels total %.1f ms\n", tot);
+ }
+ for (double& v : cprof_ms) v = 0;
+ prof_chunk = false;
+ }
+ if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {
+ const double ms = now_ms() - tl;
+ for (size_t di = 0; di < chunks.size(); ++di) {
+ const Chunk& c = *chunks[di];
+ if (!c.flow) continue;
+ const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;
+ const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;
+ const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)
+ char lcs[160] = "";
+ if (c.lm)
+ std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "
+ "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);
+ char dbgs[200] = "";
+ if (pm.on)
+ std::snprintf(dbgs, sizeof dbgs,
+ "; io threads busy %.0f ms, copy_ahead staging %.0f ms, DMA issues %.0f ms, slot waits %.0f ms",
+ c.io->busy_ms() - c.io_busy0, c.dbg_stage_ms, c.dbg_dma_ms, c.dbg_evt_ms);
+ std::fprintf(stderr,
+ "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "
+ "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "
+ "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "
+ "%.0f ms for reads%s\n",
+ devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),
+ (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,
+ (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,
+ (unsigned long long) c.it_spec_unused, c.wait_ms, dbgs);
+ }
+ }
+ if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)
+ bool tp = false;
+ for (const Layer& l : L) tp = tp || l.nm > 1;
+ double picked = 0;
+ for (const auto& cp : chunks)
+ if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;
+ if (!tp) pm_learn(n, picked);
+ }
+ if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)
+ pos += n;
+ st.tokens += n;
+ cache_tick(n);
+ if (lm_end_.on) { // (the layer cache's RAM back before the store and the swaps want it)
+ lm_end();
+ lm_end_.on = false;
+ }
+ cache_settle();
+ const int nr = std::min(T_last, tap_cap);
+ if (!seed_drafter(pos - nr, nr, 0, err)) return false;
+ if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -
+ // the next feed seeds them again from its own rows, past this position)
+ Check& ckp = ck[snap_slot & 1];
+ ckp.pos = run.snap_pos;
+ ckp.win.resize(L.size());
+ ckp.comp.resize(L.size() * 2);
+ for (size_t i = 0; i < L.size(); ++i) {
+ Layer& l = L[i];
+ Dev& dv = devs[(size_t) l.dev];
+ cudaSetDevice(dv.id);
+ const bool kept = !l.mtp && l.snap_win != nullptr;
+ ckp.win[i].resize((size_t) g.win * win_rb());
+ CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),
+ cudaMemcpyDeviceToHost, dv.st));
+ if (l.c_kv) {
+ ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);
+ ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);
+ CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,
+ ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,
+ ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));
+ }
+ }
+ for (Dev& dv : devs) {
+ cudaSetDevice(dv.id);
+ CK(cudaStreamSynchronize(dv.st));
+ }
+ }
+ return true;
+ }
+
+ // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)
+ void plan_window(int64_t p0, int T, WinPlan& wp) {
+ wp.par = par;
+ wp.async = par || (!sync_decode && T <= async_max_t);
+ wp.vl = 0;
+ for (int t = 0; t < T && !imgs.empty(); ++t)
+ if (in_image(p0 + t)) wp.vl |= 1u << t;
+ for (int il = 0; il < g.nl; ++il) {
+ const Layer& l = L[(size_t) il];
+ if (l.ratio > 0) {
+ const int R = l.ratio;
+ int n[kWin], k[kWin], clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ n[t] = (int) ((p0 + t + 1) / R);
+ k[t] = std::min(g.ix_topk, n[t]);
+ clen_max = std::max(clen_max, n[t]);
+ }
+ if (l.ix_src) {
+ for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];
+ if (clen_max > 0 && l.cand_src)
+ for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);
+ for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];
+ }
+ for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];
+ }
+ if (wp.async) {
+ wp.seq[il] = ++seq_issued;
+ layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);
+ }
+ }
+ }
+
+ int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }
+ // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's
+ // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows
+ void head_cols(int nm, int mi, int& v0, int& vn) const {
+ const Tensor* tw = pk.get("head.weight");
+ if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {
+ vn = g.vocab / nm;
+ v0 = mi * vn;
+ } else
+ split128(g.vocab, nm, mi, v0, vn);
+ }
+ // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's
+ // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two
+ // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather
+ // false: each member keeps its share (the drafter's markov steps over the group, mk_split)
+ bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {
+ const int V = g.vocab, D = g.dim, nm = head_nm();
+ int v0 = 0, vn = 0, Vmax = 0, z = 0;
+ head_cols(nm, mi, v0, vn);
+ head_cols(nm, 0, z, Vmax);
+ Group& G = groups[(size_t) d.grp];
+ float* out = mi == 0 ? d.logits : d.logits_part;
+ const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;
+ cudaSetDevice(d.id);
+ // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4
+ // for a whole one, each pass reading the whole share; a GGUF's: 8, so that every window the engine can build (rows
+ // <= kWin = 8) walks the head once instead of twice - measured on the drafter's fixed T=5: 4.55 -> 2.83 ms, its
+ // round 14.49 -> 13.46; the verify window's head 3.48 -> 2.68 ms, the same logits. DS4_HEAD_PER overrides.)
- const int per = d.head.fmt != 0 ? (std::getenv("DS4_HEAD_PER") ? std::atoi(std::getenv("DS4_HEAD_PER")) : strata::ds4::head_per_default(defaults_026()))
++ const int per = d.head.fmt != 0 ? (std::getenv("DS4_HEAD_PER") ? std::atoi(std::getenv("DS4_HEAD_PER")) :
++#if defined(STRATA_METAL)
++ 4 // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)
++#else
++ strata::ds4::head_per_default(defaults_026())
++#endif
++ )
+ : std::max(1, x3::mv_cols(d.head));
+ for (int c = 0; c < rows; c += per)
+ if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))
+ return false;
+ if (nm == 1 || !gather) return true;
+ const unsigned seq = ++d.head_seq;
+ const int slot = (int) (seq % 2);
+ auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };
+ auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };
+ if (mi > 0) {
+ if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = d.logits_part;
+ x.dst[0] = stage(mi);
+ x.bytes[0] = (size_t) rows * vn * 4;
+ xfer_put(x, flag(mi), seq, d.xcnt, d.st);
+ return true;
+ }
+ for (int m = 1; m < nm; ++m) {
+ int m0 = 0, mn = 0;
+ head_cols(nm, m, m0, mn);
+ Xfer x;
+ x.n = 1;
+ x.src[0] = stage(m);
+ x.dst[0] = d.logits_part;
+ x.bytes[0] = (size_t) rows * mn * 4;
+ xfer_get(x, flag(m), seq, d.st);
+ CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,
+ (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));
+ }
+ set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);
+ return true;
+ }
+
+ // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)
+ bool head_rows_dev(int di, int T, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ const int D = g.dim, H = g.hc;
+ if (n_logits <= 0) return true;
+ const int t0 = T - n_logits;
+ hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);
+ return head_share(d, d.mi, d.x, n_logits, err);
+ }
+
+ // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read
+ // h_taps)
+ bool tap(Dev& d, int il, int T, std::string& err) {
+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);
+ if (g.n_mtp == 0 || it == g.d_targets.end()) return true;
+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();
+ const int D = g.dim;
+ cudaSetDevice(d.id);
+ hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);
+ CK(cudaStreamWaitEvent(d.st, ev_seed, 0));
+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,
+ cudaMemcpyDeviceToHost, d.st));
+ return true;
+ }
+
+ bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {
++#if defined(STRATA_METAL)
++ struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)
++ bool on;
++ BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }
++ ~BusyAll() { if (on) ds4_busy_end(); }
++ } busy_all;
++#endif
+ if (T < 1 || T > kWin) {
+ err = "a window of " + std::to_string(T) + " tokens";
+ return false;
+ }
+ if (pos + T > o.max_context) {
+ err = "the context is full";
+ return false;
+ }
+ for (int t = 0; t < T; ++t)
+ if (tk[t] < 0 || tk[t] >= g.vocab) {
+ err = "token " + std::to_string(tk[t]) + " out of range";
+ return false;
+ }
+ const int64_t p0 = pos;
+ slot_held = false; // (the state grows now: what slot_save took is no longer the conversation)
+ for (int t = 0; t < T; ++t) {
+ toks.push_back(tk[t]);
+ ctoks.push_back(ctok(pos + t, tk[t]));
+ }
+ // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is
+ // left of their reads by then is the window's Engram time
+ const double te = now_ms();
+ const int64_t eg_w0 = eg_wait_ns.load();
+ for (int li = 0; li < pk.eg_layers; ++li) {
+ EgBatch& b = *eg_win[(size_t) li];
+ if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)
+ eg_issue(p0, T, li, nullptr, b);
+ }
+ st.engram_ms += now_ms() - te;
+ const int D = g.dim;
+ for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)
+ std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);
+ plan_window(p0, T, wplan);
+ prof_window = prof_on && T == 1;
+ n_logits = std::min(logits_rows, T);
+ const double tl = now_ms();
+ if (par) {
+ if (!forward_par(p0, T, err)) return false;
+ } else if (!forward_serial(p0, T, err)) return false;
+ st.layer_ms += now_ms() - tl;
+ for (auto& b : eg_win)
+ if (b->live) eg_finish(*b);
+ st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;
+ eg_stats();
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ const double th = now_ms();
+ CK(cudaStreamSynchronize(d.st));
+ st.head_ms += now_ms() - th;
+ prof_collect();
+ pos += T;
+ st.tokens += T;
+ cache_tick(T);
+ if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;
+ return true;
+ }
+
+ // one thread: the layers device after device, the hand-offs through pinned memory and events
++ void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer
++ ds4_metal_flush();
++ const float* hp = (const float*) strata::gpu::host_view(dev);
++ size_t nf = 0, first = 0; float mx = 0.0f;
++ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }
++ std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);
++ }
++ void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains
++ ds4_metal_flush();
++ const float* hp = (const float*) strata::gpu::host_view(d.h);
++ float mx = 0.0f; size_t nf = 0;
++ const size_t n = (size_t) T * g.hc * g.dim;
++ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }
++ std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);
++ }
+ bool forward_serial(int64_t p0, int T, std::string& err) {
+ const int D = g.dim, H = g.hc;
+ int cur = L[0].dev;
++ if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();
+ {
+ Dev& d = devs[(size_t) cur];
+ cudaSetDevice(d.id);
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));
++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, d.st);
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float *hp = d.h;
++ float hh[4] = {0,0,0,0};
++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
++ }
+ init_pre(d.pre, T, d.st);
++ if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);
+ }
+ std::vector<float> dump;
+ if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& l = L[(size_t) il];
+ if (l.dev != cur) {
+ Dev& a = devs[(size_t) cur];
+ Dev& b = devs[(size_t) l.dev];
+ cudaSetDevice(a.id);
+ const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ uint8_t* bb = a.bounce;
+ CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));
+ CK(cudaEventRecord(a.ev_out, a.st));
+ cudaSetDevice(b.id);
+ CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));
+ CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));
+ CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));
+ cur = l.dev;
+ }
+ if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
++ if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);
+ if (!layer(l, 0, p0, T, wplan, err)) return false;
+ if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
+ Dev& dd = devs[(size_t) l.dev];
+ std::vector<float> hv((size_t) T * H * D);
+ cudaSetDevice(dd.id);
+ CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));
+ CK(cudaStreamSynchronize(dd.st));
+ for (int t = 0; t < T; ++t)
+ std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,
+ (size_t) H * D * 4);
+ }
+ }
+ if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);
+ if (cur != head_dev) {
+ err = "the head is not on the last layer's GPU";
+ return false;
+ }
+ return head_rows_dev(head_dev, T, err);
+ }
+
+ // threads: each device's thread enqueues its layers (a group's members each their share) at once, the
+ // devices waiting on each other through host-mapped flags (the window's seq)
+ bool forward_par(int64_t p0, int T, std::string& err) {
+ wplan.win = ++win_seq;
+ std::vector<std::string> errs(devs.size());
+ for (size_t di = 0; di < devs.size(); ++di)
+ workers[di]->submit([this, di, p0, T, &errs] {
+ std::string e;
+ if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;
+ });
+ for (auto& w : workers) w->wait();
+ for (const std::string& e : errs)
+ if (!e.empty()) {
+ err = e;
+ return false;
+ }
+ return true;
+ }
+
+ // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer
+ Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {
+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,
+ nc = (size_t) T * cand_stride() * 4;
+ void* dv[4] = {d.h, d.pre, d.topk, d.cand};
+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};
+ const size_t nb[4] = {nh, np, nk, nc};
+ Xfer x;
+ x.n = 4;
+ for (int i = 0; i < 4; ++i) {
+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];
+ x.dst[i] = put ? (void*) hv[i] : dv[i];
+ x.bytes[i] = nb[i];
+ }
+ return x;
+ }
+
+ bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Dev& d = devs[(size_t) di];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int D = g.dim, H = g.hc;
+ bool started = false;
+ std::vector<char> got(srcs.size(), 0);
+ for (int il = 0; il < g.nl; ++il) {
+ Layer& L0 = L[(size_t) il];
+ int mi = -1;
+ for (int m = 0; m < L0.nm; ++m)
+ if (member(L0, m).dev == di) mi = m;
+ if (mi < 0) continue;
+ if (!started) { // the window's input: the embedding rows, or the previous group's hand-off
+ started = true;
+ if (il == 0) {
- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, s));
++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));
+ embed(d.emb_rows, T, d.h, D, H, s);
++ if (std::getenv("DS4_DBG_EMB")) {
++ ds4_metal_flush();
++ float *hp = d.h;
++ float hh[4] = {0,0,0,0};
++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);
++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
++ }
+ init_pre(d.pre, T, s);
+ } else {
+ const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];
+ xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);
+ }
+ }
+ // the compressed KV / index keys another group's source layer wrote this window
+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {
+ if (sidx < 0 || got[(size_t) sidx]) continue;
+ got[(size_t) sidx] = 1;
+ Src& sr = srcs[(size_t) sidx];
+ if (L[(size_t) sr.layer].grp == L0.grp) continue;
+ const int64_t g_first = p0 / sr.ratio;
+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);
+ if (ng == 0) continue;
+ Xfer x;
+ if (sr.ckv[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage;
+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();
+ x.bytes[x.n++] = (size_t) ng * ckv_rb();
+ }
+ if (sr.ik[(size_t) di] != nullptr) {
+ x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();
+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();
+ x.bytes[x.n++] = (size_t) ng * ik_rb();
+ }
+ if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);
+ }
+ if (mi == 0 && !tap(d, il, T, err)) return false;
+ if (!layer(L0, mi, p0, T, wp, err)) return false;
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);
+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;
+ if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);
+ }
+ if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);
+ return true;
+ }
+
+ // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,
+ // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it
+ bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,
+ size_t bytes, int from, std::string& err) {
+ bool any = false;
+ for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);
+ if (!any) return true;
+ Dev& d = devs[(size_t) from];
+ CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));
+ CK(cudaEventRecord(ev, d.st));
+ for (size_t od = 0; od < devs.size(); ++od) {
+ if (copies[od] == nullptr || (int) od == from) continue;
+ Dev& o2 = devs[od];
+ cudaSetDevice(o2.id);
+ CK(cudaStreamWaitEvent(o2.st, ev, 0));
+ CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));
+ }
+ cudaSetDevice(d.id);
+ return true;
+ }
+
+ // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;
+ // the partial sums all-reduced across the group)
+ bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {
+ Layer& l = member(L0, mi);
+ Dev& d = devs[(size_t) l.dev];
+ cudaSetDevice(d.id);
+ cudaStream_t s = d.st;
+ const int nm = L0.nm;
+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
+ const ArGroup& ag = groups[(size_t) L0.grp].ar;
+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
++ if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's
++ ds4_metal_flush();
++ int shown = 0;
++ for (int e = 0; e < l.n_exp && shown < 3; ++e) {
++ if (l.slot[(size_t) e] < 0) continue;
++ std::vector<uint8_t> want(l.blob, 0);
++ read_mx_file(l, e, want.data());
++ const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));
++ size_t bad = 0, first = 0;
++ for (size_t i = 0; i < l.blob; ++i)
++ if (want[i] != have[i]) { if (!bad) first = i; ++bad; }
++ std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);
++ ++shown;
++ }
++ }
++ if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);
++ if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
++ std::vector<float> hv((size_t) T * H * D);
++ cudaSetDevice(d.id);
++ ds4_metal_flush();
++ {
++ const float* hvp = (const float*) strata::gpu::host_view(d.h);
++ if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);
++ }
++ int nf = 0, nzv = 0;
++ float mx = 0.0f;
++ size_t firstnan = 0, lastnan = 0;
++ for (size_t i = 0; i < hv.size(); ++i) {
++ const float v = hv[i];
++ if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }
++ if (v != 0.0f) ++nzv;
++ if (std::fabs(v) > mx) mx = std::fabs(v);
++ }
++ std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",
++ l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],
++ hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);
++ }
+ prof_mark(d, 0);
+ // Engram
+ if (l.eg >= 0) {
+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
+ const size_t per = (size_t) cols * (ehd + ehd / 32);
+ eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)
+ uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;
+ for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]
+ CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,
+ cudaMemcpyHostToDevice, s));
+ uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;
+ for (int t = 0; t < T; ++t)
+ CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
+ (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
++ dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);
++ dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);
++ if (std::getenv("DS4_DBG_EG")) {
++ dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
++ const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
++ for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)
++ if (!std::isfinite(xv[i])) {
++ const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;
++ const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;
++ const uint8_t* hv = src + t * per + c * ehd;
++ std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);
++ for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);
++ std::fprintf(stderr, "\n");
++ break;
++ }
++ }
+ if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
++ if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
++ dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);
++ dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);
++ dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);
+ engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
++ dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);
++ if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
+ }
++ dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
++ dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));
+ // attention
++#if defined(STRATA_METAL)
++ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, l.hca_scale, l.hca_base, H,
++ g.sinkhorn, g.hc_eps, d.a_pre, d.a_post, d.a_comb, s);
++#else
+ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps,
+ d.a_pre, d.a_post, d.a_comb, s);
++#endif
++ dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);
++ dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);
++ dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);
++ dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);
+ hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);
++ dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);
++ dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
+ {
+ x3::MvJob j[4];
+ int nj = 0;
+ j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);
+ j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);
+ if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);
+ if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);
+ if (!mv_n(d, j, nj, err)) return false;
+ }
++ dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
++ dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);
+ Finish fq;
+ fq.w = l.q_norm;
+ fq.eps = g.eps;
+ finish(d.qr, T, g.q_lora, g.q_lora, fq, s);
++ dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);
+ if (NHL > 0) {
+ if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;
+ Finish fr;
+ fr.rd = g.rd;
+ fr.pos = (int) p0;
+ fr.vpp = NHL;
+ fr.freqs = freqs;
+ finish(d.q, T * NHL, HD, HD, fr, s);
+ }
++ if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);
+ // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark
+ // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of
+ // that ring and to the whole block, and leaves the ring as it was
+ ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)
+ Finish fk;
+ fk.w = l.kv_norm;
+ fk.eps = g.eps;
+ fk.rd = g.rd;
+ fk.pos = (int) p0;
+ fk.freqs = freqs;
+ fk.quant = 1;
+ fk.dst = d.ext + (size_t) g.win * HD;
+ fk.dst_stride = HD;
+ finish(d.kvr, T, HD, HD, fk, s);
+ if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
++ dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);
++ dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)
+ AttnRows ar;
+ for (int t = 0; t < T; ++t) {
+ if (l.mtp) {
+ ar.win0[t] = 0;
+ ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);
+ ar.nsel[t] = T;
+ continue;
+ }
+ const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)
+ ar.win0[t] = first;
+ ar.nwin[t] = g.win + t - first + 1;
+ ar.nsel[t] = 0;
+ }
+ const void* ckv = nullptr;
+ int ckv_fmt = kKvFp4;
+ if (l.ratio > 0) {
+ Src& sr = srcs[(size_t) l.src];
+ const int R = l.ratio;
+ // the groups this window completes: g_first .. g_first + ng - 1
+ const int64_t g_first = p0 / R; // the first group whose last position is >= p0
+ const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1
+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);
+ if (l.kv_src) {
+ if (R == 1) {
+ Finish fc;
+ fc.w = l.c_norm;
+ fc.eps = g.eps;
+ finish(d.ckr, T, HD, HD, fc, s);
+ CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));
+ } else {
+ if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected
+ const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;
+ CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));
+ CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
+ }
+ compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
++ dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);
++ dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);
+ }
+ }
+ Rows lim;
+ int clen_max = 0;
+ for (int t = 0; t < T; ++t) {
+ lim.n[t] = (int) ((p0 + t + 1) / R);
+ lim.k[t] = std::min(g.ix_topk, lim.n[t]);
+ clen_max = std::max(clen_max, lim.n[t]);
+ }
+ if (l.ix_src) {
+ Src& ks = srcs[(size_t) l.ik_src];
+ if (l.owns_k && ng > 0) {
+ if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.w = l.ik_norm;
+ fi.eps = g.eps;
+ fi.rd = g.rd;
+ fi.pos = (int) (g_first * R);
+ fi.pos_step = R;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ fi.pdst_stride = (long long) ik_rb();
+ finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);
+ if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),
+ (size_t) ng * ik_rb(), l.dev, err))
+ return false;
+ }
+ if (clen_max > 0) {
+ if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;
+ Finish fi;
+ fi.rd = g.rd;
+ fi.pos = (int) p0;
+ fi.vpp = g.ix_heads;
+ fi.freqs = freqs;
+ fi.quant = 3;
+ finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);
+ // (iw rows are 64 apart)
+ gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,
+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);
+ const long long ss = (long long) o.max_context + 64;
+ bool restrict_ = false;
+ for (int t = 0; t < T; ++t)
+ restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;
+ if (l.uses_cand && restrict_) {
+ Rows cl; // the candidate positions of each query: its blocks x blk
+ int nmax = 0;
+ for (int t = 0; t < T; ++t) {
+ cl.n[t] = lim.n[t];
+ cl.k[t] = wp.cand[l.il][t];
+ nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);
+ }
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,
+ (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);
+ Rows tr;
+ for (int t = 0; t < T; ++t) {
+ tr.n[t] = wp.cand[l.il][t] * g.cand_blk;
+ tr.k[t] = lim.k[t];
+ }
+ topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);
+ } else {
+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,
+ lim, d.scores, ss, s);
+ if (l.cand_src) {
+ Rows cr;
+ for (int t = 0; t < T; ++t) {
+ cr.n[t] = lim.n[t];
+ cr.k[t] = g.cand_topk;
+ }
+ cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,
+ (int) cand_stride(), s);
+ }
+ topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);
+ }
+ }
+ }
+ if (l.kv_src && ng > 0) {
+ Finish fc;
+ fc.rd = g.rd;
+ fc.pos = (int) (g_first * R);
+ fc.pos_step = R;
+ fc.freqs = freqs;
+ fc.quant = 2;
+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();
+ fc.pdst_stride = (long long) ckv_rb();
+ finish(d.latent, ng, HD, HD, fc, s);
+ if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),
+ (size_t) ng * ckv_rb(), l.dev, err))
+ return false;
+ if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers
+ Xfer x;
+ x.src[0] = fc.pdst;
+ x.dst[0] = sr.x_stage;
+ x.bytes[0] = (size_t) ng * ckv_rb();
+ x.n = 1;
+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {
+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();
+ x.dst[1] = sr.x_stage + x_cap() * ckv_rb();
+ x.bytes[1] = (size_t) ng * ik_rb();
+ x.n = 2;
+ }
+ xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
+ }
+ }
++ if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);
++ dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);
+ for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
+ ckv = sr.ckv[(size_t) l.dev];
+ }
+ if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection
+ ckv = d.ext + (size_t) g.win * HD;
+ ckv_fmt = kKvBf16;
+ iota_rows(d.topk, T, T, (int) topk_stride(), s);
+ }
+ if (NHL > 0) {
+ sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,
+ 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);
++ dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);
+ Finish fo;
+ fo.rd = g.rd;
+ fo.pos = (int) p0;
+ fo.vpp = NHL;
+ fo.freqs = freqs;
+ fo.inverse = 1;
+ fo.quant = 1;
+ finish(d.o, T * NHL, HD, HD, fo, s);
++ dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);
+ x3::MvJob j[16];
+ const int per = HQ / OG;
+ for (int gi = 0; gi < OG; ++gi)
+ j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);
+ if (!mv_n(d, j, OG, err)) return false;
+ if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;
+ } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)
++ dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);
+ if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)
++ dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ prof_mark(d, 1);
+ // the MoE
++#if defined(STRATA_METAL)
++ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, l.hcf_scale, l.hcf_base, H,
++ g.sinkhorn, g.hc_eps, d.f_pre, d.f_post, d.f_comb, s);
++#else
+ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps,
+ d.f_pre, d.f_post, d.f_comb, s);
++#endif
+ hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);
++ dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);
++ dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);
++ dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);
++ dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);
+ if (wp.async) {
+ // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members
+ // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the
+ // router made), and waits for the answer only before the experts' sum
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ const unsigned sq = wp.seq[l.il];
+ const int slot = (int) (sq % kRing);
+ const bool lead = mi == 0;
+ int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below
+ for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {
+ const Layer* nx = ahead_layer(L0, a);
+ if (nx == nullptr) break;
+ const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);
+ float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;
+ gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN
+ // weights instead: the same hit rate)
+ route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);
+ na[a - 1] = m;
+ }
+ route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,
+ d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],
+ na[1]);
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;
+ swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;
+ prof_mark(d, 2);
+ const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;
+ if (T == 1 && moe_token_path) {
+ x3::MoeArgs ma;
+ ma.plan_ptr = d.plan_ptr;
+ ma.plan_w = d.rw;
+ ma.cpu_part = lead ? resps[slot].part[0] : nullptr;
+ ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;
+ ma.lay = lay;
+ ma.k = l.k_exp;
+ ma.n_embd = D;
+ ma.n_ff = FF;
+ ma.limit = g.lim;
+ const bool mx = lay.fmt == gg::kFmtMXFP4;
+ if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)
+ zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);
++ // DS4_TLOG2: the expert stage's three host spans (gate/up issue, the CPU-lane wait, down issue)
++ static const bool tlog2 = std::getenv("DS4_TLOG2") != nullptr;
++ static double t_moe_gu = 0, t_moe_wait = 0, t_moe_dn = 0; static long t_moe_n = 0; static bool t_moe_reg = false;
++ if (tlog2 && !t_moe_reg) {
++ t_moe_reg = true;
++ std::atexit([] {
++ std::fprintf(stderr, "tlog2: %ld stages | host ms a stage: gate_up %.2f cpu_wait %.2f down %.2f\n",
++ t_moe_n, t_moe_n ? t_moe_gu / (double) t_moe_n : 0.0,
++ t_moe_n ? t_moe_wait / (double) t_moe_n : 0.0,
++ t_moe_n ? t_moe_dn / (double) t_moe_n : 0.0);
++ });
++ }
++ const double m0 = tlog2 ? now_ms() : 0.0;
+ if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);
+ else x3::moe_gate_up(ma, d.x, d.xg, s);
++ const double m1 = tlog2 ? now_ms() : 0.0;
+ prof_mark(d, 3);
+ if (lead) wait_seq(&resps[slot].seq, sq, s);
++ const double m2 = tlog2 ? now_ms() : 0.0;
+ prof_mark(d, 4);
+ if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
+ else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);
++ if (tlog2) { t_moe_gu += m1 - m0; t_moe_wait += m2 - m1; t_moe_dn += now_ms() - m2; ++t_moe_n; }
+ } else { // each distinct expert read once for the window's tokens
+ gf::MoeWin mw;
+ mw.plan_ptr = d.plan_ptr;
+ mw.T = T;
+ mw.k = l.k_exp;
+ mw.grp_ptr = d.grp_ptr;
+ mw.grp_start = d.grp_start;
+ mw.grp_ent = d.grp_ent;
+ mw.n_groups = d.n_groups;
+ gf::moe_win_group(mw, s);
+ x3::WinArgs wa;
+ wa.grp_ptr = d.grp_ptr;
+ wa.grp_start = d.grp_start;
+ wa.grp_ent = d.grp_ent;
+ wa.n_groups = d.n_groups;
+ wa.plan_ptr = d.plan_ptr;
+ wa.plan_w = d.rw;
+ wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;
+ wa.cpu_stride = kMaxDim;
+ wa.cpu_flag = lead ? resps[slot].cpu : nullptr;
+ wa.wait_ptr = lead ? &resps[slot].seq : nullptr;
+ wa.wait_val = sq;
+ wa.x = d.x;
+ wa.x_stride = (size_t) D;
+ wa.h = d.xg;
+ wa.part = d.ed;
+ wa.sh_out = d.sh_out;
+ wa.out = d.moe_out;
+ wa.out_stride = (size_t) D;
+ wa.lay = lay;
+ wa.T = T;
+ wa.k = l.k_exp;
+ wa.n_embd = D;
+ wa.n_ff = FF;
+ wa.limit = g.lim;
+ if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {
+ err = "exl3 win_experts failed";
+ return false;
+ }
+ }
+ if (lead) st.experts += (int64_t) T * l.k_exp;
+ if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
+ prof_mark(d, 5);
+ return true;
+ }
+ if (nm > 1) {
+ err = "a tensor-parallel layer on the host-routed path";
+ return false;
+ }
+ // (glog rows are 1024 apart: route reads them so)
++ static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes
++ static double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;
++ const double ph0 = phases ? now_ms() : 0.0;
+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);
+ VlBits vb = {};
+ vb.w[0] = l.mtp ? 0u : wp.vl;
+ route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,
+ vb.w[0] != 0 ? &vb : nullptr);
+ CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));
+ CK(cudaEventRecord(d.ev, s));
+ // the shared expert while the host looks at the picks
+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;
++#if defined(STRATA_METAL)
++ int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer
++ if (T == 1 && la_enabled()) {
++ for (int a = 1; a <= la_depth(); ++a) {
++ const int il = l.il + a;
++ if (il >= g.nl) break;
++ const Layer& nx = L[(size_t) il];
++ if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;
++ gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);
++ route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,
++ d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);
++ CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));
++ la_n = a;
++ }
++ if (la_n > 0) CK(cudaEventRecord(d.ev2, s));
++ }
++#endif
++ if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);
++ std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }
+ swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);
++ if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);
+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;
+ const double tw = now_ms();
+ CK(cudaEventSynchronize(d.ev));
+ st.gpu_wait_ms += now_ms() - tw;
++ const double ph2 = phases ? now_ms() : 0.0;
++ dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);
++ dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);
++ dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);
++ dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);
+ // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the
+ // CPU lane's entries
+ Combine cm;
+ Gather ga;
+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;
+ int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];
+ int nvex = 0, nrows = 0;
+ std::vector<Ent> ents;
+ std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)
+ std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)
+ for (int t = 0; t < T; ++t) {
+ cm.n[t] = 0;
+ cm.cpu[t] = 0;
+ for (int i = 0; i < l.k_exp; ++i) {
+ const int e = h_idx[t * kPicks + i];
+ if (e < 0 || e >= l.n_exp) {
- err = "router picked expert " + std::to_string(e);
++ err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";
+ return false;
+ }
+ ++pr[e];
+ l.heat[(size_t) e] += 1.0f;
+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);
+ per[(size_t) e].push_back({t, i});
+ }
+ }
+ int nslots = 0;
+ for (int e = 0; e < l.n_exp; ++e) {
+ const auto& pe = per[(size_t) e];
+ if (pe.empty()) continue;
- if (l.slot[(size_t) e] >= 0) {
++#if defined(STRATA_METAL)
++ const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));
++#else
++ const bool on_gpu = l.slot[(size_t) e] >= 0;
++#endif
++ if (on_gpu) {
+ vex[nvex] = e;
+ vrow0[nvex] = nrows;
+ vn[nvex] = (int) pe.size();
+ for (const auto& tp : pe) {
+ ga.src[nrows] = tp.first;
+ const int t = tp.first;
+ cm.row[t][cm.n[t]] = nrows;
+ cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];
+ ++cm.n[t];
+ ++nrows;
+ }
+ ++nvex;
+ } else {
+ for (size_t a = 0; a < pe.size(); a += (size_t) ent_cap(l)) {
+ Ent en;
+ en.e = e;
+ en.slot0 = nslots;
+ en.n = (int) std::min<size_t>((size_t) ent_cap(l), pe.size() - a);
+ for (int i = 0; i < en.n; ++i) {
+ en.tok[i] = pe[a + (size_t) i].first;
+ cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});
+ cm.cpu[en.tok[i]] = 1;
+ }
+ nslots += en.n;
+ ents.push_back(en);
+ }
+ }
+ }
+ st.experts += (int64_t) T * l.k_exp;
+ st.cpu_experts += nslots;
++#if defined(STRATA_METAL)
++ if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);
++#endif
+ if (nrows > 0) {
+ gather_rows(d.x, ga, nrows, D, d.xe, s);
++ dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);
++ { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }
++ dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }
+ x3::MvJob j[2 * kWin * kPicks];
+ int nj = 0;
+ for (int i = 0; i < nvex; ++i) {
+ const auto& vm = l.vm[(size_t) vex[i]];
+ j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);
+ }
+ if (!mv_n(d, j, nj, err)) return false;
++ dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);
++ dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);
+ swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);
++ dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);
+ nj = 0;
+ for (int i = 0; i < nvex; ++i)
+ j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,
+ vn[i], g.ff, D);
+ if (!mv_n(d, j, nj, err)) return false;
++ dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);
+ }
++ const double ph3 = phases ? now_ms() : 0.0;
+ if (!ents.empty()) {
+ const double tc = now_ms();
- cpu_experts(l, h_x, ents);
++#if defined(STRATA_METAL)
++ ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
++ ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)
++#endif
++ static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
++ if (!skip_cpu) cpu_experts(l, h_x, ents);
++ else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)
++#if defined(STRATA_METAL)
++ ds4_busy_end();
++#endif
+ std::fill(h_part, h_part + (size_t) T * D, 0.0f);
+ for (int sl = 0; sl < nslots; ++sl) {
+ float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;
+ const float wv = cpu_rows[(size_t) sl].second;
+ const float* y = c_yd.data() + (size_t) sl * D;
+ for (int e = 0; e < D; ++e) o[e] += wv * y[e];
+ }
++ { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;
++ ms += now_ms() - tc; ++calls; exps += (long) ents.size();
++ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }
++ (void) 0; (void) ms; }
+ st.cpu_ms += now_ms() - tc;
++ dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);
+ CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));
+ }
+ combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);
++ dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);
++ if (std::getenv("DS4_DBG_MOE")) {
++ float hp = 0.0f;
++ for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));
++ std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);
++ dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);
++ dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);
++ }
+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);
+ std::swap(d.h, d.h2);
++ dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);
++ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)
+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));
++ if (phases && T == 1) {
++ const double ph4 = now_ms();
++ ph[0] += ph2 - ph0; ph[1] += ph3 - ph2; ph[2] += ph4 - ph3; if (ph_prev > 0 && ph0 - ph_prev < 5.0) ph[3] += ph0 - ph_prev; ph_prev = ph4; ++ph_n;
++ static bool reg = false;
++ if (!reg) { reg = true; std::atexit([] { std::fprintf(stderr, "phases per layer call (%ld): route enqueue + shared expert + wait %.3f ms, routing analysis + GPU experts enqueue %.3f ms, CPU lane + combine + hc_post %.3f ms, attention enqueue %.3f ms%s", ph_n, ph[0] / ph_n, ph[1] / ph_n, ph[2] / ph_n, ph[3] / ph_n, "\n"); }); }
++ }
+ // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)
+ return true;
+ }
+
+ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ strata::dsv41k::SamplerParams sp;
+ sp.temperature = temperature;
+ sp.greedy = temperature <= 0.0f;
+ sp.top_p = top_p;
+ sp.top_k = top_k;
+ sp.min_p = min_p;
+ sp.seed = seed;
+ sp.counter = counter;
+ strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,
+ sp, d.tok, d.st);
+ int t = -1;
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);
+ cudaStreamSynchronize(d.st);
+ return t;
+ }
+
+ bool logits(std::vector<float>& out) {
+ if (logits_all) {
+ out = chunk_logits;
+ return true;
+ }
+ Dev& d = devs[(size_t) head_dev];
+ cudaSetDevice(d.id);
+ out.resize((size_t) std::max(1, n_logits) * g.vocab);
++#if defined(STRATA_METAL)
++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)
++#endif
+ return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;
+ }
+
+ void save_profile() const {
- if (o.profile.empty()) return;
++ if (o.profile.empty() || o.profile_readonly) return; // (--profile-readonly: a test keeps the file it read)
+ const std::string tmp = o.profile + ".tmp";
+ std::FILE* f = std::fopen(tmp.c_str(), "wb");
+ if (f == nullptr) return;
+ const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();
+ std::fclose(f);
+ // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile
+ // never moved on and only .tmp files were left)
+ if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());
+ }
+
+ std::vector<std::string> info_lines() const {
+ std::vector<std::string> v;
+ int nh = 0;
+ for (const Layer& l : L) nh += l.n_exp - l.n_vram;
+ char b[512];
+ // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)
+ std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "
+ "cpu_experts=%d ram_experts=%d engine=ds4-0.1",
+ (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);
+ v.push_back(b);
+ if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)
+ std::snprintf(b, sizeof b,
+ "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "
+ "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "
+ "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);
+ v.push_back(b);
+ std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),
+ pagefile_hint);
+ v.push_back(b);
+ }
+ if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor
+ std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,
+ pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);
+ v.push_back(b);
+ }
+ if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)
+ std::snprintf(b, sizeof b,
+ "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "
+ "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",
+ vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,
+ (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),
+ (long long) (vt.device_bytes() >> 20));
+ v.push_back(b);
+ } else if (!vision_err.empty()) { // (the server offers no pictures and says why)
+ std::string why = vision_err;
+ for (char& ch : why)
+ if (ch == '\n' || ch == '\r') ch = ' ';
+ v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");
+ v.push_back("INFO vision=off");
+ }
+ return v;
+ }
+};
+
+Model::Model() : p_(new Impl) {}
+Model::~Model() = default;
+bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }
+bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }
+bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ p.halted = false;
++ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);
+ if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here
+ const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)
+ const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)
+ : p.feed_windows(tokens, n, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ return p.feed_windows(tokens, n, logits_rows, err);
+}
+bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {
+ Impl& p = *p_;
+ p.pm.last = -1;
+ p.halted = false;
+ const double t0 = now_ms();
+ if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers
+ const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)
+ if (ok) p.path_done(now_ms() - t0);
+ return ok;
+ }
+ if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;
+ p.checkpoint(slot);
+ const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);
+ if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)
+ return ok;
+}
+int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }
+void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }
+void Model::set_cancel(std::function<bool()> f) { p_->cancel = std::move(f); }
+bool Model::cancelled() const { return p_->halted; }
+int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {
+ return p_->sample(t, tp, tk, mp, seed, counter);
+}
+bool Model::logits(std::vector<float>& out) { return p_->logits(out); }
+int Model::image_token() const { return p_->img_tok; }
+int Model::dim() const { return p_->g.dim; }
+bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {
+ return p_->encode_image(rgb, width, height, span, err);
+}
+void Model::set_images(std::vector<Image> images) {
+ std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });
+ p_->imgs = std::move(images);
+}
+int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }
+bool Model::gpus_are_gfx12() const { return p_->rdna4(); }
+bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,
+ float mp, uint64_t seed, uint64_t counter) {
+ return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);
+}
+bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,
+ int& accepted, int& next, std::string& err) {
+ return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);
+}
+int64_t Model::pos() const { return p_->pos; }
+int64_t Model::max_context() const { return p_->o.max_context; }
+int Model::vocab() const { return p_->g.vocab; }
+const std::vector<int32_t>& Model::tokens() const { return p_->toks; }
+void Model::reset() { p_->reset(); }
+void Model::checkpoint(int slot) { p_->checkpoint(slot); }
+bool Model::restore(int64_t limit) { return p_->restore(limit); }
+void Model::pin() { p_->pin(); }
+std::vector<int64_t> Model::check_positions() const { return p_->check_positions(); }
+int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }
+uint64_t Model::slot_save(std::vector<uint8_t>& out, std::string& err) { return p_->slot_save(out, err); }
+bool Model::slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {
+ return p_->slot_load(data, bytes, n_pos, err);
+}
+std::string Model::slot_identity() const { return p_->slot_identity(); }
+void Model::save_profile() const { p_->save_profile(); }
+const Stats& Model::stats() const {
+ Stats& st = p_->st;
+ if (p_->store) { // (the store keeps its own counters: copied in when asked)
+ const ExpertStore& x = *p_->store;
+ st.store_hits = x.stats().hits.load();
+ st.store_misses = x.stats().misses.load();
+ st.store_prefetched = x.stats().prefetched.load();
+ st.store_pf_used = x.stats().prefetch_used.load();
+ st.store_pf_wasted = x.stats().prefetch_wasted.load();
+ st.store_evictions = x.stats().evictions.load();
+ st.store_bytes = x.stats().bytes_read.load();
+ st.store_fails = x.stats().fails.load();
+ st.store_wait_ms = x.stats().wait_ns.load() / 1e6;
+ st.store_busy_ms = x.busy_ms();
+ st.store_zc_visible = (int64_t) x.visible_count();
+ st.store_slots = (int64_t) x.slots_allocated();
+ st.store_slots_max = (int64_t) x.slots_max();
+ }
+ return st;
+}
+std::string Model::describe() const { return p_->describe(); }
+std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }
+
+} // namespace strata::ds4
diff --cc src/dsv41/ds4_model.hpp
index afa9d2a69,f8b087c70..6cf65dbad
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@@ -30,6 -29,6 +31,9 @@@ struct Options
double vram_experts = 1.0; // the share of the expert budget to use (tests: less, so the CPU lane works)
int cpu_threads = 0; // 0: the physical cores
std::string profile; // expert hit counts (read to place the hottest experts in VRAM, written back)
++ // --profile-readonly: read the profile to place the experts, never write it back. Without this the run's routing
++ // counts go into the file, so the next run places experts differently and greedy output / KL move between runs.
++ bool profile_readonly = false;
bool expert_cache = true; // experts move between VRAM and the CPU lane as they are used
// the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of
// copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold them
diff --cc src/dsv41/ds4_tq.cu
index 724072e67,10c7261f8..a55759404
--- a/src/dsv41/ds4_tq.cu
+++ b/src/dsv41/ds4_tq.cu
@@@ -10,6 -10,6 +10,7 @@@
// block scale d is the lane's own. Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose
// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a
// double reference and on the card by tests/dsv41/tqw_hip_test.cpp). Elsewhere the same lane mapping in plain FMAs.
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_iq.cuh"
#include <cuda_fp16.h>
diff --cc src/dsv41/ds4_vision.cu
index 38916c755,1d94ebe4b..ec112ca5c
--- a/src/dsv41/ds4_vision.cu
+++ b/src/dsv41/ds4_vision.cu
@@@ -10,6 -10,6 +10,7 @@@
// x[piece] += wo(O) + bias
// xn = norm2(x); per row piece: x += w2(silu(gate) * up), gate | up = w1(xn)
// xn = norm(x); the 3 x 3 unfold; GELU(w1 + b1); w2 + b2 -> BF16 rows (the aligner's weights stream in pieces too)
++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "ds4_vision.cuh"
#include <cublas_v2.h>
diff --cc src/dsv41/vendor/exl3.cu
index 501010668,01bf0cd44..e94ffd7b6
--- a/src/dsv41/vendor/exl3.cu
+++ b/src/dsv41/vendor/exl3.cu
@@@ -19,6 -19,9 +19,10 @@@
// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy
// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The
// prompt path below is this file's on every GPU.
+ #if defined(STRATA_METAL)
++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
+ #include <ds4_metal_host.h>
+ #endif
#include "strata/dsv41/exl3.hpp"
#include <cuda_fp16.h>
@@@ -1266,7 -1269,7 +1270,10 @@@ int sm_count()
// token path and the verify windows take the same family on a device, so a window stays bit for bit its token path.
// Asked once per device.
bool use_legacy() {
--#if defined(STRATA_USE_HIP)
++#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)
++ // HIP has no v1.1.8 kernels. Metal has neither them nor a compute capability to ask for: the shim answers
++ // cudaDeviceGetAttribute with an error, so `major` stayed 0 and every Metal run without STRATA_GLM_EXL3_DECODE=new
++ // took this branch and aborted in exl3::legacy::prepare's stub. The 2.25 bpw pack never uses EXL3.
return false;
#else
static std::atomic<int> cache[64]; // 0 not asked, 1 v1.1.8's kernels, 2 this file's
diff --cc src/dsv41/vendor/exl3_legacy.cu
index bcc389b1e,0a028bbda..a35a9a1f4
--- a/src/dsv41/vendor/exl3_legacy.cu
+++ b/src/dsv41/vendor/exl3_legacy.cu
@@@ -19,6 -19,6 +19,7 @@@
// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in
// split order (deterministic) and runs the output rotation.
#if !defined(STRATA_USE_HIP) // (AMD runs exl3.cu's kernels; EXL3 does not run there anyway)
++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "strata/dsv41/exl3.hpp"
#include <cuda_fp16.h>
diff --cc src/dsv41/vendor/moe_win.cu
index fadbad001,1395738d0..6254e1fd1
--- a/src/dsv41/vendor/moe_win.cu
+++ b/src/dsv41/vendor/moe_win.cu
@@@ -2,6 -2,6 +2,7 @@@
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/dsv41/vendor/moe_win.cu - the window's expert grouping kernel.
// Extracted from Project Maya's src/kernels/cuda/glm_fast.cu (MIT, see third_party/project-maya/NOTICE.md).
++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "strata/dsv41/moe_win.hpp"
#include <cstdio>
#include <cstdlib>
diff --cc src/dsv41/vendor/sampler.cu
index b406df584,302637ec7..60062fbd5
--- a/src/dsv41/vendor/sampler.cu
+++ b/src/dsv41/vendor/sampler.cu
@@@ -15,11 -15,19 +15,20 @@@
// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,
// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /
// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).
++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal
#include "strata/dsv41/sampler.hpp"
+ #if defined(STRATA_METAL)
+ #include <ds4_metal_host.h>
+ #endif
#include <cuda_runtime.h>
+ #if defined(STRATA_METAL)
+ #include <strata/gpu/runtime.hpp>
+ #endif
+ #include <algorithm>
#include <cmath>
+ #include <vector>
#include <cstdio>
#include <cstdlib>
diff --cc strata_control/dsv41_fit.py
index ef7f354d8,f9dcbaeb0..69492c15f
--- a/strata_control/dsv41_fit.py
+++ b/strata_control/dsv41_fit.py
@@@ -478,3 -226,3 +478,87 @@@ def options(cards: list[dict], ram_gb:
opts[0]["recommended"] = True
out["options"], out["recommended"] = opts, "all"
return out
++
++
++# ---------------------------------------------------------------------------------------------------------------------
++# Apple Silicon: one pool of unified memory, the routed experts are file-backed and stay resident up to the engine's cap.
++# The numbers are the three GSQ-RCO packs (the research plan's table, measured from the public allocations) and the
++# engine's own cap rule (src/dsv41/ds4_model.cu metal_expert_cap, metal/src/engine/arena_budget.h).
++
++MAC_PACKS: dict[str, dict[str, Any]] = {
++ "IQ1_M": {"name": "1.75 bpw (IQ1_M)", "routed_gb": 114.56, "backbone_gb": 120.72, "bpw": 1.75},
++ "IQ2_XXS": {"name": "2.00 bpw (IQ2_XXS), the default for most users", "routed_gb": 132.07, "backbone_gb": 138.24, "bpw": 2.00},
++ "IQ2_XS": {"name": "2.25 bpw (IQ2_XS), the quality one", "routed_gb": 149.06, "backbone_gb": 155.23, "bpw": 2.25},
++}
++MAC_ENGRAM_GB = 110.60 # the same Engram table for all three, read row by row off the SSD
++MAC_ENGINE_FOOTPRINT_GB = 10.0 # the process' anonymous memory after the load (Metal buffers, host buffers, the KV cache)
++MAC_DEVICE_SHARE = 0.75 # recommendedMaxWorkingSetSize as a share of the RAM (107.5 of 128 GB on the M5 Max)
++MAC_FILECACHE_FLOOR_GB = 37.2 # vm.vm_page_filecache_min on the 128 GB M5 Max
++MAC_HEADROOM_GB = 12.0 # beside that floor: macOS itself and one more program
++MAC_MEASURED = ("M5 Max, 128 GB, 2.25 bpw, 2026-10-25: 14.4-15.8 tokens/s for an answer (12.4-12.6 on agent "
++ "work), a 3,000-token prompt at 380-382 tokens/s, 32,000 at 306; the engine kept 71.4 GB of the "
++ "experts resident and read 93.4 MB a token from the SSD")
++
++
++def mac_expert_cap_gb(ram_gb: float) -> float:
++ """The expert budget the engine will pick on a Mac of `ram_gb` (GiB, as the rest of the app reports memory), in
++ decimal GB: the RAM less what macOS keeps, less the process' own footprint. Mirrors metal_expert_cap() with its
++ big-Mac rule (>= 64 GB: 40% left, at least the file-cache floor + 12 GB). 128 GiB gives the 71.4 GB measured on
++ the M5 Max."""
++ gib = 1.073741824
++ p = float(ram_gb) * gib
++ if p <= 0:
++ return 0.0
++ keep = max(7.0, 0.15 * p)
++ leave = max(keep, 0.40 * p, MAC_FILECACHE_FLOOR_GB + MAC_HEADROOM_GB) if p >= 64.0 else keep
++ cap = min(p - leave, MAC_DEVICE_SHARE * p)
++ return max(0.0, cap - MAC_ENGINE_FOOTPRINT_GB - 1.0)
++
++
++def mac_plan(ram_gb: float, quant: str = "IQ2_XS", disk_free_gb: float | None = None) -> dict[str, Any]:
++ """Where the DeepSeek pack goes on an Apple Silicon Mac. Keys: quant, name, routed_gb, dense_gb, resident_gb,
++ streaming_gb, engram_gb, disk_need_gb, disk_ok, ram_gb, verdict (fits / streams / tight / small), lines, warnings,
++ words, measured. `fits` means every routed expert stays in memory; `streams` is the normal case and is still
++ allowed (the engine reads the rest from the SSD in place)."""
++ if quant not in MAC_PACKS:
++ raise ValueError(f"unknown Mac pack {quant!r}; one of {sorted(MAC_PACKS)}")
++ pack = MAC_PACKS[quant]
++ routed = float(pack["routed_gb"])
++ dense = round(float(pack["backbone_gb"]) - routed, 2)
++ cap = mac_expert_cap_gb(ram_gb or 0.0)
++ resident = min(routed, cap)
++ streaming = max(0.0, routed - resident)
++ disk_need = float(pack["backbone_gb"]) + MAC_ENGRAM_GB + 6.0 # the shards, the Engram table, the pack folder
++ out: dict[str, Any] = {"quant": quant, "name": pack["name"], "routed_gb": round(routed, 1), "dense_gb": dense,
++ "resident_gb": round(resident, 1), "streaming_gb": round(streaming, 1),
++ "engram_gb": MAC_ENGRAM_GB, "disk_need_gb": round(disk_need, 1),
++ "disk_ok": None if disk_free_gb is None else float(disk_free_gb) >= disk_need,
++ "ram_gb": ram_gb, "measured": MAC_MEASURED, "lines": [], "warnings": []}
++ lines, warns = out["lines"], out["warnings"]
++ lines.append(f"DeepSeek-V4.1-Flash {pack['name']}: {routed:.0f} GB of routed experts and {dense:.1f} GB of dense "
++ "weights, plus a 111 GB Engram table that stays on the SSD and is read a few rows for each word.")
++ if ram_gb and ram_gb >= 96:
++ lines.append(f"This Mac ({ram_gb:.0f} GB): the engine will keep about {resident:.0f} GB of the experts in memory "
++ f"and read the other {streaming:.0f} GB from the SSD while it answers.")
++ elif ram_gb:
++ lines.append(f"This Mac ({ram_gb:.0f} GB) can hold about {resident:.0f} GB of the experts; the other "
++ f"{streaming:.0f} GB stream from the SSD, which is slow on a small Mac.")
++ if cap <= 0:
++ out["verdict"] = "small"
++ warns.append("Too little memory to hold a useful part of the model: 64 GB or more is wanted.")
++ elif streaming == 0:
++ out["verdict"] = "fits"
++ lines.append("Every routed expert stays in memory: the fastest case.")
++ elif resident < 0.25 * routed:
++ out["verdict"] = "small"
++ warns.append(f"Only about {resident:.0f} GB of {routed:.0f} GB of experts fit in memory (under a quarter): "
++ "most of the model is read from the SSD for every word. Expect seconds a word rather than tokens a second.")
++ else:
++ out["verdict"] = "streams"
++ lines.append("The model streams: answers are slower than on a machine that holds all the experts, and they still "
++ "work. An NVMe SSD is wanted, not a USB or network drive.")
++ if disk_free_gb is not None and disk_free_gb < disk_need:
++ warns.append(f"The files need about {disk_need:.0f} GB of free disk space and {disk_free_gb:.0f} GB are free.")
++ lines.append("Measured on " + MAC_MEASURED + ".")
++ out["words"] = " ".join(lines)
++ return out
diff --cc tests/terminal/test_dsv41_fit_mac.py
index 000000000,000000000..3f2e0cfb0
new file mode 100644
--- /dev/null
+++ b/tests/terminal/test_dsv41_fit_mac.py
@@@ -1,0 -1,0 +1,72 @@@
++"""DeepSeek-V4.1-Flash on an Apple Silicon Mac: the fit numbers (`strata_control/dsv41_fit.py`, `mac_expert_cap_gb` and
++`mac_plan`). The cap mirrors the engine's own rule, so 128 GiB must give the 71.4 GB of VRAM experts measured on the M5
++Max on 2026-10-25. Pure: nothing here builds, downloads or starts anything."""
++
++from __future__ import annotations
++
++import unittest
++
++from strata_control import dsv41_fit as F
++
++
++class MacExpertCap(unittest.TestCase):
++ def test_the_m5_max_gives_the_measured_71_4_gb(self):
++ self.assertAlmostEqual(F.mac_expert_cap_gb(128), 71.4, delta=0.3)
++
++ def test_a_32_gb_mac_mini_keeps_little(self):
++ # 32 GiB: leave = 15% (4.8 GiB = 5.15 GB) and the working set caps it at 3/4 of 34.4 GB
++ cap = F.mac_expert_cap_gb(32)
++ self.assertGreater(cap, 0.0)
++ self.assertLess(cap, 131.0 * 0.25) # well under a quarter of the 2.00 pack
++ self.assertLess(F.mac_expert_cap_gb(16), F.mac_expert_cap_gb(32))
++ self.assertEqual(F.mac_expert_cap_gb(0), 0.0)
++
++ def test_a_big_mac_stops_at_the_working_set(self):
++ # 512 GiB: the file-cache floor rule leaves 40% and the device share is the tighter of the two? no: 40% wins
++ cap = F.mac_expert_cap_gb(512)
++ self.assertGreater(cap, 300.0) # far more than any pack needs
++
++
++class MacPlan(unittest.TestCase):
++ def test_the_default_pack_on_the_m5_max_streams(self):
++ p = F.mac_plan(128, "IQ2_XS")
++ self.assertEqual(p["verdict"], "streams")
++ self.assertAlmostEqual(p["resident_gb"], 71.4, delta=0.3)
++ self.assertAlmostEqual(p["streaming_gb"], 149.06 - 71.4, delta=0.3)
++ self.assertEqual(p["dense_gb"], 6.17)
++ self.assertIn("streams", p["words"])
++ self.assertIn("M5 Max", p["words"])
++
++ def test_the_1_75_pack_streams_less_on_the_same_mac(self):
++ small = F.mac_plan(128, "IQ2_XS")["streaming_gb"]
++ self.assertLess(F.mac_plan(128, "IQ1_M")["streaming_gb"], small)
++ self.assertGreater(F.mac_plan(128, "IQ2_XXS")["streaming_gb"], F.mac_plan(128, "IQ1_M")["streaming_gb"])
++
++ def test_a_mac_with_enough_memory_fits(self):
++ p = F.mac_plan(512, "IQ2_XS")
++ self.assertEqual(p["verdict"], "fits")
++ self.assertEqual(p["streaming_gb"], 0.0)
++
++ def test_a_small_mac_is_told_plainly(self):
++ p = F.mac_plan(32, "IQ2_XS")
++ self.assertEqual(p["verdict"], "small")
++ self.assertTrue(p["warnings"])
++ self.assertTrue(any("read from the SSD" in w or "Expert" in w or "quarter" in w for w in p["warnings"]))
++
++ def test_disk_and_unknown_pack(self):
++ p = F.mac_plan(128, "IQ2_XS", disk_free_gb=50.0)
++ self.assertFalse(p["disk_ok"])
++ self.assertTrue(any("disk space" in w for w in p["warnings"]))
++ self.assertTrue(F.mac_plan(128, "IQ2_XS", disk_free_gb=900.0)["disk_ok"])
++ with self.assertRaises(ValueError):
++ F.mac_plan(128, "IQ9_Z")
++
++ def test_every_pack_is_described(self):
++ for q in F.MAC_PACKS:
++ p = F.mac_plan(128, q)
++ self.assertTrue(p["words"])
++ self.assertGreater(p["disk_need_gb"], F.MAC_ENGRAM_GB)
++
++
++if __name__ == "__main__":
++ unittest.main()
diff --cc third_party/ggml/ggml-common.h
index 1dbbe326d,1dbbe326d..ca283d18e
--- a/third_party/ggml/ggml-common.h
+++ b/third_party/ggml/ggml-common.h
@@@ -1131,7 -1131,7 +1131,7 @@@ GGML_TABLE_END(
#define NGRID_IQ1S 2048
#define IQ1S_DELTA 0.125f
#define IQ1M_DELTA 0.125f
--#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)
++#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP) || defined(GGML_COMMON_IMPL_METAL) // (Strata: ds4_iq.metal's IQ1_M matvec and moe_mm.metal's tile read iq1s_grid)
GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)
0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,
0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,